https://prefect.io logo
Join Slack
Powered by
Hello All! Have you seen the error `ERROR | prefe...
# ask-community
m
Hello All! Have you seen the error 
ERROR | prefect.server.events.services.event_persister - Error trimming events and resources
?
🙅‍♂️ 1
Copy code
prefect.server.events.services.event_persister - Error trimming events and resources
    Traceback (most recent call last):
    File "/usr/local/lib/python3.14/site-packages/prefect/server/events/services/event_persister.py", line 184, in trim
    resource_count = await batch_delete(
    ^^^^^^^^^^^^^^^^^^^
    ...<4 lines>...
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/prefect/server/events/services/event_persister.py", line 71, in batch_delete
    result = await session.execute(delete_stmt)
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/ext/asyncio/session.py", line 449, in execute
    result = await greenlet_spawn(
    ^^^^^^^^^^^^^^^^^^^^^
    ...<6 lines>...
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 203, in greenlet_spawn
    result = context.switch(value)
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/session.py", line 2351, in execute
    return self._execute_internal(
    ~~~~~~~~~~~~~~~~~~~~~~^
    statement,
    ^^^^^^^^^^
    ...<4 lines>...
    _add_event=_add_event,
    ^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/session.py", line 2249, in _execute_internal
    result: Result[Any] = compile_state_cls.orm_execute_statement(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    self,
    ^^^^^
    ...<4 lines>...
    conn,
    ^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/bulk_persistence.py", line 2033, in orm_execute_statement
    return super().orm_execute_statement(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    session, statement, params, execution_options, bind_arguments, conn
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/context.py", line 306, in orm_execute_statement
    result = conn.execute(
    statement, params or {}, execution_options=execution_options
    )
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1419, in execute
    return meth(
    self,
    distilled_parameters,
    execution_options or NO_OPTIONS,
    )
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection
    return connection._execute_clauseelement(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    self, distilled_params, execution_options
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1641, in _execute_clauseelement
    ret = self._execute_context(
    dialect,
    ...<8 lines>...
    cache_hit=cache_hit,
    )
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
    return self._exec_single_context(
    ~~~~~~~~~~~~~~~~~~~~~~~~~^
    dialect, context, statement, parameters
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1986, in _exec_single_context
    self._handle_dbapi_exception(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    e, str_statement, effective_parameters, cursor, context
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 2358, in _handle_dbapi_exception
    raise exc_info[1].with_traceback(exc_info[2])
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1967, in _exec_single_context
    self.dialect.do_execute(
    ~~~~~~~~~~~~~~~~~~~~~~~^
    cursor, str_statement, effective_parameters, context
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/default.py", line 951, in do_execute
    cursor.execute(statement, parameters)
    ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 585, in execute
    self._adapt_connection.await_(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
    self._prepare_and_execute(operation, parameters)
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 132, in await_only
    return current.parent.switch(awaitable) # type: ignore[no-any-return,attr-defined] # noqa: E501
    ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 196, in greenlet_spawn
    value = await result
    ^^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 563, in _prepare_and_execute
    self._handle_exception(error)
    ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 513, in _handle_exception
    self._adapt_connection._handle_exception(error)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 799, in _handle_exception
    raise error
    File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 550, in _prepare_and_execute
    self._rows = deque(await prepared_stmt.fetch(*parameters))
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/asyncpg/prepared_stmt.py", line 176, in fetch
    data = await self.__bind_execute(args, 0, timeout)
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/asyncpg/prepared_stmt.py", line 267, in __bind_execute
    data, status, _ = await self.__do_execute(
    ^^^^^^^^^^^^^^^^^^^^^^^^
    lambda protocol: protocol.bind_execute(
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    self._state, args, '', limit, True, timeout))
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    File "/usr/local/lib/python3.14/site-packages/asyncpg/prepared_stmt.py", line 256, in __do_execute
    return await executor(protocol)
    ^^^^^^^^^^^^^^^^^^^^^^^^
    File "asyncpg/protocol/protocol.pyx", line 206, in bind_execute
    TimeoutError
n
hi @Mattia Sappa - what version of prefect server do you have? also how big is your events table?
m
Hello! I am running on 3.5.0 this is the size of the biggest tables
Copy code
`event_resources` 11 GB
`events` 12 GB
`artifact` 13 GB
`task_run` 19 GB
`log` 19 GB
`task_run_state` 26 GB
I have a flow to delete older data and then run 
vacuum analyze {_table_};
I don't do 
vacuum full
I've increased the database timeout to
Copy code
"timeout": 60,
"connection_timeout": 30
Now I mostly get 
ERROR: Exception in ASGI application
Copy code
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/usr/local/lib/python3.14/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
self.scope, self.receive, self.send
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await <http://self.app|self.app>(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/fastapi/applications.py", line 1134, in __call__
await super().__call__(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/applications.py", line 113, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/errors.py", line 164, in __call__
await <http://self.app|self.app>(scope, receive, _send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/cors.py", line 93, in __call__
await self.simple_response(scope, receive, send, request_headers=headers)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/cors.py", line 144, in simple_response
await <http://self.app|self.app>(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(<http://self.app|self.app>, conn)(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.14/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.14/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await <http://self.app|self.app>(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/routing.py", line 716, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/routing.py", line 462, in handle
await <http://self.app|self.app>(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/fastapi/applications.py", line 1134, in __call__
await super().__call__(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/applications.py", line 113, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/errors.py", line 164, in __call__
await <http://self.app|self.app>(scope, receive, _send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/base.py", line 191, in __call__
with recv_stream, send_stream, collapse_excgroups():
~~~~~~~~~~~~~~~~~~^^
File "/usr/local/lib/python3.14/contextlib.py", line 162, in __exit__
self.gen.throw(value)
~~~~~~~~~~~~~~^^^^^^^
File "/usr/local/lib/python3.14/site-packages/starlette/_utils.py", line 85, in collapse_excgroups
raise exc
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/base.py", line 193, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/prefect/server/api/server.py", line 455, in token_validation
return await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/base.py", line 168, in call_next
raise app_exc from app_exc.__cause__ or app_exc.__context__
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/base.py", line 144, in coro
await <http://self.app|self.app>(scope, receive_or_disconnect, send_no_error)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/gzip.py", line 29, in __call__
await responder(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/gzip.py", line 130, in __call__
await super().__call__(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/gzip.py", line 46, in __call__
await <http://self.app|self.app>(scope, receive, self.send_with_compression)
File "/usr/local/lib/python3.14/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(<http://self.app|self.app>, conn)(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.14/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.14/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await <http://self.app|self.app>(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/routing.py", line 716, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/routing.py", line 290, in handle
await <http://self.app|self.app>(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/fastapi/routing.py", line 124, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/usr/local/lib/python3.14/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.14/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.14/site-packages/fastapi/routing.py", line 110, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/prefect/server/utilities/server.py", line 50, in handle_response_scoped_depends
response = await default_handler(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/fastapi/routing.py", line 390, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<3 lines>...
)
^
File "/usr/local/lib/python3.14/site-packages/fastapi/routing.py", line 289, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/prefect/server/api/task_runs.py", line 123, in count_task_runs
return await models.task_runs.count_task_runs(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<5 lines>...
)
^
File "/usr/local/lib/python3.14/site-packages/prefect/server/models/task_runs.py", line 393, in count_task_runs
result = await session.execute(query)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/ext/asyncio/session.py", line 449, in execute
result = await greenlet_spawn(
^^^^^^^^^^^^^^^^^^^^^
...<6 lines>...
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 203, in greenlet_spawn
result = context.switch(value)
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/session.py", line 2351, in execute
return self._execute_internal(
~~~~~~~~~~~~~~~~~~~~~~^
statement,
^^^^^^^^^^
...<4 lines>...
_add_event=_add_event,
^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/session.py", line 2249, in _execute_internal
result: Result[Any] = compile_state_cls.orm_execute_statement(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self,
^^^^^
...<4 lines>...
conn,
^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/orm/context.py", line 306, in orm_execute_statement
result = conn.execute(
statement, params or {}, execution_options=execution_options
)
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1419, in execute
return meth(
self,
distilled_parameters,
execution_options or NO_OPTIONS,
)
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection
return connection._execute_clauseelement(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self, distilled_params, execution_options
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1641, in _execute_clauseelement
ret = self._execute_context(
dialect,
...<8 lines>...
cache_hit=cache_hit,
)
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
return self._exec_single_context(
~~~~~~~~~~~~~~~~~~~~~~~~~^
dialect, context, statement, parameters
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1986, in _exec_single_context
self._handle_dbapi_exception(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
e, str_statement, effective_parameters, cursor, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 2358, in _handle_dbapi_exception
raise exc_info[1].with_traceback(exc_info[2])
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/base.py", line 1967, in _exec_single_context
self.dialect.do_execute(
~~~~~~~~~~~~~~~~~~~~~~~^
cursor, str_statement, effective_parameters, context
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/engine/default.py", line 951, in do_execute
cursor.execute(statement, parameters)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 585, in execute
self._adapt_connection.await_(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self._prepare_and_execute(operation, parameters)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 132, in await_only
return current.parent.switch(awaitable) # type: ignore[no-any-return,attr-defined] # noqa: E501
~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 196, in greenlet_spawn
value = await result
^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 563, in _prepare_and_execute
self._handle_exception(error)
~~~~~~~~~~~~~~~~~~~~~~^^^^^^^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 513, in _handle_exception
self._adapt_connection._handle_exception(error)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 799, in _handle_exception
raise error
File "/usr/local/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 550, in _prepare_and_execute
self._rows = deque(await prepared_stmt.fetch(*parameters))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/asyncpg/prepared_stmt.py", line 176, in fetch
data = await self.__bind_execute(args, 0, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/asyncpg/prepared_stmt.py", line 267, in __bind_execute
data, status, _ = await self.__do_execute(
^^^^^^^^^^^^^^^^^^^^^^^^
lambda protocol: protocol.bind_execute(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
self._state, args, '', limit, True, timeout))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/site-packages/asyncpg/prepared_stmt.py", line 256, in __do_execute
return await executor(protocol)
^^^^^^^^^^^^^^^^^^^^^^^^
File "asyncpg/protocol/protocol.pyx", line 206, in bind_execute
TimeoutError
3 Views
Open in Slack
PreviousNext
Powered by