hey, I am using prefect open-source on k8s with he...
# ask-community
r
hey, I am using prefect open-source on k8s with helm, I have one flow that run each day, after couples of days the ui stop working well and responding/ not showing data, also the pod cpu is looking very high and I dont understand why ( around 1.1 c), when I look on the pod server logs I see a lot of errors like this:
Copy code
07:35:21.231 | ERROR   | prefect.server.services.cancellationcleanup - Unexpected error in: TimeoutError()
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/prefect/server/services/loop_service.py", line 78, in start
    await self.run_once()
  File "/usr/local/lib/python3.10/site-packages/prefect/server/database/dependencies.py", line 119, in async_wrapper
    return await fn(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/prefect/server/services/cancellation_cleanup.py", line 48, in run_once
    await self.clean_up_cancelled_subflow_runs(db)
  File "/usr/local/lib/python3.10/site-packages/prefect/server/services/cancellation_cleanup.py", line 96, in clean_up_cancelled_subflow_runs
    subflow_run_result = await session.execute(subflow_query)
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/ext/asyncio/session.py", line 436, in execute
    result = await greenlet_spawn(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 190, in greenlet_spawn
    result = context.throw(*sys.exc_info())
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 2225, in execute
    return self._execute_internal(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 2120, in _execute_internal
    result: Result[Any] = compile_state_cls.orm_execute_statement(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/context.py", line 293, in orm_execute_statement
    result = conn.execute(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1412, in execute
    return meth(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/sql/elements.py", line 483, in _execute_on_connection
    return connection._execute_clauseelement(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1635, in _execute_clauseelement
    ret = self._execute_context(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1844, in _execute_context
    return self._exec_single_context(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1984, in _exec_single_context
    self._handle_dbapi_exception(    raise exc_info[1].with_traceback(exc_info[2])
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1965, in _exec_single_context
    self.dialect.do_execute(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py", line 921, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 561, in execute
    self._adapt_connection.await_(
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 125, in await_only
    return current.driver.switch(awaitable)  # type: ignore[no-any-return]
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 185, in greenlet_spawn
    value = await result
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 540, in _prepare_and_execute
    self._handle_exception(error)
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 491, in _handle_exception
    self._adapt_connection._handle_exception(error)
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 780, in _handle_exception
    raise error
  File "/usr/local/lib/python3.10/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 528, in _prepare_and_execute
    self._rows = await prepared_stmt.fetch(*parameters)
  File "/usr/local/lib/python3.10/site-packages/asyncpg/prepared_stmt.py", line 176, in fetch
    data = await self.__bind_execute(args, 0, timeout)
  File "/usr/local/lib/python3.10/site-packages/asyncpg/prepared_stmt.py", line 241, in __bind_execute
    data, status, _ = await self.__do_execute(
  File "/usr/local/lib/python3.10/site-packages/asyncpg/prepared_stmt.py", line 230, in __do_execute
    return await executor(protocol)
  File "asyncpg/protocol/protocol.pyx", line 201, in bind_execute
asyncio.exceptions.TimeoutError