The UI on flow runs has stopped displaying the flow run graph and only shows "This flow run did not generate any task or subflow runs" even though a task has been executed. Any ideas on how to troubleshoot and identify the root cause to get this view back? We are using Prefect Cloud on Kubernetes infra with Prefect version 3.0. Our subflows do produce a result correctly, so it is just tasks.

(prefect 2.0) We're reliably seeing 503 errors when trying to delete deployments. I suspect that this is happening bc these deployments have many flow runs associated with them. debug info in 🧵

Hey team, can you help me getting around this issue I'm having in the past few weeks on a Prefect Agent constantly stopping after this error:

Failed the last 3 attempts. Please check your environment and configuration.
Examples of recent errors:
Traceback (most recent call last):
"  File ""/usr/local/lib/python3.10/site-packages/h2/connection.py"", line 224, in "
process_input
"    func, target_state = self._transitions[(self.state, input_)]"
"KeyError: (<ConnectionState.CLOSED: 3>, <ConnectionInputs.SEND_HEADERS: 0>)"
"During handling of the above exception, another exception occurred:"
Traceback (most recent call last):
"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/http2.py"", line "
"116, in handle_async_request"
"    await self._send_request_headers(request=request, stream_id=stream_id)"
"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/http2.py"", line "
"213, in _send_request_headers"
"    self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)"
"  File ""/usr/local/lib/python3.10/site-packages/h2/connection.py"", line 766, in "
send_headers
    self.state_machine.process_input(ConnectionInputs.SEND_HEADERS)
"  File ""/usr/local/lib/python3.10/site-packages/h2/connection.py"", line 228, in "
process_input
    raise ProtocolError(
h2.exceptions.ProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in 
state ConnectionState.CLOSED
"During handling of the above exception, another exception occurred:"
Traceback (most recent call last):
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", "
"line 60, in map_httpcore_exceptions"
    yield
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", "
"line 353, in handle_async_request"
    resp = await self._pool.handle_async_request(req)
  File 
"""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py"", "
"line 253, in handle_async_request"
    raise exc
  File 
"""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py"", "
"line 237, in handle_async_request"
    response = await connection.handle_async_request(request)
"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py"", "
"line 90, in handle_async_request"
    return await self._connection.handle_async_request(request)
"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/http2.py"", line "
"152, in handle_async_request"
    raise LocalProtocolError(exc)  # pragma: nocover
httpcore.LocalProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in 
state ConnectionState.CLOSED
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
"  File ""/usr/local/lib/python3.10/site-packages/prefect/utilities/services.py"", "
"line 46, in critical_service_loop"
    await workload()
"  File ""/usr/local/lib/python3.10/site-packages/prefect/agent.py"", line 261, in "
check_for_cancelled_flow_runs
    async for work_queue in self.get_work_queues():
"  File ""/usr/local/lib/python3.10/site-packages/prefect/agent.py"", line 144, in "
get_work_queues
    work_queue = await self.client.read_work_queue_by_name(
  File 
"""/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py"", line "
"850, in read_work_queue_by_name"
"    response = await self._client.get(f""/work_queues/name/{name}"")"
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1754, in"
get
    return await self.request(
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1530, in"
request
"    return await self.send(request, auth=auth, "
follow_redirects=follow_redirects)
"  File ""/usr/local/lib/python3.10/site-packages/prefect/client/base.py"", line "
"251, in send"
    response = await self._send_with_retry(
"  File ""/usr/local/lib/python3.10/site-packages/prefect/client/base.py"", line "
"194, in _send_with_retry"
    response = await request()
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1617, in"
send
    response = await self._send_handling_auth(
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1645, in"
_send_handling_auth
    response = await self._send_handling_redirects(
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1682, in"
_send_handling_redirects
    response = await self._send_single_request(request)
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1719, in"
_send_single_request
    response = await transport.handle_async_request(request)
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", "
"line 352, in handle_async_request"
    with map_httpcore_exceptions():
"  File ""/usr/local/lib/python3.10/contextlib.py"", line 153, in __exit__"
"    self.gen.throw(typ, value, traceback)"
"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", "
"line 77, in map_httpcore_exceptions"
    raise mapped_exc(message) from exc
httpx.LocalProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in state 
ConnectionState.CLOSED
Traceback (most recent call last):
"  File ""/usr/local/lib/python3.10/site-packages/prefect/cli/_utilities.py"", line 41, in wrapper"
"    return fn(*args, **kwargs)"
"  File ""/usr/local/lib/python3.10/site-packages/prefect/utilities/asyncutils.py"", line 260, in coroutine_wrapper"
    return call()
"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 245, in __call__"
    return self.result()
"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 173, in result"
    return self.future.result(timeout=timeout)
"  File ""/usr/local/lib/python3.10/concurrent/futures/_base.py"", line 451, in result"
    return self.__get_result()
"  File ""/usr/local/lib/python3.10/concurrent/futures/_base.py"", line 403, in __get_result"
    raise self._exception
"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 218, in _run_async"
    result = await coro
"  File ""/usr/local/lib/python3.10/site-packages/prefect/cli/agent.py"", line 189, in start"
    async with anyio.create_task_group() as tg:
"  File ""/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py"", line 662, in __aexit__"
    raise exceptions[0]
"  File ""/usr/local/lib/python3.10/site-packages/prefect/utilities/services.py"", line 104, in critical_service_loop"
"    raise RuntimeError(""Service exceeded error threshold."")"
RuntimeError: Service exceeded error threshold.
An exception occurred.

I'm running the Agent as an ECS Service on Fargate, so it's always starting again but with unexpected random behavior, sometimes the flows keep frozen after this issue. I think this is happening after I updated to the 2.10.6 version of prefect. Before the update, the agent was not showing that Stopping behavior. Has anyone experienced something similar? I see in this thread some kind of similar error, but I'm not sure if doing the

prefect config set PREFECT_API_ENABLE_HTTP2=False

thing didn't actually solve that issue and I'm also not sure how to specify that on the ECS Task Definition? Should I add this config as an Environment Variable?

When setting up an automation in Prefect 2 cloud, is there a way to test to make sure it works? Or would you just have to wait for the event to happen to trigger it

Hi. Looking for some tips on self-hosted authentication / authorization (user logins, roles) when running our own Prefect Orion (2.0) server. We can't use Prefect Cloud due to compliance issues with using the SaaS offering (despite its hybrid security model), and this seems to be about the only box not checked for what we want the tool to do in standalone form. I don't want to have to open it up and bolt on FastAPI OAuth2 support to the web api. Any other solutions or workarounds to make it so that anyone on the network can't arbitrarily start, stop, modify flows? Maybe there's a way to allow all GET requests and block any POST/PUT/DELETE etc. to effectively make it read-only by default, and writable for privileged users?

@Marvin How can I fix the RuntimeWarning: coroutine 'run_flow_async' was never awaited when using await in my code?

Hello , I am trying to run a prefect in docker. I looked in this community. But it seems that there is no proper docker compose file for prefect. I was trying to use this for the prefect docker compose. I am very new to docker. So, I tried to write a docker compose to up and prefect. I am also learning prefect very recently. I tried to write the docker compose below following this example. Is there any proper docker compose.yml file that I can use to create nodes for prefect and view the prefect dashboard? The dashboard is not showing. I am looking for docker-compose.yml for prefect v3. Thank you in advance for any assistance. https://github.com/PrefectHQ/prefect-background-task-examples/blob/main/chaos-duck/docker-compose.yaml But I seem to be encountering this issue in the docker compose logs.

prefect  | During handling of the above exception, another exception occurred:
prefect  | 
prefect  | Traceback (most recent call last):
prefect  |   File "/usr/local/bin/prefect", line 8, in <module>
prefect  |     sys.exit(cli())
prefect  |   File "/usr/local/lib/python3.7/site-packages/click/core.py", line 1130, in __call__
prefect  |     return self.main(*args, **kwargs)
prefect  |   File "/usr/local/lib/python3.7/site-packages/click/core.py", line 1055, in main
prefect  |     rv = self.invoke(ctx)
prefect  |   File "/usr/local/lib/python3.7/site-packages/click/core.py", line 1657, in invoke
prefect  |     return _process_result(sub_ctx.command.invoke(sub_ctx))
prefect  |   File "/usr/local/lib/python3.7/site-packages/click/core.py", line 1657, in invoke
prefect  |     return _process_result(sub_ctx.command.invoke(sub_ctx))
prefect  |   File "/usr/local/lib/python3.7/site-packages/click/core.py", line 1404, in invoke
prefect  |     return ctx.invoke(self.callback, **ctx.params)
prefect  |   File "/usr/local/lib/python3.7/site-packages/click/core.py", line 760, in invoke
prefect  |     return __callback(*args, **kwargs)
prefect  |   File "/usr/local/lib/python3.7/site-packages/prefect/cli/server.py", line 674, in start
prefect  |     ["docker-compose", "down"], cwd=compose_dir_path, env=env
prefect  |   File "/usr/local/lib/python3.7/subprocess.py", line 411, in check_output
prefect  |     **kwargs).stdout
prefect  |   File "/usr/local/lib/python3.7/subprocess.py", line 488, in run
prefect  |     with Popen(*popenargs, **kwargs) as process:
prefect  |   File "/usr/local/lib/python3.7/subprocess.py", line 800, in __init__
prefect  |     restore_signals, start_new_session)
prefect  |   File "/usr/local/lib/python3.7/subprocess.py", line 1551, in _execute_child
prefect  |     raise child_exception_type(errno_num, err_msg, err_filename)
prefect  | FileNotFoundError: [Errno 2] No such file or directory: 'docker-compose': 'docker-compose'

My docker-compose.yml

services:

  postgres:
    image: postgres:16
    restart: always
    environment:
      POSTGRES_USER: prefect
      POSTGRES_PASSWORD: password
      POSTGRES_DB: prefect
    volumes:
      - postgres-data:/var/lib/postgresql/data
    networks:
      - prefect-net

  prefect:
    image: prefecthq/prefect:latest
    container_name: prefect
    restart: always
    environment:
      PREFECT_HOME: /data
      PREFECT_API_DATABASE_CONNECTION_URL: <postgresql+asyncpg://prefect:password@postgres/prefect>
    command: prefect server start
    ports:
      - "4200:4200"
    volumes:
      - prefect-data:/data
    depends_on:
      - postgres
    networks:
      - prefect-net

volumes:
  postgres-data:
  prefect-data:

networks:
  prefect-net:

@Marvin my flow run is stuck in "awaiting concurrency slot" state, but there is no other ongoing runs. Why?

@Marvin how to specify parameter during deployment using prefect.yaml

@Marvin I have a simple flow that users can call. However, when a user tried to create many flow runs, some of the flow runs had this error

Job 'piquant-stingray-49qqg': Job reached backoff limit.

what might have caused this?