Hi Community,
We need help with Prefect agent setup.
We have hosted Prefect server in EC2 instance and created prefect agent in ECS Faregate.
We are using docker image as “prefecthq/prefect:2.10.8-python3.10”.
We configured environment variables "‘PREFECT_API_URL’ , PREFECT_CLOUD_API’ ‘PREFECT_LOGGING_LEVEL’:INFO’,‘AWS_RETRY_MODE’:‘adaptive’,‘AWS_MAX_ATTEMPTS’:‘10’ in ECS Fargate Agent Service and run ""prefect agent start -p default-agent-pool -q default"" command through command line.
In ECS Fargate while running services for agent we are getting below error.
Error logs :
Agent started! Looking for work from work pool ‘default-agent-pool’…
Failed the last 3 attempts. Please check your environment and configuration.
Examples of recent errors:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 111, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_sockets.py”, line
213, in connect_tcp
async with create_task_group() as tg:
File “/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”,
line 658, in aexit
raise CancelledError
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
10, in map_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 110, in connect_tcp
with anyio.fail_after(timeout):
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_tasks.py”, line
118, in exit
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 60, in map_httpcore_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 253, in handle_async_request
raise exc
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 237, in handle_async_request
response = await connection.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 86, in handle_async_request
raise exc
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 63, in handle_async_request
stream = await self._connect(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/auto.py”, line
29, in connect_tcp
return await self._backend.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 109, in connect_tcp
with map_exceptions(exc_map):
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
14, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/prefect/utilities/services.py”,
line 46, in critical_service_loop
await workload()
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 196, in
get_and_submit_flow_runs
work_queue_names=,
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 196, in
work_queue_names=,
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 144, in
get_work_queues
work_queue = await self.client.read_work_queue_by_name(
File
“/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py”, line
844, in read_work_queue_by_name
response = await self._client.get(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1754, in
get
return await self.request(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1530, in
request
return await self.send(request, auth=auth,
follow_redirects=follow_redirects)
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
246, in send
response = await self._send_with_retry(
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
192, in _send_with_retry
response = await request()
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1617, in
send
response = await self._send_handling_auth(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1645, in
_send_handling_auth
response = await self._send_handling_redirects(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1682, in
_send_handling_redirects
response = await self._send_single_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1719, in
_send_single_request
response = await transport.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 352, in handle_async_request
with map_httpcore_exceptions():
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
Failed the last 3 attempts. Please check your environment and configuration.
Examples of recent errors:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 111, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_sockets.py”, line
213, in connect_tcp
async with create_task_group() as tg:
File “/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”,
line 658, in aexit
raise CancelledError
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
10, in map_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 110, in connect_tcp
with anyio.fail_after(timeout):
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_tasks.py”, line
118, in exit
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 60, in map_httpcore_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 253, in handle_async_request
raise exc
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 237, in handle_async_request
response = await connection.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 86, in handle_async_request
raise exc
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 63, in handle_async_request
stream = await self._connect(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/auto.py”, line
29, in connect_tcp
return await self._backend.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 109, in connect_tcp
with map_exceptions(exc_map):
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
14, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/prefect/utilities/services.py”,
line 46, in critical_service_loop
await workload()
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 261, in
check_for_cancelled_flow_runs
async for work_queue in self.get_work_queues():
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 144, in
get_work_queues
work_queue = await self.client.read_work_queue_by_name(
File
“/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py”, line
844, in read_work_queue_by_name
response = await self._client.get(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1754, in
get
return await self.request(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1530, in
request
return await self.send(request, auth=auth,
follow_redirects=follow_redirects)
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
246, in send
response = await self._send_with_retry(
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
192, in _send_with_retry
response = await request()
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1617, in
send
response = await self._send_handling_auth(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1645, in
_send_handling_auth
response = await self._send_handling_redirects(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1682, in
_send_handling_redirects
response = await self._send_single_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1719, in
_send_single_request
response = await transport.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 352, in handle_async_request
with map_httpcore_exceptions():
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
Agent stopped!
Thanks,
Deep