Undiagnosable error while moving from Prefect-Agents to ECS Work Pool on self-hosted ECS Prefect stack - Need help I am trying to get to a point of running test code from https://prefecthq.github.io/prefect-aws/ecs_guide/ and https://docs.prefect.io/latest/concepts/infrastructure/#kubernetesjob-overrides-and-customizations. The same code that can run off Prefect docker image is on S3 and I have a ecs-work-pool running on ECS cluster. I am running into trimmed out exception from Cloudwatch logs

timestamp,message
1717774576769,Traceback (most recent call last):
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", line 69, in map_httpcore_exceptions"
1717774576769,    yield
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", line 373, in handle_async_request"
1717774576769,    resp = await self._pool.handle_async_request(req)
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py"", line 216, in handle_async_request"
1717774576769,    raise exc from None
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py"", line 196, in handle_async_request"
1717774576769,    response = await connection.handle_async_request(
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py"", line 99, in handle_async_request"
1717774576769,    raise exc
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py"", line 76, in handle_async_request"
1717774576769,    stream = await self._connect(request)
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py"", line 122, in _connect"
1717774576769,    stream = await self._network_backend.connect_tcp(**kwargs)
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_backends/auto.py"", line 30, in connect_tcp"
1717774576769,    return await self._backend.connect_tcp(
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_backends/anyio.py"", line 114, in connect_tcp"
1717774576769,    with map_exceptions(exc_map):
1717774576769,"  File ""/usr/local/lib/python3.10/contextlib.py"", line 153, in __exit__"
1717774576769,"    self.gen.throw(typ, value, traceback)"
1717774576769,"  File ""/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py"", line 14, in map_exceptions"
1717774576769,    raise to_exc(exc) from exc
1717774576769,httpcore.ConnectError: [Errno -2] Name or service not known
1717774576770,The above exception was the direct cause of the following exception:
1717774576770,Traceback (most recent call last):
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/cli/_utilities.py"", line 42, in wrapper"
1717774576770,"    return fn(*args, **kwargs)"
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/utilities/asyncutils.py"", line 304, in coroutine_wrapper"
1717774576770,    return call()
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 432, in __call__"
1717774576770,    return self.result()
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 318, in result"
1717774576770,    return self.future.result(timeout=timeout)
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 179, in result"
1717774576770,    return self.__get_result()
1717774576770,"  File ""/usr/local/lib/python3.10/concurrent/futures/_base.py"", line 403, in __get_result"
1717774576770,    raise self._exception
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/_internal/concurrency/calls.py"", line 389, in _run_async"
1717774576770,    result = await coro
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/cli/flow_run.py"", line 348, in execute"
1717774576770,    await Runner().execute_flow_run(id)
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/runner/runner.py"", line 479, in execute_flow_run"
1717774576770,    flow_run = await self._client.read_flow_run(flow_run_id)
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py"", line 2013, in read_flow_run"
1717774576770,"    response = await self._client.get(f""/flow_runs/{flow_run_id}"")"
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1801, in get"
1717774576770,    return await self.request(
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1574, in request"
1717774576770,"    return await self.send(request, auth=auth, follow_redirects=follow_redirects)"
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/client/base.py"", line 327, in send"
1717774576770,    response = await self._send_with_retry(
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/prefect/client/base.py"", line 251, in _send_with_retry"
1717774576770,"    response = await send(request, *send_args, **send_kwargs)"
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1661, in send"
1717774576770,    response = await self._send_handling_auth(
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1689, in _send_handling_auth"
1717774576770,    response = await self._send_handling_redirects(
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1726, in _send_handling_redirects"
1717774576770,    response = await self._send_single_request(request)
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_client.py"", line 1763, in _send_single_request"
1717774576770,    response = await transport.handle_async_request(request)
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", line 372, in handle_async_request"
1717774576770,    with map_httpcore_exceptions():
1717774576770,"  File ""/usr/local/lib/python3.10/contextlib.py"", line 153, in __exit__"
1717774576770,"    self.gen.throw(typ, value, traceback)"
1717774576770,"  File ""/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py"", line 86, in map_httpcore_exceptions"
1717774576770,    raise mapped_exc(message) from exc
1717774576770,httpx.ConnectError: [Errno -2] Name or service not known
1717774576777,An exception occurred.

Can someone help me successfully move to ECS work pool, if this works I've got an ECR image backed Prefect source code that needs to work in this cluster

Looks like a DNS error or something. Is your Prefect API reachable?

I was suspecting that, my server is running off ServiceDiscovery server:4200/api on the ECS cluster. I've put in an internal Load Balancer to allow ECS tasks to connect to Prefect API which worked through. I've been trying to debug the Python code from ECS Worker but I don't think ServiceDiscovery is looked up while parsing the configuration.