Nimesh Kumar
04/18/2024, 8:59 AMTraceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/prefect/agent.py", line 568, in _propose_pending_state
state = await propose_state(self.client, Pending(), flow_run_id=flow_run.id)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/engine.py", line 2574, in propose_state
response = await set_state_and_handle_waits(set_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/engine.py", line 2558, in set_state_and_handle_waits
response = await set_state_func()
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/orchestration.py", line 2056, in set_flow_run_state
response = await <http://self._client.post|self._client.post>(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1848, in post
return await self.request(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 312, in send
response.raise_for_status()
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 164, in raise_for_status
raise PrefectHTTPStatusError.from_httpx_error(exc) from exc.__cause__
prefect.exceptions.PrefectHTTPStatusError: Server error '504 Gateway Timeout' for url '<https://orion-uat.services.carpl.ai/api/flow_runs/7f946f5c-495e-463f-b5a3-a82b29305be5/set_state>'
For more information check: <https://httpstatuses.com/504>
08:49:16.713 | ERROR | prefect.agent - Failed to get infrastructure for flow run '566552c3-282d-4d04-a46b-dd7903d71d5e'.
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/prefect/agent.py", line 471, in submit_run
infrastructure = await self.get_infrastructure(flow_run)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/agent.py", line 431, in get_infrastructure
infra_document = await self.client.read_block_document(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/orchestration.py", line 1400, in read_block_document
response = await self._client.get(
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1757, in get
return await self.request(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 312, in send
response.raise_for_status()
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 164, in raise_for_status
raise PrefectHTTPStatusError.from_httpx_error(exc) from exc.__cause__
prefect.exceptions.PrefectHTTPStatusError: Server error '504 Gateway Timeout' for url '<https://orion-uat.services.carpl.ai/api/block_documents/52a73a78-7d3a-464d-b189-b4dcc184658e?include_secrets=true>'
For more information check: <https://httpstatuses.com/504>
08:50:16.717 | ERROR | prefect.agent - Failed to update state of flow run '566552c3-282d-4d04-a46b-dd7903d71d5e'
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/prefect/agent.py", line 597, in _propose_failed_state
await propose_state(
File "/usr/local/lib/python3.11/site-packages/prefect/engine.py", line 2574, in propose_state
response = await set_state_and_handle_waits(set_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/engine.py", line 2558, in set_state_and_handle_waits
response = await set_state_func()
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/orchestration.py", line 2056, in set_flow_run_state
response = await <http://self._client.post|self._client.post>(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1848, in post
return await self.request(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 312, in send
response.raise_for_status()
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 164, in raise_for_status
raise PrefectHTTPStatusError.from_httpx_error(exc) from exc.__cause__
prefect.exceptions.PrefectHTTPStatusError: Server error '504 Gateway Timeout' for url '<https://orion-uat.services.carpl.ai/api/flow_runs/566552c3-282d-4d04-a46b-dd7903d71d5e/set_state>'
For more information check: <https://httpstatuses.com/504>
08:54:22.870 | ERROR | prefect.agent - Failed to create work queue 'algo-40'.
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/prefect/agent.py", line 159, in get_work_queues
work_queue = await self.client.create_work_queue(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/orchestration.py", line 907, in create_work_queue
response = await <http://self._client.post|self._client.post>(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1848, in post
return await self.request(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 312, in send
response.raise_for_status()
File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 164, in raise_for_status
raise PrefectHTTPStatusError.from_httpx_error(exc) from exc.__cause__
prefect.exceptions.PrefectHTTPStatusError: Server error '504 Gateway Timeout' for url '<https://orion-uat.services.carpl.ai/api/work_pools/normal-pool/queues>'
For more information check: <https://httpstatuses.com/504>