Hello. Iām starting to see a low of flows runs cra...
# prefect-communityj
Josh Paulin
09/16/2022, 6:00 PMHello. Iām starting to see a low of flows runs crashing when trying to update task state (500 error from https://api.prefect.cloud/api/accounts/<uuid>/workspaces/<uuid>/task_runs/<uuid>/set_state) Full stack trace in the thead. Is there any way to diagnose further?
š 2
ā
1
Josh Paulin
09/16/2022, 6:10 PM Copy code
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 595, in orchestrate_flow_run
result = await run_sync(flow_call)
File "/usr/local/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 57, in run_sync_in_worker_thread
return await anyio.to_thread.run_sync(call, cancellable=True)
File "/usr/local/lib/python3.9/site-packages/anyio/to_thread.py", line 31, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
return await future
File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 867, in run
result = context.run(func, *args)
File "activity_processor/flow/parent.py", line 35, in api_activity_processor
process_hourly_api(normalized_process_time, dry_run, validate_results)
File "/usr/local/lib/python3.9/site-packages/prefect/flows.py", line 384, in __call__
return enter_flow_run_engine_from_flow_call(
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 162, in enter_flow_run_engine_from_flow_call
return run_async_from_worker_thread(begin_run)
File "/usr/local/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 137, in run_async_from_worker_thread
return anyio.from_thread.run(call)
File "/usr/local/lib/python3.9/site-packages/anyio/from_thread.py", line 49, in run
return asynclib.run_async_from_thread(func, *args)
File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 970, in run_async_from_thread
return f.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 446, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/prefect/client.py", line 103, in with_injected_client
return await fn(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 521, in create_and_begin_subflow_run
return terminal_state.result()
File "/usr/local/lib/python3.9/site-packages/prefect/orion/schemas/states.py", line 143, in result
raise data
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 595, in orchestrate_flow_run
result = await run_sync(flow_call)
File "/usr/local/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 57, in run_sync_in_worker_thread
return await anyio.to_thread.run_sync(call, cancellable=True)
File "/usr/local/lib/python3.9/site-packages/anyio/to_thread.py", line 31, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
return await future
File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 867, in run
result = context.run(func, *args)
File "/project/activity_processor/flow/process_hourly.py", line 44, in process_hourly_api
counts = [mapper.count_events(k, source_bucket) for k in unprocessed_api_keys]
File "/project/activity_processor/flow/process_hourly.py", line 44, in <listcomp>
counts = [mapper.count_events(k, source_bucket) for k in unprocessed_api_keys]
File "/usr/local/lib/python3.9/site-packages/prefect/tasks.py", line 295, in __call__
return enter_task_run_engine(
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 735, in enter_task_run_engine
return run_async_from_worker_thread(begin_run)
File "/usr/local/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 137, in run_async_from_worker_thread
return anyio.from_thread.run(call)
File "/usr/local/lib/python3.9/site-packages/anyio/from_thread.py", line 49, in run
return asynclib.run_async_from_thread(func, *args)
File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 970, in run_async_from_thread
return f.result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 446, in result
return self.__get_result()
File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
raise self._exception
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 873, in get_task_call_return_value
return await future._result()
File "/usr/local/lib/python3.9/site-packages/prefect/futures.py", line 236, in _result
return final_state.result(raise_on_failure=raise_on_failure)
File "/usr/local/lib/python3.9/site-packages/prefect/orion/schemas/states.py", line 143, in result
raise data
File "/usr/local/lib/python3.9/site-packages/prefect/task_runners.py", line 203, in submit
result = await call()
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 1103, in begin_task_run
return await orchestrate_task_run(
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 1247, in orchestrate_task_run
state = await propose_state(
File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 1463, in propose_state
response = await client.set_task_run_state(
File "/usr/local/lib/python3.9/site-packages/prefect/client.py", line 1826, in set_task_run_state
response = await <http://self._client.post|self._client.post>(
File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1842, in post
return await self.request(
File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1527, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/usr/local/lib/python3.9/site-packages/prefect/client.py", line 279, in send
response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/prefect/client.py", line 225, in raise_for_status
raise PrefectHTTPStatusError.from_httpx_error(exc) from exc.__cause__
prefect.exceptions.PrefectHTTPStatusError: Server error '500 Internal Server Error' for url '<https://api.prefect.cloud/api/accounts/><uuid>/workspaces/<uuid>/task_runs/<uuid>/set_state'm
Mason Menges
09/16/2022, 6:28 PMHey @Josh Paulin Thanks for reporting this, we have an internal ticket opened to investigated these intermittent issues š Let us know if you notice any other strange behavior or that might seem relevant š
j
Josh Paulin
09/16/2022, 6:30 PMPerfect. Thanks!
v
Viet Nguyen
09/17/2022, 1:32 PMthanks that I found this Josh and Mason, been freaking out as I thought something is wrong to my flows. I have the same error at the end of the flow.
8 Views