@Marvin why does my task break although i have defined retries? Process for flow run 'devious-whale' exited with status code: 1 085127 AM prefect.flow_runs.runner Engine execution exited with unexpected exception Traceback (most recent call last): File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/timeout.py", line 47, in timeout yield File "/app/.venv/lib/python3.12/site-packages/prefect/task_engine.py", line 990, in run_context yield self File "/app/.venv/lib/python3.12/site-packages/prefect/task_engine.py", line 1644, in run_task_sync engine.call_task_fn(txn) File "/app/.venv/lib/python3.12/site-packages/prefect/task_engine.py", line 1007, in call_task_fn result = call_with_parameters(self.task.fn, parameters) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/callables.py", line 210, in call_with_parameters return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/app/src/download.py", line 263, in download_interval metadata = downloader.download( ^^^^^^^^^^^^^^^^^^^^ File "/app/src/uvd_um_engine_onboarding/data_access/download.py", line 196, in download job.start_and_wait( File "/app/.venv/lib/python3.12/site-packages/openeo/rest/job.py", line 371, in start_and_wait time.sleep(poll_interval) prefect._internal.concurrency.cancellation.CancelledError During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/app/.venv/lib/python3.12/site-packages/prefect/flow_engine.py", line 1582, in run_flow ret_val = run_flow_sync(**kwargs) ^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/flow_engine.py", line 1427, in run_flow_sync return engine.state if return_type == "state" else engine.result() ^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/flow_engine.py", line 363, in result raise self._raised File "/app/.venv/lib/python3.12/site-packages/prefect/flow_engine.py", line 797, in run_context yield self File "/app/.venv/lib/python3.12/site-packages/prefect/flow_engine.py", line 1425, in run_flow_sync engine.call_flow_fn() File "/app/.venv/lib/python3.12/site-packages/prefect/flow_engine.py", line 817, in call_flow_fn result = call_with_parameters(self.flow.fn, self.parameters) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/callables.py", line 210, in call_with_parameters return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/app/src/download.py", line 436, in download_in_slices_from_open_eo results = [init_result] + [f.result() for f in futures] ^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/futures.py", line 222, in result _result = self._final_state.result( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/compatibility/async_dispatch.py", line 94, in wrapper return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/client/schemas/objects.py", line 375, in result return run_coro_as_sync( ^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/asyncutils.py", line 207, in run_coro_as_sync return call.result() ^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/concurrency/calls.py", line 365, in result return self.future.result(timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/concurrency/calls.py", line 192, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/concurrency/calls.py", line 441, in _run_async result = await coro ^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/asyncutils.py", line 188, in coroutine_wrapper return await task ^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/states.py", line 85, in get_state_result return await _get_state_result( ^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/states.py", line 157, in _get_state_result raise await aget_state_exception(state) File "/app/src/download.py", line 173, in retry_download_slice state.result() # will raise task's real exception ^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/compatibility/async_dispatch.py", line 94, in wrapper return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/client/schemas/objects.py", line 375, in result return run_coro_as_sync( ^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/asyncutils.py", line 207, in run_coro_as_sync return call.result() ^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/concurrency/calls.py", line 365, in result return self.future.result(timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/concurrency/calls.py", line 192, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/app/.venv/lib/python3.12/site-packages/prefect/_internal/concurrency/calls.py", line 441, in _run_async result = await coro ^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/asyncutils.py", line 188, in coroutine_wrapper return await task ^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/states.py", line 85, in get_state_result return await _get_state_result( ^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/.venv/lib/python3.12/site-packages/prefect/states.py", line 157, in _get_state_result raise await aget_state_exception(state) File "/app/.venv/lib/python3.12/site-packages/prefect/task_engine.py", line 980, in run_context with timeout( ^^^^^^^^ File "/usr/local/lib/python3.12/contextlib.py", line 158, in exit self.gen.throw(value) File "/app/.venv/lib/python3.12/site-packages/prefect/utilities/timeout.py", line 49, in timeout raise timeout_exc_type(f"Scope timed out after {seconds} second(s).") prefect.task_engine.TaskRunTimeoutError: Scope timed out after 3600.0 second(s). 085122 AM prefect.flow_runs @task( retries=50, #effective endless retry_delay_seconds=two_scale_delays, retry_jitter_factor=0.25, retry_condition_fn=retry_download_slice, timeout_seconds=3600, #retry task taking longer than one hour ) Finished in state TimedOut("Flow run failed due to timeout: TaskRunTimeoutError('Scope timed out after 3600.0 second(s).')", type=FAILED) 085122 AM prefect.flow_runs Flow run failed due to timeout: TaskRunTimeoutError('Scope timed out after 3600.0 second(s).') 085122 AM prefect.flow_runs Finished in state TimedOut('Task run exceeded timeout of 3600.0 second(s)', type=FAILED) 085122 AM download_interval-466 prefect.task_runs Task run exceeded timeout of 3600.0 second(s)

Your message was too long, here's your message at the allowed limit: why does my task break although i have defined retries? Process for flow run 'devious-whale' exited with status code 1 08 51 27 AM prefect flow_runs runner Engine execution exited with unexpected exception Traceback (most recent call last) File /app/ venv/lib/python3 12/site-packages/prefect/utilities/timeout py line 47 in timeout yield File /app/ venv/lib/python3 12/site-packages/prefect/task_engine py line 990 in run_context yield self File /app/ venv/lib/python3 12/site-packages/prefect/task_engine py line 1644 in run_task_sync engine call_task_fn(txn) File /app/ venv/lib/python3 12/site-packages/prefect/task_engine py line 1007 in call_task_fn result = call_with_parameters(self task fn parameters) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/utilities/callables py line 210 in call_with_parameters return fn(*args *kwargs) ^^^^^^^^^^^^^^^^^^^ File /app/src/download py line 263 in download_interval metadata = downloader download( ^^^^^^^^^^^^^^^^^^^^ File /app/src/uvd_um_engine_onboarding/data_access/download py line 196 in download job start_and_wait( File /app/ venv/lib/python3 12/site-packages/openeo/rest/job py line 371 in start_and_wait time sleep(poll_interval) prefect _internal concurrency cancellation CancelledError During handling of the above exception another exception occurred Traceback (most recent call last) File /app/ venv/lib/python3 12/site-packages/prefect/flow_engine py line 1582 in run_flow ret_val = run_flow_sync(*kwargs) ^^^^^^^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/flow_engine py line 1427 in run_flow_sync return engine state if return_type == state else engine result() ^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/flow_engine py line 363 in result raise self _raised File /app/ venv/lib/python3 12/site-packages/prefect/flow_engine py line 797 in run_context yield self File /app/ venv/lib/python3 12/site-packages/prefect/flow_engine py line 1425 in run_flow_sync engine call_flow_fn() File /app/ venv/lib/python3 12/site-packages/prefect/flow_engine py line 817 in call_flow_fn result = call_with_parameters(self flow fn self parameters) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/utilities/callables py line 210 in call_with_parameters return fn(*args *kwargs) ^^^^^^^^^^^^^^^^^^^ File /app/src/download py line 436 in download_in_slices_from_open_eo results = [init_result] + [f result() for f in futures] ^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/futures py line 222 in result _result = self _final_state result( ^^^^^^^^^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/_internal/compatibility/async_dispatch py line 94 in wrapper return fn(*args *kwargs) ^^^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/client/schemas/objects py line 375 in result return run_coro_as_sync( ^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/utilities/asyncutils py line 207 in run_coro_as_sync return call result() ^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/_internal/concurrency/calls py line 365 in result return self future result(timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/_internal/concurrency/calls py line 192 in result return self __get_result() ^^^^^^^^^^^^^^^^^^^ File /usr/local/lib/python3 12/concurrent/futures/_base py line 401 in __get_result raise self _exception File /app/ venv/lib/python3 12/site-packages/prefect/_internal/concurrency/calls py line 441 in _run_async result = await coro ^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/utilities/asyncutils py line 188 in coroutine_wrapper return await task ^^^^^^^^^^ File /app/ venv/lib/python3 12/site-packages/prefect/states py line 85 in get_state_resu