https://prefect.io logo
Join Slack
Powered by
<@ULVA73B9P> I have a self-hosted Prefect on k8s. ...
# ask-marvin
k
@Marvin I have a self-hosted Prefect on k8s. I am getting the following error. (see in thread)
m
Could you please share the error message you're encountering with your self-hosted Prefect on Kubernetes? I'll do my best to help once I have more details!
k
Copy code
Exception Group Traceback (most recent call last):
  |   File "/usr/local/lib/python3.9/site-packages/prefect/cli/_utilities.py", line 42, in wrapper
  |     return fn(*args, **kwargs)
  |   File "/usr/local/lib/python3.9/site-packages/prefect/cli/_types.py", line 153, in sync_fn
  |     return asyncio.run(async_fn(*args, **kwargs))
  |   File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
  |     return loop.run_until_complete(main)
  |   File "/usr/local/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
  |     return future.result()
  |   File "/usr/local/lib/python3.9/site-packages/prefect/cli/flow_run.py", line 353, in execute
  |     await Runner().execute_flow_run(id)
  |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 533, in execute_flow_run
  |     tg.start_soon(
  |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 1312, in __aexit__
  |     await self._runs_task_group.__aexit__(*exc_info)
  |   File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 815, in __aexit__
  |     raise BaseExceptionGroup(
  | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 510, in execute_flow_run
    |     flow_run = await self._client.read_flow_run(flow_run_id)
    |   File "/usr/local/lib/python3.9/site-packages/prefect/client/orchestration.py", line 2082, in read_flow_run
    |     response = await self._client.get(f"/flow_runs/{flow_run_id}")
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1768, in get
    |     return await self.request(
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1540, in request
    |     return await self.send(request, auth=auth, follow_redirects=follow_redirects)
    |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 324, in send
    |     response = await self._send_with_retry(
    |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 248, in _send_with_retry
    |     response = await send(request, *send_args, **send_kwargs)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1629, in send
    |     response = await self._send_handling_auth(
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
    |     response = await self._send_handling_redirects(
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
    |     response = await self._send_single_request(request)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1730, in _send_single_request
    |     response = await transport.handle_async_request(request)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
    |     resp = await self._pool.handle_async_request(req)
    |   File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    |     self.gen.throw(typ, value, traceback)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
    |     raise mapped_exc(message) from exc
    | httpx.ConnectTimeout
    | 
    | During handling of the above exception, another exception occurred:
    | 
    | Exception Group Traceback (most recent call last):
    |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 533, in execute_flow_run
    |     tg.start_soon(
    |   File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 815, in __aexit__
    |     raise BaseExceptionGroup(
    | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
    +-+---------------- 1 ----------------
      | Traceback (most recent call last):
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
      |     yield
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
      |     resp = await self._pool.handle_async_request(req)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 256, in handle_async_request
      |     raise exc from None
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 236, in handle_async_request
      |     response = await connection.handle_async_request(
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 101, in handle_async_request
      |     raise exc
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 78, in handle_async_request
      |     stream = await self._connect(request)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 124, in _connect
      |     stream = await self._network_backend.connect_tcp(**kwargs)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_backends/auto.py", line 31, in connect_tcp
      |     return await self._backend.connect_tcp(
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_backends/anyio.py", line 122, in connect_tcp
      |     stream._raw_socket.setsockopt(*option)  # type: ignore[attr-defined] # pragma: no cover
      |   File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
      |     self.gen.throw(typ, value, traceback)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
      |     raise to_exc(exc) from exc
      | httpcore.ConnectTimeout
      | 
      | The above exception was the direct cause of the following exception:
      | 
      | Traceback (most recent call last):
      |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 510, in execute_flow_run
      |     flow_run = await self._client.read_flow_run(flow_run_id)
      |   File "/usr/local/lib/python3.9/site-packages/prefect/client/orchestration.py", line 2082, in read_flow_run
      |     response = await self._client.get(f"/flow_runs/{flow_run_id}")
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1768, in get
      |     return await self.request(
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1540, in request
      |     return await self.send(request, auth=auth, follow_redirects=follow_redirects)
      |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 324, in send
      |     response = await self._send_with_retry(
      |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 248, in _send_with_retry
      |     response = await send(request, *send_args, **send_kwargs)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1629, in send
      |     response = await self._send_handling_auth(
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
      |     response = await self._send_handling_redirects(
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
      |     response = await self._send_single_request(request)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1730, in _send_single_request
      |     response = await transport.handle_async_request(request)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
      |     resp = await self._pool.handle_async_request(req)
      |   File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
      |     self.gen.throw(typ, value, traceback)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
      |     raise mapped_exc(message) from exc
      | httpx.ConnectTimeout
@Marvin still waiting
m
It seems like the error message might not have come through. Could you try sending it again? I'm here to help once I get the details!
k
Copy code
Exception Group Traceback (most recent call last):
  |   File "/usr/local/lib/python3.9/site-packages/prefect/cli/_utilities.py", line 42, in wrapper
  |     return fn(*args, **kwargs)
  |   File "/usr/local/lib/python3.9/site-packages/prefect/cli/_types.py", line 153, in sync_fn
  |     return asyncio.run(async_fn(*args, **kwargs))
  |   File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
  |     return loop.run_until_complete(main)
  |   File "/usr/local/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
  |     return future.result()
  |   File "/usr/local/lib/python3.9/site-packages/prefect/cli/flow_run.py", line 353, in execute
  |     await Runner().execute_flow_run(id)
  |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 533, in execute_flow_run
  |     tg.start_soon(
  |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 1312, in __aexit__
  |     await self._runs_task_group.__aexit__(*exc_info)
  |   File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 815, in __aexit__
  |     raise BaseExceptionGroup(
  | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 510, in execute_flow_run
    |     flow_run = await self._client.read_flow_run(flow_run_id)
    |   File "/usr/local/lib/python3.9/site-packages/prefect/client/orchestration.py", line 2082, in read_flow_run
    |     response = await self._client.get(f"/flow_runs/{flow_run_id}")
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1768, in get
    |     return await self.request(
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1540, in request
    |     return await self.send(request, auth=auth, follow_redirects=follow_redirects)
    |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 324, in send
    |     response = await self._send_with_retry(
    |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 248, in _send_with_retry
    |     response = await send(request, *send_args, **send_kwargs)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1629, in send
    |     response = await self._send_handling_auth(
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
    |     response = await self._send_handling_redirects(
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
    |     response = await self._send_single_request(request)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1730, in _send_single_request
    |     response = await transport.handle_async_request(request)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
    |     resp = await self._pool.handle_async_request(req)
    |   File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    |     self.gen.throw(typ, value, traceback)
    |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
    |     raise mapped_exc(message) from exc
    | httpx.ConnectTimeout
    | 
    | During handling of the above exception, another exception occurred:
    | 
    | Exception Group Traceback (most recent call last):
    |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 533, in execute_flow_run
    |     tg.start_soon(
    |   File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 815, in __aexit__
    |     raise BaseExceptionGroup(
    | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
    +-+---------------- 1 ----------------
      | Traceback (most recent call last):
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
      |     yield
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
      |     resp = await self._pool.handle_async_request(req)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 256, in handle_async_request
      |     raise exc from None
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 236, in handle_async_request
      |     response = await connection.handle_async_request(
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 101, in handle_async_request
      |     raise exc
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 78, in handle_async_request
      |     stream = await self._connect(request)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 124, in _connect
      |     stream = await self._network_backend.connect_tcp(**kwargs)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_backends/auto.py", line 31, in connect_tcp
      |     return await self._backend.connect_tcp(
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_backends/anyio.py", line 122, in connect_tcp
      |     stream._raw_socket.setsockopt(*option)  # type: ignore[attr-defined] # pragma: no cover
      |   File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
      |     self.gen.throw(typ, value, traceback)
      |   File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
      |     raise to_exc(exc) from exc
      | httpcore.ConnectTimeout
      | 
      | The above exception was the direct cause of the following exception:
      | 
      | Traceback (most recent call last):
      |   File "/usr/local/lib/python3.9/site-packages/prefect/runner/runner.py", line 510, in execute_flow_run
      |     flow_run = await self._client.read_flow_run(flow_run_id)
      |   File "/usr/local/lib/python3.9/site-packages/prefect/client/orchestration.py", line 2082, in read_flow_run
      |     response = await self._client.get(f"/flow_runs/{flow_run_id}")
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1768, in get
      |     return await self.request(
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1540, in request
      |     return await self.send(request, auth=auth, follow_redirects=follow_redirects)
      |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 324, in send
      |     response = await self._send_with_retry(
      |   File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 248, in _send_with_retry
      |     response = await send(request, *send_args, **send_kwargs)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1629, in send
      |     response = await self._send_handling_auth(
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
      |     response = await self._send_handling_redirects(
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
      |     response = await self._send_single_request(request)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1730, in _send_single_request
      |     response = await transport.handle_async_request(request)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
      |     resp = await self._pool.handle_async_request(req)
      |   File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
      |     self.gen.throw(typ, value, traceback)
      |   File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
      |     raise mapped_exc(message) from exc
      | httpx.ConnectTimeout
@Marvin find the error message above
m
I'm afraid the error message didn't come through in your previous messages. Could you please try sharing it again here? Once I have the error details, I'll be able to assist you further.
4 Views
Open in Slack
PreviousNext
Powered by