Norman
10/05/2022, 8:12 PMRyan Peden
10/05/2022, 8:27 PMmy-docker-infrastructure
, you could use it when building a deployment with the CLI by doing something like:
prefect deployment build flows.py:my_flow -n "My Deployment" --storage-block s3/my-s3-storage --infra-block docker-container/my-docker-infrastructure --work-queue my-work-queue
Norman
10/05/2022, 9:00 PMNate
10/05/2022, 10:59 PMoverride
like
prefect deployment build flows.py:my_flow -n "My Deployment" --storage-block s3/my-s3-storage --infra-block docker-container/my-docker-infrastructure --override "image=mydockerhubuser/repo:tag" --work-queue my-work-queue
Nils
12/09/2022, 12:52 PMSubmission failed. Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 703, in urlopen httplib_response = self._make_request( ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 398, in _make_request conn.request(method, url, **httplib_request_kw) File "/usr/local/lib/python3.11/http/client.py", line 1282, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/local/lib/python3.11/http/client.py", line 1328, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.11/http/client.py", line 1277, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.11/http/client.py", line 1037, in _send_output self.send(msg) File "/usr/local/lib/python3.11/http/client.py", line 975, in send self.connect() File "/usr/local/lib/python3.11/site-packages/docker/transport/unixconn.py", line 30, in connect sock.connect(self.unix_socket) FileNotFoundError: [Errno 2] No such file or directory During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/requests/adapters.py", line 489, in send resp = conn.urlopen( ^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 787, in urlopen retries = retries.increment( ^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/urllib3/util/retry.py", line 550, in increment raise six.reraise(type(error), error, _stacktrace) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/urllib3/packages/six.py", line 769, in reraise raise value.with_traceback(tb) File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 703, in urlopen httplib_response = self._make_request( ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/urllib3/connectionpool.py", line 398, in _make_request conn.request(method, url, **httplib_request_kw) File "/usr/local/lib/python3.11/http/client.py", line 1282, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/local/lib/python3.11/http/client.py", line 1328, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.11/http/client.py", line 1277, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.11/http/client.py", line 1037, in _send_output self.send(msg) File "/usr/local/lib/python3.11/http/client.py", line 975, in send self.connect() File "/usr/local/lib/python3.11/site-packages/docker/transport/unixconn.py", line 30, in connect sock.connect(self.unix_socket) urllib3.exceptions.ProtocolError: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/docker/api/client.py", line 214, in _retrieve_server_version return self.version(api_version=False)["ApiVersion"] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/docker/api/daemon.py", line 181, in version return self._result(self._get(url), json=True) ^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/docker/utils/decorators.py", line 46, in inner return f(self, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/docker/api/client.py", line 237, in _get return self.get(url, **self._set_request_timeout(kwargs)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/requests/sessions.py", line 600, in get return self.request("GET", url, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/requests/sessions.py", line 587, in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/requests/sessions.py", line 701, in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/requests/adapters.py", line 547, in send raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/prefect/infrastructure/docker.py", line 90, in _get_docker_client docker_client = docker.from_env() ^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/docker/client.py", line 96, in from_env return cls( ^^^^ File "/usr/local/lib/python3.11/site-packages/docker/client.py", line 45, in __init__ self.api = APIClient(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/docker/api/client.py", line 197, in __init__ self._version = self._retrieve_server_version() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/docker/api/client.py", line 221, in _retrieve_server_version raise DockerException( docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory')) The above exception was the direct cause of the following exception: RuntimeError: Could not connect to Docker.
Ryan Peden
12/09/2022, 1:20 PMdocker-py
(Docker's official Python SDK) can't connect to the local Docker API, so it's not even getting to the point where it would try using the private registry.
When I've seen this in the past, it's often been a permission error where the user account running Prefect didn't have read/write access to /var/run/docker.sock
(the SDK doesn't use the CLI but instead connects directly to the Docker API via that socket)Nils
12/09/2022, 3:00 PM/var/run/docker.sock
. As in the following docker compose I've added the volume mount on the Prefect Agent. In the logging I see the image being pulled from the GHCR registry. I'm now running into another issue, with very little logging 😐
14:55:12.274 | INFO | prefect.infrastructure.docker-container - Creating Docker container 'zippy-quokka'...
14:55:12.301 | INFO | prefect.infrastructure.docker-container - Docker container 'zippy-quokka' has status 'created'
14:55:12.546 | INFO | prefect.agent - Completed submission of flow run '936dfd63-37da-4524-adc2-866924fe3d44'
14:55:12.549 | INFO | prefect.infrastructure.docker-container - Docker container 'zippy-quokka' has status 'running'
14:55:12.763 | INFO | prefect.infrastructure.docker-container - Docker container 'zippy-quokka' has status 'exited'
14:55:12.764 | INFO | prefect.infrastructure.docker-container - Docker container 'zippy-quokka' has status 'exited'
14:55:12.776 | INFO | prefect.agent - Reported flow run '936dfd63-37da-4524-adc2-866924fe3d44' as crashed: Flow run infrastructure exited with non-zero status code 1.
Ryan Peden
12/09/2022, 3:31 PMdocker logs
tell you anything more about what happened inside the container?Nils
12/09/2022, 3:41 PMdocker ps
(Auto Remove is off). However, when I try docker logs zippy-quokka
I get the following result: /usr/local/bin/python: Error while finding module specification for 'prefect.engine' (ModuleNotFoundError: No module named 'prefect')
Ryan Peden
12/09/2022, 3:45 PMprefect
package using either pip
or conda
during image buildNils
12/09/2022, 4:11 PMFROM python:latest
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
RUN pip install prefect
COPY . .
CMD ["python", "main.py"]
Ryan Peden
12/09/2022, 4:19 PMDockerContainer
infrastructure block will supply a command to run; usually python -m prefect.engine
. The flow run ID gets set in the container as an env variable and Prefect uses that ID to pull the information about your flow from either Orion or Prefect cloud.
I think the command supplied by DockerContainer
will override the CMD, though.Nils
12/09/2022, 4:25 PM2022-12-09T16:22:28.310805163Z /usr/local/lib/python3.8/runpy.py:127: RuntimeWarning: 'prefect.engine' found in sys.modules after import of package 'prefect', but prior to execution of 'prefect.engine'; this may result in unpredictable behaviour
2022-12-09T16:22:28.310885116Z warn(RuntimeWarning(msg))
2022-12-09T16:22:28.310968514Z 16:22:28.307 | DEBUG | prefect.client - Connecting to API at <http://orion:4200/api/>
2022-12-09T16:22:29.910982537Z 16:22:29.833 | ERROR | prefect.engine - Engine execution of flow run '957f15f6-0087-4b8b-8863-9ef073d25db0' exited with unexpected exception
2022-12-09T16:22:29.911030910Z Traceback (most recent call last):
2022-12-09T16:22:29.911036640Z File "/usr/local/lib/python3.8/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
2022-12-09T16:22:29.911040347Z yield
2022-12-09T16:22:29.911043313Z File "/usr/local/lib/python3.8/site-packages/httpcore/backends/asyncio.py", line 109, in connect_tcp
2022-12-09T16:22:29.911047190Z stream: anyio.abc.ByteStream = await anyio.connect_tcp(
2022-12-09T16:22:29.911050688Z File "/usr/local/lib/python3.8/site-packages/anyio/_core/_sockets.py", line 189, in connect_tcp
2022-12-09T16:22:29.911053473Z gai_res = await getaddrinfo(
2022-12-09T16:22:29.911056619Z File "/usr/local/lib/python3.8/site-packages/anyio/_core/_sockets.py", line 496, in getaddrinfo
2022-12-09T16:22:29.911060105Z gai_res = await get_asynclib().getaddrinfo(
2022-12-09T16:22:29.911063662Z File "/usr/local/lib/python3.8/site-packages/anyio/_backends/_asyncio.py", line 1754, in getaddrinfo
2022-12-09T16:22:29.911066317Z result = await get_running_loop().getaddrinfo(
2022-12-09T16:22:29.911069343Z File "/usr/local/lib/python3.8/asyncio/base_events.py", line 825, in getaddrinfo
2022-12-09T16:22:29.911073280Z return await self.run_in_executor(
2022-12-09T16:22:29.911075935Z File "/usr/local/lib/python3.8/concurrent/futures/thread.py", line 57, in run
2022-12-09T16:22:29.911079552Z result = self.fn(*self.args, **self.kwargs)
2022-12-09T16:22:29.911082318Z File "/usr/local/lib/python3.8/socket.py", line 918, in getaddrinfo
2022-12-09T16:22:29.911085073Z for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
2022-12-09T16:22:29.911087889Z socket.gaierror: [Errno -5] No address associated with hostname
2022-12-09T16:22:29.911091545Z
2022-12-09T16:22:29.911095412Z During handling of the above exception, another exception occurred:
2022-12-09T16:22:29.911100903Z
2022-12-09T16:22:29.911105381Z Traceback (most recent call last):
2022-12-09T16:22:29.911108668Z File "/usr/local/lib/python3.8/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
2022-12-09T16:22:29.911111763Z yield
2022-12-09T16:22:29.911115261Z File "/usr/local/lib/python3.8/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
2022-12-09T16:22:29.911118186Z resp = await self._pool.handle_async_request(req)
2022-12-09T16:22:29.911139647Z File "/usr/local/lib/python3.8/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
2022-12-09T16:22:29.911142712Z raise exc
2022-12-09T16:22:29.911145888Z File "/usr/local/lib/python3.8/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
2022-12-09T16:22:29.911148995Z response = await connection.handle_async_request(request)
2022-12-09T16:22:29.911153924Z File "/usr/local/lib/python3.8/site-packages/httpcore/_async/connection.py", line 86, in handle_async_request
2022-12-09T16:22:29.911156720Z raise exc
2022-12-09T16:22:29.911159334Z File "/usr/local/lib/python3.8/site-packages/httpcore/_async/connection.py", line 63, in handle_async_request
2022-12-09T16:22:29.911162861Z stream = await self._connect(request)
2022-12-09T16:22:29.911165737Z File "/usr/local/lib/python3.8/site-packages/httpcore/_async/connection.py", line 111, in _connect
2022-12-09T16:22:29.911168782Z stream = await self._network_backend.connect_tcp(**kwargs)
2022-12-09T16:22:29.911171938Z File "/usr/local/lib/python3.8/site-packages/httpcore/backends/auto.py", line 29, in connect_tcp
2022-12-09T16:22:29.911174894Z return await self._backend.connect_tcp(
2022-12-09T16:22:29.911177649Z File "/usr/local/lib/python3.8/site-packages/httpcore/backends/asyncio.py", line 109, in connect_tcp
2022-12-09T16:22:29.911181046Z stream: anyio.abc.ByteStream = await anyio.connect_tcp(
2022-12-09T16:22:29.911184302Z File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
2022-12-09T16:22:29.911187127Z self.gen.throw(type, value, traceback)
2022-12-09T16:22:29.911190474Z File "/usr/local/lib/python3.8/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
2022-12-09T16:22:29.911194932Z raise to_exc(exc)
2022-12-09T16:22:29.911197417Z httpcore.ConnectError: [Errno -5] No address associated with hostname
2022-12-09T16:22:29.911200302Z
2022-12-09T16:22:29.911202787Z The above exception was the direct cause of the following exception:
2022-12-09T16:22:29.911205722Z
2022-12-09T16:22:29.911208107Z Traceback (most recent call last):
2022-12-09T16:22:29.911210623Z File "/usr/local/lib/python3.8/site-packages/prefect/engine.py", line 1785, in <module>
2022-12-09T16:22:29.911213969Z enter_flow_run_engine_from_subprocess(flow_run_id)
2022-12-09T16:22:29.911216514Z File "/usr/local/lib/python3.8/site-packages/prefect/engine.py", line 186, in enter_flow_run_engine_from_subprocess
2022-12-09T16:22:29.911219349Z return anyio.run(retrieve_flow_then_begin_flow_run, flow_run_id)
2022-12-09T16:22:29.911221864Z File "/usr/local/lib/python3.8/site-packages/anyio/_core/_eventloop.py", line 70, in run
2022-12-09T16:22:29.911224328Z return asynclib.run(func, *args, **backend_options)
2022-12-09T16:22:29.911231802Z File "/usr/local/lib/python3.8/site-packages/anyio/_backends/_asyncio.py", line 292, in run
2022-12-09T16:22:29.911235038Z return native_run(wrapper(), debug=debug)
2022-12-09T16:22:29.911237794Z File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
2022-12-09T16:22:29.911240669Z return loop.run_until_complete(main)
2022-12-09T16:22:29.911243505Z File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
2022-12-09T16:22:29.911246511Z return future.result()
2022-12-09T16:22:29.911249797Z File "/usr/local/lib/python3.8/site-packages/anyio/_backends/_asyncio.py", line 287, in wrapper
2022-12-09T16:22:29.911252643Z return await func(*args)
2022-12-09T16:22:29.911255368Z File "/usr/local/lib/python3.8/site-packages/prefect/client/utilities.py", line 47, in with_injected_client
2022-12-09T16:22:29.911258273Z return await fn(*args, **kwargs)
2022-12-09T16:22:29.911260968Z File "/usr/local/lib/python3.8/site-packages/prefect/engine.py", line 258, in retrieve_flow_then_begin_flow_run
2022-12-09T16:22:29.911264334Z flow_run = await client.read_flow_run(flow_run_id)
2022-12-09T16:22:29.911267240Z File "/usr/local/lib/python3.8/site-packages/prefect/client/orion.py", line 1443, in read_flow_run
2022-12-09T16:22:29.911270276Z response = await self._client.get(f"/flow_runs/{flow_run_id}")
2022-12-09T16:22:29.911272950Z File "/usr/local/lib/python3.8/site-packages/httpx/_client.py", line 1751, in get
2022-12-09T16:22:29.911275787Z return await self.request(
2022-12-09T16:22:29.911278652Z File "/usr/local/lib/python3.8/site-packages/httpx/_client.py", line 1527, in request
2022-12-09T16:22:29.911281157Z return await self.send(request, auth=auth, follow_redirects=follow_redirects)
2022-12-09T16:22:29.911283802Z File "/usr/local/lib/python3.8/site-packages/prefect/client/base.py", line 229, in send
2022-12-09T16:22:29.911286457Z response = await self._send_with_retry(
2022-12-09T16:22:29.911289252Z File "/usr/local/lib/python3.8/site-packages/prefect/client/base.py", line 187, in _send_with_retry
2022-12-09T16:22:29.911292859Z response = await request()
2022-12-09T16:22:29.911295724Z File "/usr/local/lib/python3.8/site-packages/httpx/_client.py", line 1614, in send
2022-12-09T16:22:29.911298970Z response = await self._send_handling_auth(
2022-12-09T16:22:29.911302437Z File "/usr/local/lib/python3.8/site-packages/httpx/_client.py", line 1642, in _send_handling_auth
2022-12-09T16:22:29.911305342Z response = await self._send_handling_redirects(
2022-12-09T16:22:29.911308599Z File "/usr/local/lib/python3.8/site-packages/httpx/_client.py", line 1679, in _send_handling_redirects
2022-12-09T16:22:29.911311525Z response = await self._send_single_request(request)
2022-12-09T16:22:29.911314200Z File "/usr/local/lib/python3.8/site-packages/httpx/_client.py", line 1716, in _send_single_request
2022-12-09T16:22:29.911321013Z response = await transport.handle_async_request(request)
2022-12-09T16:22:29.911323808Z File "/usr/local/lib/python3.8/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
2022-12-09T16:22:29.911326513Z resp = await self._pool.handle_async_request(req)
2022-12-09T16:22:29.911329649Z File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
2022-12-09T16:22:29.911332223Z self.gen.throw(type, value, traceback)
2022-12-09T16:22:29.911335189Z File "/usr/local/lib/python3.8/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
2022-12-09T16:22:29.911338005Z raise mapped_exc(message) from exc
2022-12-09T16:22:29.911341221Z httpx.ConnectError: [Errno -5] No address associated with hostname
Ryan Peden
12/09/2022, 4:32 PMorion
container? I see it is looking for the server at http://orion:4200, but the docker-compose.yml in the repository you linked to earlier names the Orion container prefect-server
. orion
sounds more like what I have in my Docker Compose config here: https://github.com/rpeden/prefect-docker-compose/blob/main/docker-compose.yml
So it might just be a name mismatch?Nils
12/09/2022, 4:45 PMhttpx.HTTPStatusError: Client error '403 Forbidden' for url
. I guess it's better to get it working without Traefik first