Daniel Manson
08/10/2023, 1:50 PMJoshua Greenhalgh
08/10/2023, 2:12 PMFROM python:3.9.9
COPY . /opt/src
WORKDIR /opt/src
RUN pip install --upgrade pip && pip install poetry && poetry build && pip install ./dist/*.whl
/usr/local/lib/python3.9/site-packages/
Daniel Manson
08/10/2023, 2:17 PMFROM prefecthq/prefect:2-python3.9 as prefect
FROM <http://gcr.io/deeplearning-platform-release/tf-gpu.2-6.py39|gcr.io/deeplearning-platform-release/tf-gpu.2-6.py39>
WORKDIR /opt/prefect
COPY --from=prefect /opt/prefect/ /opt/prefect/
Joshua Greenhalgh
08/10/2023, 2:22 PMDaniel Manson
08/14/2023, 9:17 AMdid you look into just starting a vertex AI job from a standard flowno i didn't try that. I think I have now got it working in GKE as I wanted, using roughly your approach rather than copying from the prefect image. My docker file is roughly the following:
FROM <http://gcr.io/deeplearning-platform-release/tf-gpu.2-6.py39|gcr.io/deeplearning-platform-release/tf-gpu.2-6.py39>
# this is basically how the official prefect image works if you look carefully
ADD <https://github.com/krallin/tini/releases/download/v0.19.0/tini> /usr/bin/tini
RUN chmod +x /usr/bin/tini
CMD ["python", "-m", "prefect.engine"]
ENTRYPOINT ["/usr/bin/tini", "-g", "--"]
RUN pip install {" ".join(f'"{dep}"' for dep in python_dependencies)}
COPY pipelines/flows_config.py /opt/prefect/flows/flows.py
COPY . /opt/prefect/my_project/
WORKDIR /opt/prefect/my_project/
ENV PYTHONPATH=/opt/my_project/
and i think it's working, but need to double check a few things.FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
# setup python 3.9, using deadsnakes ppa - <https://launchpad.net/~deadsnakes>
ENV TZ=Europe/London
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN apt update -y \
&& apt -y install software-properties-common wget gcc \
&& add-apt-repository -y ppa:deadsnakes/ppa \
&& apt install -y python3.9 python3.9-dev python3.9-distutils \
&& ln -s /usr/bin/python3.9 /usr/bin/python \
&& wget <https://bootstrap.pypa.io/get-pip.py> \
&& python get-pip.py
# Install the hefty ML dependencies
RUN pip install tensorflow==2.12.0
RUN pip install torch --index-url <https://download.pytorch.org/whl/cu118>
RUN pip install sentence-transformers~=2.2.2
RUN pip install {" ".join(f'"{dep}"' for dep in python_dependencies)}
COPY pipelines/flows_config.py /opt/prefect/flows/flows.py
COPY . /opt/prefect/my_project/
WORKDIR /opt/prefect/my_project/
ENV PYTHONPATH=/opt/my_project/