Tom Manterfield
04/21/2022, 2:42 PMKevin Kho
04/21/2022, 2:46 PMTom Manterfield
04/21/2022, 2:57 PMKevin Kho
04/21/2022, 3:56 PMTom Manterfield
04/21/2022, 4:01 PMKevin Kho
04/21/2022, 4:04 PMTom Manterfield
04/21/2022, 4:15 PMAlexander Butler
04/22/2022, 2:36 PMTom Manterfield
04/22/2022, 4:44 PMAlexander Butler
04/22/2022, 6:35 PMFROM python:3.9-slim as base
ENV PYTHONFAULTHANDLER=1 \
PYTHONHASHSEED=random \
PYTHONUNBUFFERED=1
RUN apt-get update && apt-get install -y gcc libffi-dev g++
WORKDIR /app
FROM base as staging
ENV PIP_DEFAULT_TIMEOUT=100 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_NO_CACHE_DIR=1 \
POETRY_VERSION=1.1.13
RUN pip install "poetry==$POETRY_VERSION"
RUN python -m venv /venv
COPY pyproject.toml poetry.lock ./
RUN . /venv/bin/activate && poetry install --no-dev --no-root
COPY . .
RUN . /venv/bin/activate && poetry build
FROM base as output
COPY --from=staging /venv /venv
COPY --from=staging /app/dist .
COPY docker-entrypoint.sh ./
COPY src/production_*.py ./
COPY src/deploy-production.sh ./
# USE GCLOUD FOR DOCKER OPS
RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] <http://packages.cloud.google.com/apt> cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl <https://packages.cloud.google.com/apt/doc/apt-key.gpg> | tee /usr/share/keyrings/cloud.google.gpg && apt-get update -y && apt-get install google-cloud-sdk -y
RUN yes | gcloud auth configure-docker us-west1-docker.pkg.dev
# ENSURE WORK-QUEUES HAVE WORKERS IN ENTRYPOINT
RUN . /venv/bin/activate && \
pip install *.whl && \
prefect work-queue create -t pipeline pipeline-jobs && \
prefect work-queue create -t reverse-etl reverse-etl-jobs && \
prefect work-queue create -t ml machine-learning-jobs && \
prefect work-queue create -t audit audit-jobs && \
chmod +x ./deploy-production.sh && \
chmod +x ./docker-entrypoint.sh && \
./deploy-production.sh
EXPOSE 4200
# START AGENTS + SERVER (run with --network="host")
CMD ["./docker-entrypoint.sh"]
for deployments in ./production_*.py; do
prefect deployment create $deployments
echo "Deployments loaded from $deployments"
done
#!/bin/sh
. /venv/bin/activate
# Initialize Prefect Agents
prefect agent start pipeline-jobs \
2> errfile-pipelines &
prefect agent start reverse-etl-jobs \
2> errfile-reverse-etl &
prefect agent start machine-learning-jobs \
2> errfile-machine-learning &
prefect agent start audit-jobs \
2> errfile-audit &
# Initialize Prefect Server
prefect orion start --host 0.0.0.0
Tom Manterfield
04/22/2022, 7:34 PM