brokoli
11/15/2023, 1:14 PMbrokoli
11/15/2023, 2:31 PMprefect==2.14.3
prefect-aws==0.4.1
prefect-docker==0.4.1
Nate
11/15/2023, 2:59 PMbrokoli
11/15/2023, 3:18 PM# Generic metadata about this project
name: "redacted-data-pipeline"
prefect-version: "2.14.3"
# build section allows you to manage and build docker images
build:
- prefect_docker.deployments.steps.build_docker_image:
id: build-image
# requires: prefect-docker>=0.4.0
# image_name: "{{ $PREFECT_IMAGE_NAME }}"
image_name: "<http://000.dkr.ecr.us-east-1.amazonaws.com/prefect-minimal|000.dkr.ecr.us-east-1.amazonaws.com/prefect-minimal>"
# tag: latest
dockerfile: ../docker/data-pipeline-Dockerfile
platform: "linux/amd64"
# push section allows you to manage if and how this project is uploaded to remote locations
push:
- prefect_aws.deployments.steps.push_to_s3:
id: push-code
requires: prefect-aws>=0.3.0
bucket: redacted-prefect-blocks
folder: k8s/test
credentials: null
- prefect_docker.deployments.steps.push_docker_image:
requires: prefect-docker>=0.4.0
image_name: "{{ build-image.image_name }}"
tag: "{{ build-image.tag }}"
# pull section allows you to provide instructions for cloning this project in remote locations
pull:
# - prefect.deployments.steps.set_working_directory:
# directory: /opt/prefect/flows
- prefect_aws.deployments.steps.pull_from_s3:
requires: prefect-aws>=0.3.0
bucket: "{{ push-code.bucket }}"
folder: "{{ push-code.folder }}"
credentials: null
# the definitions section allows you to define reusable components for your deployments
definitions:
tags: &common_tags
- "eks"
work_pool: &common_work_pool
name: "kubernetes-pool"
job_variables:
image: "{{ build-image.image }}"
# the deployments section allows you to provide configuration for deploying flows
deployments:
- name: "test"
tags: *common_tags
schedule: null
entrypoint: "flows/utilities/maintenance.py:maintenance"
work_pool: *common_work_pool
brokoli
11/15/2023, 3:27 PMpull_from_s3
is brokenbrokoli
11/15/2023, 3:27 PMBring your towel and join one of the fastest growing data communities. Welcome to our second-generation open source orchestration platform, a completely rethought approach to dataflow automation.
Powered by