Deepanshu Aggarwal
09/25/2022, 8:10 PMAnna Geller
09/25/2022, 8:28 PMDeepanshu Aggarwal
09/25/2022, 8:41 PMname: 1 AWS EKS Cluster + Prefect Agent
env:
EKS_CLUSTER: prefect
NAMESPACE_ENV: dev-prefect
K8_SECRET_NAME: prefect-secrets
PREFECT_VERSION: '2.*'
AWS_DEFAULT_REGION: 'us-west-2'
AWS_ACCOUNT_ID: '***'
on:
push:
branches:
- develop
jobs:
ecr-repo: # used by flows only; the agent uses image provided as input above
name: Create ECR repository, build and push the image
runs-on: ubuntu-latest
outputs:
image: ${{ steps.build-image.outputs.image }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.DEV_AWS_DEPLOYMENT_BOT_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.DEV_AWS_DEPLOYMENT_BOT_SECRET_KEY }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
mask-aws-account-id: 'no'
- name: Login to Amazon ECR
id: login-ecr
env:
ECR_REGISTRY: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_DEFAULT_REGION }}.<http://amazonaws.com|amazonaws.com>
run: |
aws ecr get-login-password --region ${{ env.AWS_DEFAULT_REGION }} | docker login --username AWS --password-stdin ${{ env.ECR_REGISTRY }}
- name: Build, tag, and push image to Amazon ECR
id: build-image
env:
ECR_REGISTRY: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_DEFAULT_REGION }}.<http://amazonaws.com|amazonaws.com>
IMAGE_TAG: ${{ github.sha }}
run: |
docker build -t $ECR_REGISTRY/$NAMESPACE_ENV:$IMAGE_TAG .
docker push $ECR_REGISTRY/$NAMESPACE_ENV:$IMAGE_TAG
echo "::set-output name=image::$ECR_REGISTRY/$NAMESPACE_ENV:$IMAGE_TAG"
- name: Flow deployments finished
run: echo "ECR image ${{ steps.build-image.outputs.image }} built at $(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_STEP_SUMMARY
blocks:
name: Create blocks
runs-on: ubuntu-latest
needs: ecr-repo
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Python dependencies
run: |
pip install -U "prefect==$PREFECT_VERSION"
pip install s3fs
- name: Prefect Cloud login
run: |
prefect config set PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}
prefect config set PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}
- name: AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.DEV_AWS_DEPLOYMENT_BOT_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.DEV_AWS_DEPLOYMENT_BOT_SECRET_KEY }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
mask-aws-account-id: 'no'
- name: Build S3 block
run: |
cat <<EOF > s3_block.py
from prefect.filesystems import S3
s3 = S3(bucket_path="zluri-dev-lambda-packages/", aws_access_key_id="${{ secrets.DEV_AWS_DEPLOYMENT_BOT_ACCESS_KEY }}", aws_secret_access_key="${{ secrets.DEV_AWS_DEPLOYMENT_BOT_SECRET_KEY }}")
s3.save("${{ env.NAMESPACE_ENV }}", overwrite=True)
EOF
python s3_block.py
- name: Build KubernetesJob block
run: |
cat <<EOF > k8s_block.py
from prefect.infrastructure import KubernetesJob
k8s = KubernetesJob(image="${{ needs.ecr-repo.outputs.image }}", namespace="${{ env.NAMESPACE_ENV }}",
customizations=[{"op": "add", "path": "/spec/ttlSecondsAfterFinished", "value": 10}], image_pull_policy="IfNotPresent")
k8s.save("${{ env.NAMESPACE_ENV }}", overwrite=True)
EOF
python k8s_block.py
- name: Summary
run: echo 'Blocks created successfully! :duck:' >> $GITHUB_STEP_SUMMARY
list-flows:
runs-on: ubuntu-latest
needs: blocks
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
uses: actions/checkout@v3
- id: set-matrix
run: echo "::set-output name=matrix::$(ls flows/*.py | jq -R -s -c 'split("\n")[:-1]')"
deploy:
needs: list-flows
runs-on: ubuntu-latest
strategy:
matrix:
flows: ${{ fromJson(needs.list-flows.outputs.matrix) }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Python dependencies
run: |
pip install -U "prefect==$PREFECT_VERSION"
pip install s3fs
pip install .
- name: Prefect Cloud login
run: |
prefect config set PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}
prefect config set PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}
- name: AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.DEV_AWS_DEPLOYMENT_BOT_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.DEV_AWS_DEPLOYMENT_BOT_SECRET_KEY }}
aws-region: ${{ env.AWS_DEFAULT_REGION }}
mask-aws-account-id: 'no'
- name: Deploy flows to S3
id: build
run: |
FLOW_NAME=$(basename ${{ matrix.flows }} .py)
prefect deployment build ${{ matrix.flows }}:$FLOW_NAME --name ${{ env.NAMESPACE_ENV }} -q ${{ env.NAMESPACE_ENV }} \
-sb s3/${{ env.NAMESPACE_ENV }} -ib kubernetes-job/${{ env.NAMESPACE_ENV }} -a -v $GITHUB_SHA -o $FLOW_NAME.yaml
echo ::set-output name=flow_manifest::$FLOW_NAME.yaml
- name: Upload deployment manifest as artifact
uses: actions/upload-artifact@v3
with:
name: Deployment YAML manifests
path: ${{ steps.build.outputs.flow_manifest }}
Anna Geller
09/25/2022, 11:26 PMDeepanshu Aggarwal
09/26/2022, 7:02 AMWhile trying to Deploy flows to s3 using github actions
FLOW_NAME=$(basename flows/hello.py .py)
prefect deployment build flows/hello.py:$FLOW_NAME --name dev-prefect -q dev-prefect -sb s3/dev-prefect -ib kubernetes-job/dev-prefect -a -v $GITHUB_SHA -o $FLOW_NAME.yaml echo ::set-output name=flow_manifest::$FLOW_NAME.yaml
ERROR
prefect.exceptions.PrefectHTTPStatusError: Client error '400 Bad Request' for url '***/block_types/slug/kubernetes-job/block_documents/name/dev-prefect?include_secrets=true'
@Anna Gellerfrom prefect.filesystems import S3
s3_block = S3.load("dev-prefect")
but this is also throwing errorAnna Geller
09/26/2022, 10:27 AMDeepanshu Aggarwal
09/26/2022, 10:29 AM