bash
   prefect work-pool create my-docker-pool-2 --type docker

bash
   prefect worker start --pool my-docker-pool-2

bash
   export PREFECT_API_URL="<http://localhost:4200/api>"  # Adjust if needed

bash
   prefect deploy

prefect worker ls

/opt/prefect/flows/test_flow.py

dockerfile
FROM prefecthq/prefect:3-python3.12

WORKDIR /opt/prefect

# Create necessary directories
RUN mkdir -p /opt/prefect/flows

# Copy requirements first for better caching
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy only the flows directory
COPY flows/ /opt/prefect/flows/

your_project_directory/
├── Dockerfile.prefect_flows
├── prefect.yaml
├── requirements.txt
└── flows/
    └── test_flow.py

yaml
name: ds-pipelines
prefect-version: 3.4.11

build:
- prefect_docker.deployments.steps.build_docker_image:
    id: build-image
    requires: prefect-docker
    image_name: ds-pipelines-flow-runner
    tag: latest
    dockerfile: Dockerfile.prefect_flows

deployments:
- name: My Test Flow Deployment
  description: My first deployment with a custom Dockerfile
  entrypoint: flows/test_flow.py:cat_fact_flow  # Make sure this path matches your actual file structure
  work_pool:
    name: my-docker-pool-2
    work_queue_name: null
    job_variables: {}

bash
# Check if the work pool exists
prefect work-pool ls

# Create the work pool if it doesn't exist
prefect work-pool create my-docker-pool-2 --type docker

# Start a worker (in a separate terminal)
prefect worker start --pool my-docker-pool-2

# Deploy your flow
prefect deploy

bash
# Find your running container
docker ps

# Check the contents of the /opt/prefect directory in the container
docker exec <container_id> ls -la /opt/prefect
docker exec <container_id> ls -la /opt/prefect/flows

# Check if your test_flow.py is actually in the container
docker exec <container_id> cat /opt/prefect/flows/test_flow.py

test_flow.py

flows

entrypoint

bash
pip install prefect-docker

test_flow.py

docker build -t ds-pipelines-flow-runner -f Dockerfile.prefect_flows .

prefect work-pool ls

/opt/prefect

dockerfile
WORKDIR /opt/prefect
COPY flows/ /opt/prefect/flows/

/opt/prefect

/opt/prefect/flows/

test_flow.py

/opt/prefect/flows/test_flow.py

/opt/prefect

/opt/prefect/src

bash
# Check if the container has the correct structure
docker exec <container_id> ls -la /opt/prefect
docker exec <container_id> ls -la /opt/prefect/flows

flows

bash
# Run this in your project directory
ls -la
ls -la flows/

test_flow.py

flows

prefect-worker-image

ds-pipelines-flow-runner

bash
# Stop the current worker
docker stop kind_nightingale

# Build the image with the correct name
docker build -t ds-pipelines-flow-runner -f Dockerfile.prefect_flows .

dockerfile
FROM prefecthq/prefect:3-python3.12

WORKDIR /opt/prefect

# Create flows directory
RUN mkdir -p /opt/prefect/flows

# Copy requirements first for better caching
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Debug: List contents before copy
RUN pwd && ls -la

# Copy flows directory - make sure this exists in your build context
COPY flows/ /opt/prefect/flows/

# Debug: List contents after copy
RUN ls -la /opt/prefect/flows/

yaml
name: ds-pipelines
prefect-version: 3.4.11

build:
- prefect_docker.deployments.steps.build_docker_image:
    id: build-image
    requires: prefect-docker
    image_name: ds-pipelines-flow-runner
    tag: latest
    dockerfile: Dockerfile.prefect_flows

deployments:
- name: My Test Flow Deployment
  description: My first deployment with a custom Dockerfile
  entrypoint: flows/test_flow.py:cat_fact_flow
  work_pool:
    name: my-docker-pool-2
    work_queue_name: null
    job_variables: {}

bash
# Make sure prefect-docker is installed
pip install prefect-docker

# Create/update the work pool
prefect work-pool create my-docker-pool-2 --type docker

# Deploy
prefect deploy

# Start a new worker
prefect worker start --pool my-docker-pool-2

ls -la

test_flow.py

bash
# Build the image
docker build -t ds-pipelines-flow-runner -f Dockerfile.prefect_flows .

# Check if the image was built
docker images | grep ds-pipelines-flow-runner

dockerfile
FROM prefecthq/prefect:3-python3.12
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt "prefect[docker]"

dockerfile
FROM prefecthq/prefect:3-python3.12

WORKDIR /opt/prefect

# Create flows directory
RUN mkdir -p /opt/prefect/flows

# Copy requirements first
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy your flows
COPY flows/ /opt/prefect/flows/

your_project_directory/
├── Dockerfile.prefect_worker    # For the worker
├── Dockerfile.prefect_flows     # For your flows
├── prefect.yaml
├── requirements.txt
└── flows/
    └── test_flow.py

bash
# First, build your flow image
docker build -t ds-pipelines-flow-runner -f Dockerfile.prefect_flows .

# Then build your worker image
docker build -t prefect-worker-image -f Dockerfile.prefect_worker .

# Start your worker
docker run \
    --rm \
    -v /var/run/docker.sock:/var/run/docker.sock \
    --network="host" \
    -e "PREFECT_API_URL=<http://localhost:4200/api>" \
    prefect-worker-image \
    prefect worker start -p my-docker-pool-2 --type docker

yaml
name: ds-pipelines
prefect-version: 3.4.11

build:
- prefect_docker.deployments.steps.build_docker_image:
    id: build-image
    requires: prefect-docker
    image_name: ds-pipelines-flow-runner
    tag: latest
    dockerfile: Dockerfile.prefect_flows

deployments:
- name: My Test Flow Deployment
  description: My first deployment with a custom Dockerfile
  entrypoint: flows/test_flow.py:cat_fact_flow
  work_pool:
    name: my-docker-pool-2
    work_queue_name: null
    job_variables: {}

bash
prefect deploy

bash
# Run these commands and share the output
ls -la flows/
cat flows/test_flow.py

bash
# Build the flow image
docker build -t ds-pipelines-flow-runner -f Dockerfile.prefect_flows .

# After building, inspect the container to make sure files are copied
docker run --rm ds-pipelines-flow-runner ls -la /opt/prefect/flows/

yaml
name: ds-pipelines
prefect-version: 3.4.11

build:
- prefect_docker.deployments.steps.build_docker_image:
    id: build-image
    requires: prefect-docker
    image_name: ds-pipelines-flow-runner  # This should match the image name we built
    tag: latest
    dockerfile: Dockerfile.prefect_flows

deployments:
- name: My Test Flow Deployment
  description: My first deployment with a custom Dockerfile
  entrypoint: flows/test_flow.py:cat_fact_flow  # Make sure this matches your function name
  work_pool:
    name: my-docker-pool-2
    work_queue_name: null
    job_variables: {}

bash
# Deploy your flow
prefect deploy

# Start your worker (in a separate terminal)
docker run \
    --rm \
    -v /var/run/docker.sock:/var/run/docker.sock \
    --network="host" \
    -e "PREFECT_API_URL=<http://localhost:4200/api>" \
    prefect-worker-image \
    prefect worker start -p my-docker-pool-2 --type docker

ls -la flows/

test_flow.py

docker images | grep ds-pipelines-flow-runner

python
from prefect import flow, task

@task
def get_message():
    return "Hello from Prefect!"

@flow(name="test-flow")
def test_flow():
    message = get_message()
    print(message)
    return message

if __name__ == "__main__":
    test_flow()

yaml
entrypoint: flows/test_flow.py:test_flow

ds-pipelines-flow-runner

ds-pipelines-flow-runner

Dockerfile.prefect_flows

prefect.yaml

prefect-worker-image

Dockerfile.prefect_worker

bash
# 1. Build your flow image
docker build -t ds-pipelines-flow-runner -f Dockerfile.prefect_flows .

# 2. Build your worker image
docker build -t prefect-worker-image -f Dockerfile.prefect_worker .

# 3. Deploy your flow (this uses the flow image name from prefect.yaml)
prefect deploy

# 4. Start your worker (this uses the worker image)
docker run \
    --rm \
    -v /var/run/docker.sock:/var/run/docker.sock \
    --network="host" \
    -e "PREFECT_API_URL=<http://localhost:4200/api>" \
    prefect-worker-image \
    prefect worker start -p my-docker-pool-2 --type docker

prefect-worker-image

ds-pipelines-flow-runner

bash
# Check both images exist
docker images | grep 'ds-pipelines-flow-runner\|prefect-worker-image'

# Verify flow image has your flows
docker run --rm ds-pipelines-flow-runner ls -la /opt/prefect/flows/

# Check worker image has prefect installed
docker run --rm prefect-worker-image prefect version

prefect.yaml

ds-pipelines-flow-runner

prefect-worker-image