submit_as_runner

@flow(log_prints=True)
def ingest():
    path = "/data/staging/obj"
    files = file_checker(path)   # file_checker is a task
    submit_to_runner(
        sci_processing, [{"file": file} for file in files]
    )
    wait_for_submitted_runs()

sci_processing

prefect config set ...

PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS=true
PREFECT_RUNNER_SERVER_ENABLE=true

prefect config set

ERROR:    [Errno 99] error while attempting to bind on address ('::1', 8080, 0, 0): cannot assign requested address

PrefectHQ/prefect

docker exec

ERROR:    [Errno 99] error while attempting to bind on address ('::1', 8080, 0, 0): cannot assign requested address

python -i atlas_ingest.py

atlas_ingest()

version: "3.9"
services:
  ### Prefect Server API and UI
  server:
    image: prefecthq/prefect:2.14.17-python3.11
    restart: always
    volumes:
      - prefect:/root/.prefect
      - ./data/staging:/data/staging
      # - /data/staging:/data/staging
    entrypoint: ["/opt/prefect/entrypoint.sh", "prefect", "server", "start"]
    environment:
      - PREFECT_UI_URL=<http://127.0.0.1:4200/api>
      - PREFECT_API_URL=<http://127.0.0.1:4200/api>
      # If you want to access Prefect Server UI from anywhere other than the Docker host machine, you will need to change
      # PREFECT_UI_URL and PREFECT_API_URL to match the external hostname/IP of the host machine. For example:
      #- PREFECT_UI_URL=<http://external-ip:4200/api>
      #- PREFECT_API_URL=<http://external-ip:4200/api>
      - PREFECT_SERVER_API_HOST=0.0.0.0
      # - PREFECT_API_DATABASE_CONNECTION_URL=<postgresql+asyncpg://postgres:postgres@database:5432/prefect>
      - PREFECT_API_DATABASE_CONNECTION_URL=sqlite+aiosqlite:////root/.prefect/prefect.db
      # Uncomment the following line if you want to use the 'S3 Bucket' storage block instead of the older 'S3' storage
      - EXTRA_PIP_PACKAGES=astropy prefect-sqlalchemy psycopg2-binary datetime httpx
    ports:
      - 4200:4200
      - 8080:8080

  ### Prefect CLI
  cli:
    image: prefecthq/prefect:2.14.17-python3.11
    working_dir: /root/flows
    volumes:
      - ./flows:/root/flows
      - ./data/staging:/data/staging
      # - /data/staging:/data/staging
    environment:
      - PREFECT_API_URL=<http://server:4200/api>
      #       Use PREFECT_API_KEY to use the CLI to interact with Prefect Cloud
      #     - PREFECT_API_KEY=YOUR_API_KEY
      - EXTRA_PIP_PACKAGES=astropy prefect-sqlalchemy psycopg2-binary datetime httpx
    # profiles: ["cli"]
    command: tail -f /dev/null

volumes:
  prefect:
  db:
  # minio:
networks:
  default:
    name: prefect-network

main

prefect config set PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS=True

prefect config set PREFECT_RUNNER_SERVER_ENABLE=True

File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/local/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py", line 355, in _run_async
    result = await coro
             ^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/prefect/runner/submit.py", line 135, in submit_to_runner
    raise ValueError(
ValueError: The `submit_to_runner` utility requires the `Runner` webserver to be running and built with extra endpoints enabl
ed. To enable this, set the `PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS` setting to `True`.
03:34:38.685 | ERROR   | Flow run 'practical-cockatoo' - Finished in state Failed('Flow run encountered an exception. ValueEr
ror: The `submit_to_runner` utility requires the `Runner` webserver to be running and built with extra endpoints enabled. To 
enable this, set the `PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS` setting to `True`.')
03:34:39.037 | INFO    | prefect.flow_runs.runner - Process for flow run 'practical-cockatoo' exited cleanly.

docker compose up -d

doccker exec