MarcoM
11/17/2023, 4:34 PMMarvin
11/17/2023, 4:35 PMdeployment.yaml
. This step includes uploading your flow to its configured remote storage location, if one is specified.
2. Create the deployment on the API.
To build the deployment definition file deployment.yaml
, run the prefect deployment build
Prefect CLI command from the folder containing your flow script and any dependencies of the script.
For example:
bash
$ prefect deployment build -n marvin -p default-agent-pool -q test flows/marvin.py:say_hi
When you run this command, Prefect:
- Creates a marvin_flow-deployment.yaml
file for your deployment based on your flow code and options.
- Uploads your flow files to the configured storage location (local by default).
- Submit your deployment to the work queue test
. The work queue test
will be created if it doesn't exist.
Please note that the appropriate filesystem library supporting the storage location must be installed prior to building a deployment with a storage block.
For more details, you can check the Prefect documentation here.Nate
11/17/2023, 6:16 PMNate
11/17/2023, 6:22 PMsubprocess.check_call("prefect --no-prompt deploy --all".split())
MarcoM
11/17/2023, 6:41 PMNate
11/17/2023, 7:16 PMMarcoM
11/17/2023, 7:38 PMMarcoM
11/17/2023, 7:38 PMimport asyncio
import os
import pathlib
import subprocess
import yaml
from typing import Union
from prefect.cli.deploy import deploy
class FlowDeployer:
DEPLOYMENT_FILENAME = 'prefect.yaml'
#DEPLOYMENT_BATCH = "prefect_deploy.bat"
def __init__(self, parent_folder_path: Union[str, pathlib.Path],
deployment_name: str = None, root_folder: Union[str, pathlib.Path] = None):
self.deployment_name = deployment_name
self.root_folder = pathlib.Path(__file__).resolve().parent if not root_folder else root_folder
self.parent_folder_path = self.root_folder / parent_folder_path
@property
def parent_folder_path(self):
return self._parent_folder_path
@parent_folder_path.setter
def parent_folder_path(self, v):
if isinstance(v, str):
v = pathlib.Path(v)
self._parent_folder_path = v
async def deploy_flows_from_parent_folder(self):
try:
path = self.parent_folder_path
if isinstance(path, str):
path = pathlib.Path(path)
parent_deployment_file = path.joinpath(self.DEPLOYMENT_FILENAME)
if not parent_deployment_file.is_file():
raise Exception(f"In questo folder {path} non c'è il file {self.DEPLOYMENT_FILENAME}")
pj_root_deployment_file = self.root_folder.joinpath(self.DEPLOYMENT_FILENAME)
if pj_root_deployment_file.is_file():
os.remove(pj_root_deployment_file)
with open(parent_deployment_file) as f:
yaml_file = yaml.safe_load(f)
for d in yaml_file['deployments']:
d['entrypoint'] = str(path.joinpath(d['entrypoint'])).replace('\\', '/')
with open(pj_root_deployment_file, 'w') as f:
yaml.dump(yaml_file, f, sort_keys=False)
await deploy(deploy_all=True)
#here the old way to deploy all that we want to remove
#cmd = 'call "venv/scripts/activate" \n'
#cmd += f"prefect deploy --all"
#with open(self.DEPLOYMENT_BATCH, 'w') as f:
#f.write(cmd)
#subprocess.check_output(self.DEPLOYMENT_BATCH)
except Exception as ex:
raise ex
def execute(self):
asyncio.run(self.deploy_flows_from_parent_folder())
if __name__ == "__main__":
parent_folder = pathlib.Path('./customers/****')
root_folder = pathlib.Path('C:/prefect')
d = FlowDeployer(parent_folder_path=parent_folder, root_folder=root_folder)
d.execute()
MarcoM
11/17/2023, 7:41 PMNate
11/17/2023, 8:09 PM