Adam Everington
11/22/2021, 1:17 PMAnna Geller
11/22/2021, 1:24 PMfrom prefect import task, Flow, Parameter
@task
def return_some_number(x):
return x
with Flow('Add one') as flow:
result = Parameter('x', default=1) + 1
nr = result + 1
return_some_number(nr)
flow.visualize()
And compare it to this:
from prefect import task, Flow, Parameter
@task
def add_one(x):
return x + 1
@task
def add_one_again(x):
return x + 1
with Flow('Add one') as flow:
param = Parameter('x', default=1)
result = add_one(param)
add_one_again(result)
flow.visualize()
Adam Everington
11/22/2021, 2:28 PMwith Flow("Create-Postcode-Incode-Lookup"
,executor=LocalDaskExecutor()
,storage = Git(
flow_path = "./ONS/Create-ONS-Incode-Lookup.py",
repo="prefect",
branch_name="main",
git_clone_url_secret_name="DEVOPS_URL"
)
,run_config=UniversalRun(labels=['Cloud01'])) as flow:
#Params
working_directory = Parameter("Working Directory",default=r"/media/data0/prefect/ons",required=False)
server = Parameter("RefData_Server",default="<http://myazureserver.database.windows.net|myazureserver.database.windows.net>",required=False)
db = Parameter("RefData_Db",default="RefData",required=False)
user = Parameter('SQL User Name', default='admin', required=False)
#Secrets
pw = get_secret_value(user)
cnxn_string = get_cnxn_string(server,db,user,pw)
#Tasks
url = get_download_url()
files = download_postcode_directory(url, working_directory)
postcode_file,parish_file = unzip_files(files, working_directory)
postcodes = get_postcodes(postcode_file)
parishes = get_parishes(parish_file)
to_load = merge_datasets(postcodes, parishes)
load = load_to_sql_server(to_load, cnxn_string, 'ONSInCodeLookup')
clean_landing_area(working_directory,upstream_tasks=[load])
flow.schedule = schedules.Schedule(
clocks=[CronClock("0 18 28 * *")]
)
flow.register(
project_name='Capture-ETL',
idempotency_key=flow.serialized_hash()
)
and look at my flow visual I sent, the ref data param just gets passed to 1 task, that's it and yet it's showing 2 add operators?
It's not a biggy... I'm just curious!Anna Geller
11/22/2021, 3:42 PMAdam Everington
11/22/2021, 6:58 PM