Tim Enders
10/07/2021, 3:06 PMKevin Kho
10/07/2021, 3:11 PMfrom prefect.run_configs import LocalRun
flow.run_config = LocalRun(...)
Tim Enders
10/07/2021, 3:13 PMwith Flow(
"Project Git/SDK Version",
# executor=LocalDaskExecutor(),
state_handlers=[psh.flow_failure],
executor=LocalExecutor(),
# result=GCSResult(bucket="platformsh-vendor-prefect"),
) as flow:
project_ids = list_elite_projects()
info = fetch_system_info.map(project_ids)
flat_data = flatten_json(flatten(info), "platform")
schema = generate_schema(flat_data)
object_uri = upload_gcs(flat_data, "project", "project_system")
status = gcs_to_bq(
object_uri,
"project_system",
dataset="GCP_PLATFORMAPI_DATASET",
schema=schema,
)
flow.run_config = LocalRun()