Constantino Schillebeeckx
from prefect import task, Flow import time from prefect.executors import LocalDaskExecutor @task def m1(x): time.sleep(x) print(x) @task def m2(x): print(x) with Flow('foo') as f: l1 = m1.map([1, 2, 3, 4, 5]) l2 = m2.map(['a', 'b'], upstream_tasks=[l1]) f.executor = LocalDaskExecutor(scheduler="processes", num_workers=7)
m1
m2
unmapped
from prefect import task, Flow import time from prefect.executors import LocalDaskExecutor from prefect import unmapped @task def m1(x): time.sleep(x) print(x) @task def m2(x): print(x) @task def red(x): print('dummy reduce') with Flow('foo') as f: l1 = m1.map([1, 2, 3, 4, 5]) l2 = m2.map(['a', 'b'], upstream_tasks=[unmapped(l1)]) f.executor = LocalDaskExecutor(scheduler="processes", num_workers=7)
Anna Geller
Bring your towel and join one of the fastest growing data communities. Welcome to our second-generation open source orchestration platform, a completely rethought approach to dataflow automation.