Bradley Hurley
09/06/2022, 9:07 PMfrom prefect.tasks.jupyter.jupyter import ExecuteNotebook
def generate_notebook_tasks(flow):
for nb_file in os.listdir():
run_nb_task = ExecuteNotebook(
path=nb_file,
parameters={}
)
flow.add_task(run_nb_task)
def create_flow() -> Flow:
with Flow() as flow:
generate_notebook_tasks(flow=flow)
return flow
FLOW = create_flow()
Nate
09/06/2022, 9:45 PMExecuteNotebook
over your nb_files
like
from prefect.tasks.jupyter.jupyter import ExecuteNotebook
execute_notebook_task = ExecuteNotebook() # add default params here
with Flow('my nb execute flow') as flow:
# could also do nb_files = glob.glob("*.ipynb")
nb_files = [nb_file for nb_file in os.listdir()]
results = execute_notebook_task.map(path=nb_files)
prefect will generate an instance of execute_notebook_task
for each element in your iterable nb_files
Bradley Hurley
09/06/2022, 10:54 PMNate
09/06/2022, 11:03 PMBradley Hurley
09/06/2022, 11:25 PM