Santiago Gonzalez
11/03/2021, 9:46 PMAnna Geller
11/03/2021, 9:48 PMSantiago Gonzalez
11/03/2021, 9:50 PMresume
Anna Geller
11/03/2021, 9:58 PMSantiago Gonzalez
11/03/2021, 10:06 PMAnna Geller
11/03/2021, 10:11 PMSantiago Gonzalez
11/03/2021, 10:13 PMAnna Geller
11/03/2021, 10:16 PMSantiago Gonzalez
11/03/2021, 10:20 PMbatch_matching_full_execution_commands = create_batch_matching_ec2_commands(export_version, next_ekata_dv, False)
batch_matching_full_execution_commands.name = 'Create batch matching ec2 commands for full execution'
batch_matching_full_execution_commands.checkpoint = True
batch_matching_full_execution_commands.result = S3Result(bucket=S3_BUCKET, location=S3_LOCATION_PATTERN)
run_batch_matching_export_notebook = notebook_run(databricks_conn_secret=conn,
json=notebook_submit_config)
run_batch_matching_export_notebook.set_upstream(notebook_created)
instance_id = create_ec2_instance(volume_size=2000)
instance_id.set_upstream(run_batch_matching_export_notebook)
instance_id.skip_on_upstream_skip = False
instance_id.checkpoint = True
instance_id.result = S3Result(bucket=S3_BUCKET, location=S3_LOCATION_PATTERN)
# Execution for 5 minutes, so Ekata can scale up theirs services.
batch_matching_warm_up_execution = execute_job_in_ec2_instance(instance_id=instance_id,
commands=batch_matching_warm_up_commands,
s3_dir_prefix='batch-matching-warm-up')
batch_matching_warm_up_execution.name = 'Warm up ekata services in ec2 instance for 5 minutes'
batch_matching_warm_up_execution.set_upstream(run_batch_matching_export_notebook)
batch_matching_warm_up_execution.skip_on_upstream_skip = False
ekata_verified_their_services = has_ekata_verified_their_services() # Manual Step
ekata_verified_their_services.set_upstream(batch_matching_warm_up_execution)
# Full Execution
batch_matching_full_execution = execute_job_in_ec2_instance(instance_id=instance_id,
commands=batch_matching_full_execution_commands,
s3_dir_prefix='batch-matching',
execution_timeout=60 * 60 * 24 * 7)
batch_matching_full_execution.name = 'Batch Matching Full execution in ec2 instance'
batch_matching_full_execution.set_dependencies(upstream_tasks=[batch_matching_warm_up_execution,
ekata_verified_their_services])
Zanie
11/03/2021, 10:43 PMSantiago Gonzalez
11/03/2021, 11:19 PMAnna Geller
11/04/2021, 3:35 PMZanie
11/04/2021, 3:41 PMSantiago Gonzalez
11/04/2021, 4:13 PMAnna Geller
11/04/2021, 4:14 PM