James Phoenix
04/28/2022, 10:47 AMAnna Geller
import pendulum
from prefect.engine.signals import RETRY
def check_if_file_arrived_in_gcs():
return gcs_does_object_exist("<gcs://bucket/example_file.csv>")
@task
def gcs_polling(**kwargs):
bool_gcs_object_arrived = check_if_file_arrived_in_gcs()
if bool_s3_object_arrived is False:
raise RETRY(
"File not available yet, retrying in 20 seconds.",
start_time=pendulum.now().add(seconds=20),
)
James Phoenix
04/28/2022, 12:19 PM