Wei Mei
04/01/2022, 3:30 PMtoday, dir = get_data(endpoints=api_endpoints)
upload_to_s3(today, dir, endpoints=api_endpoints)
snowflake_load(today, schema="statistics", endpoints=api_endpoints)
Kevin Kho
04/01/2022, 3:32 PMWei Mei
04/01/2022, 3:36 PMtoday, dir = get_data(endpoints=api_endpoints)
upload = upload_to_s3(today, dir, endpoints=api_endpoints)
snowflake_load(today, schema="statistics", endpoints=api_endpoints).set_upstream(upload)
Kevin Kho
04/02/2022, 7:19 PMWei Mei
04/02/2022, 7:21 PM# returned as pandas df's
for endpoint in endpoints:
file = f"{endpoint}-{today}.csv"
df = get_viewer(seg, endpoint)
if not df.empty:
<http://logger.info|logger.info>(f"{endpoint} has data")
df.to_csv(f"{dir}/{file}", index=False)
<http://logger.info|logger.info>(f"{file} created.")
else:
<http://logger.info|logger.info>(f"{endpoint} is empty: {today}")
Kevin Kho
04/02/2022, 7:25 PMFlow
?Wei Mei
04/02/2022, 7:27 PMapi_endpoints = [ "site1", "site2", "site3" ]
today, dir = get_data(endpoints=api_endpoints)
upload = upload_to_s3(today, dir, endpoints=api_endpoints)
snowflake_load(today, schema="statistics", endpoints=api_endpoints).set_upstream(upload)
Kevin Kho
04/02/2022, 7:31 PMWei Mei
04/02/2022, 7:33 PMKevin Kho
04/02/2022, 7:46 PMWei Mei
04/02/2022, 7:50 PMKevin Kho
04/02/2022, 7:52 PMWei Mei
04/02/2022, 7:54 PM