Ben Muller
06/16/2022, 11:59 PMprefect run -p flows/harness_racing_victoria/harness_racing_victoria_results.py
error trace in π§΅
Any ideas?Retrieving local flow... Error
Traceback (most recent call last):
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/prefect/cli/run.py", line 73, in try_error_done
yield
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/prefect/cli/run.py", line 574, in run
flow = get_flow_from_path_or_module(path=path, module=module, name=name)
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/prefect/cli/run.py", line 171, in get_flow_from_path_or_module
flows = load_flows_from_script(path) if path else load_flows_from_module(module)
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/prefect/cli/run.py", line 119, in load_flows_from_script
namespace = runpy.run_path(path, run_name="<flow>")
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/runpy.py", line 265, in run_path
return _run_module_code(code, init_globals, run_name,
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/runpy.py", line 97, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "flows/aus_harness_racing/aus_harness_racing_results.py", line 6, in <module>
from flows.common.tasks import (
File "/Users/benmuller/code/es-harness-flows/flows/common/tasks.py", line 12, in <module>
from python.data_validation.expectations import Expectations
File "/Users/benmuller/code/es-harness-flows/python/data_validation/expectations.py", line 3, in <module>
import great_expectations as ge
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/__init__.py", line 7, in <module>
from great_expectations.data_context import DataContext
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/data_context/__init__.py", line 1, in <module>
from .data_context import BaseDataContext, DataContext, ExplorerDataContext
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/data_context/data_context.py", line 38, in <module>
from great_expectations.core.usage_statistics.usage_statistics import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/core/usage_statistics/usage_statistics.py", line 21, in <module>
from great_expectations.core.usage_statistics.anonymizers.data_docs_site_anonymizer import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/core/usage_statistics/anonymizers/data_docs_site_anonymizer.py", line 2, in <module>
from great_expectations.core.usage_statistics.anonymizers.site_builder_anonymizer import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/core/usage_statistics/anonymizers/site_builder_anonymizer.py", line 2, in <module>
from great_expectations.render.renderer.site_builder import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/render/renderer/__init__.py", line 2, in <module>
from .column_section_renderer import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/render/renderer/column_section_renderer.py", line 15, in <module>
from great_expectations.expectations.core.expect_column_kl_divergence_to_be_less_than import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/expectations/core/__init__.py", line 1, in <module>
from .expect_column_distinct_values_to_be_in_set import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/expectations/core/expect_column_distinct_values_to_be_in_set.py", line 16, in <module>
from ..expectation import ColumnExpectation, InvalidExpectationConfigurationError
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/expectations/expectation.py", line 34, in <module>
from great_expectations.self_check.util import (
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/great_expectations/self_check/util.py", line 91, in <module>
"DATETIME": sqlitetypes.DATETIME(truncate_microseconds=True),
File "/Users/benmuller/code/es-scraping-flows/.venv/lib/python3.8/site-packages/sqlalchemy/dialects/sqlite/base.py", line 143, in __init__
super(_DateTimeMixin, self).__init__(**kw)
TypeError: __init__() got an unexpected keyword argument 'truncate_microseconds'
with Flow(
name="harness_racing_victoria_results",
# storage=Storage.in_a_s3_bucket(),
run_config=RunConfig().fargate_on_ecs(),
schedule=Clock.set_cron("0 4 * * *"),
executor=LocalDaskExecutor(scheduler="threads"),
) as flow:
results = get_hrv_results()
results_schema_enforced = force_schema_types(results, sectional_schema)
put_dateformatted_data_to_s3.map(
df=partition_dataframes_by_column(df=results_schema_enforced, column="date"),
bucket_name=unmapped(get_key_value("raw_data_bucket")),
key_name=unmapped("harness_racing_victoria_sectionals"),
)
Kevin Kho
06/17/2022, 12:01 AMBen Muller
06/17/2022, 12:01 AMKevin Kho
06/17/2022, 12:01 AMBen Muller
06/17/2022, 12:02 AMKevin Kho
06/17/2022, 12:02 AMBen Muller
06/17/2022, 12:03 AMAnna Geller
06/17/2022, 12:28 PMBen Muller
06/18/2022, 7:55 PMBen Muller
06/19/2022, 9:27 PMAnna Geller
06/19/2022, 11:03 PM