[Traceback] [Saving parquet]
Note: we can save .csv, but parquet doesn't work
Using latest versions of Dask:
Traceback (most recent call last):
File "/opt/prefect/flows/forge/domain/features/tasks/save_dataset.py", line 29, in save_dataset
save_dataframe(
File "/opt/prefect/flows/forge/domain/features/tasks/save_dataset.py", line 58, in save_dataframe
df.to_parquet(
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_collection.py", line 3296, in to_parquet
return to_parquet(self, path, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/io/parquet.py", line 594, in to_parquet
df.to_legacy_dataframe(),
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_collection.py", line 1396, in to_legacy_dataframe
df = self.optimize(**optimize_kwargs) if optimize else self
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_collection.py", line 591, in optimize
return new_collection(self.expr.optimize(fuse=fuse))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_expr.py", line 94, in optimize
return optimize(self, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_expr.py", line 3070, in optimize
return optimize_until(expr, stage)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_expr.py", line 3021, in optimize_until
expr = result.simplify()
^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_core.py", line 374, in simplify
new = expr.simplify_once(dependents=dependents, simplified={})
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_core.py", line 352, in simplify_once
new = operand.simplify_once(
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_core.py", line 352, in simplify_once
new = operand.simplify_once(
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_core.py", line 352, in simplify_once
new = operand.simplify_once(
^^^^^^^^^^^^^^^^^^^^^^
[Previous line repeated 4 more times]
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_core.py", line 335, in simplify_once
out = child._simplify_up(expr, dependents)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_categorical.py", line 184, in _simplify_up
return Categorize(result.frame, cats, result.operand("index"))
^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/forge/lib/python3.11/site-packages/dask_expr/_core.py", line 176, in operand
return self.operands[type(self)._parameters.index(key)]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: 'index' is not in list