Stephen Herron
10/21/2022, 10:09 PMNico Neumann
10/21/2022, 10:11 PMStephen Herron
10/21/2022, 10:11 PMfrom prefect.filesystems import S3
def download_s3(block_name) -> None:
s3_block = S3.load(block_name)
s3_block.get_directory()
trying to use get_directory() is providing a stacktrace:
...
s3_block.get_directory()
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/prefect/utilities/asyncutils.py", line 201, in coroutine_wrapper
return run_async_in_new_loop(async_fn, *args, **kwargs)
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/prefect/utilities/asyncutils.py", line 152, in run_async_in_new_loop
return anyio.run(partial(__fn, *args, **kwargs))
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/anyio/_core/_eventloop.py", line 70, in run
return asynclib.run(func, *args, **backend_options)
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 292, in run
return native_run(wrapper(), debug=debug)
File "/usr/local/Cellar/python@3.10/3.10.6_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/Cellar/python@3.10/3.10.6_2/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 646, in run_until_complete
return future.result()
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 287, in wrapper
return await func(*args)
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/prefect/filesystems.py", line 466, in get_directory
return await self.filesystem.get_directory(
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/prefect/filesystems.py", line 312, in get_directory
return self.filesystem.get(from_path, local_path, recursive=True)
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/fsspec/spec.py", line 854, in get
lpaths = other_paths(rpaths, lpath)
File "/Users/stephen.herron/.local/share/virtualenvs/prefect-EAPoDWza/lib/python3.10/site-packages/fsspec/utils.py", line 379, in other_paths
assert len(paths) == len(path2)
TypeError: object of type 'PosixPath' has no len()
Nico Neumann
10/21/2022, 10:24 PMfrom prefect.filesystems import S3
block = S3(bucket_path="my-bucket/example-dir")
block.save("example-block", overwrite=True)
block2 = S3.load("example-block")
block2.get_directory()
from prefect_aws.credentials import AwsCredentials
from prefect_aws.s3 import S3Bucket
aws_credentials = AwsCredentials(region="YOUR_REGION")
block = S3Bucket(bucket_name="my-bucket", basepath="example-dir", aws_credentials=aws_credentials)
block.save("example-block", overwrite=True)
block2 = S3Bucket.load("example-block")
block2.get_directory()
Stephen Herron
10/22/2022, 12:00 AMNico Neumann
10/22/2022, 12:00 AMStephen Herron
10/22/2022, 12:07 AMNico Neumann
10/22/2022, 12:37 AM