sqlboi
08/31/2023, 5:50 PMcredentials:
AWS_ACCESS_KEY_ID: "{{ $AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "{{ $AWS_SECRET_ACCESS_KEY }}"
AWS_DEFAULT_REGION: "{{ $AWS_REGION }}"
im getting:
TypeError: client() got an unexpected keyword argument 'AWS_ACCESS_KEY_ID'
so hoping i just got the wrong key for this...Jake Kaplan
08/31/2023, 5:52 PMsqlboi
08/31/2023, 5:53 PMSuccessfully installed mypy-boto3-s3-1.28.36 mypy-boto3-secretsmanager-1.28.36 prefect-aws-0.3.7 tenacity-8.2.3
Traceback (most recent call last):
File "/job/.venv/lib/python3.9/site-packages/prefect/deployments/steps/core.py", line 122, in run_steps
step_output = await run_step(step, upstream_outputs)
File "/job/.venv/lib/python3.9/site-packages/prefect/deployments/steps/core.py", line 93, in run_step
result = await from_async.call_soon_in_new_thread(
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 291, in aresult
return await asyncio.wrap_future(self.future)
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 315, in _run_sync
result = self.fn(*self.args, **self.kwargs)
File "/job/.venv/lib/python3.9/site-packages/prefect_aws/deployments/steps.py", line 99, in push_to_s3
client = boto3.client(
File "/job/.venv/lib/python3.9/site-packages/boto3/__init__.py", line 92, in client
return _get_default_session().client(*args, **kwargs)
TypeError: client() got an unexpected keyword argument 'AWS_ACCESS_KEY_ID'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/job/.venv/lib/python3.9/site-packages/prefect/cli/_utilities.py", line 41, in wrapper
return fn(*args, **kwargs)
File "/job/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 255, in coroutine_wrapper
return call()
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 382, in __call__
return self.result()
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 282, in result
return self.future.result(timeout=timeout)
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 168, in result
return self.__get_result()
File "/usr/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 345, in _run_async
result = await coro
File "/job/.venv/lib/python3.9/site-packages/prefect/cli/deploy.py", line 249, in deploy
await _run_single_deploy(
File "/job/.venv/lib/python3.9/site-packages/prefect/client/utilities.py", line 51, in with_injected_client
return await fn(*args, **kwargs)
File "/job/.venv/lib/python3.9/site-packages/prefect/cli/deploy.py", line 511, in _run_single_deploy
await run_steps(push_steps, step_outputs, print_function=app.console.print)
File "/job/.venv/lib/python3.9/site-packages/prefect/deployments/steps/core.py", line 150, in run_steps
raise StepExecutionError(f"Encountered error while running {fqn}") from exc
prefect.deployments.steps.core.StepExecutionError: Encountered error while running prefect_aws.deployments.steps.push_to_s3
An exception occurred.
another jake...Jake Kaplan
08/31/2023, 5:55 PMaws_access_key_id
(lowercase) for the 3 inputs?sqlboi
08/31/2023, 5:58 PMJake Kaplan
08/31/2023, 5:59 PMregion_name
I think for the param namesqlboi
08/31/2023, 6:00 PMTraceback (most recent call last):
File "/job/.venv/lib/python3.9/site-packages/prefect/deployments/steps/core.py", line 122, in run_steps
step_output = await run_step(step, upstream_outputs)
File "/job/.venv/lib/python3.9/site-packages/prefect/deployments/steps/core.py", line 93, in run_step
result = await from_async.call_soon_in_new_thread(
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 291, in aresult
return await asyncio.wrap_future(self.future)
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 315, in _run_sync
result = self.fn(*self.args, **self.kwargs)
File "/job/.venv/lib/python3.9/site-packages/prefect_aws/deployments/steps.py", line 99, in push_to_s3
client = boto3.client(
File "/job/.venv/lib/python3.9/site-packages/boto3/__init__.py", line 92, in client
return _get_default_session().client(*args, **kwargs)
File "/job/.venv/lib/python3.9/site-packages/boto3/session.py", line 299, in client
return self._session.create_client(
File "/job/.venv/lib/python3.9/site-packages/botocore/session.py", line 997, in create_client
client = client_creator.create_client(
File "/job/.venv/lib/python3.9/site-packages/botocore/client.py", line 159, in create_client
client_args = self._get_client_args(
File "/job/.venv/lib/python3.9/site-packages/botocore/client.py", line 490, in _get_client_args
return args_creator.get_client_args(
File "/job/.venv/lib/python3.9/site-packages/botocore/args.py", line 137, in get_client_args
endpoint = endpoint_creator.create_endpoint(
File "/job/.venv/lib/python3.9/site-packages/botocore/endpoint.py", line 402, in create_endpoint
raise ValueError("Invalid endpoint: %s" % endpoint_url)
ValueError: Invalid endpoint: <https://s3>..<http://amazonaws.com|amazonaws.com>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/job/.venv/lib/python3.9/site-packages/prefect/cli/_utilities.py", line 41, in wrapper
return fn(*args, **kwargs)
File "/job/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 255, in coroutine_wrapper
return call()
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 382, in __call__
return self.result()
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 282, in result
return self.future.result(timeout=timeout)
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 168, in result
return self.__get_result()
File "/usr/lib/python3.9/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/job/.venv/lib/python3.9/site-packages/prefect/_internal/concurrency/calls.py", line 345, in _run_async
result = await coro
File "/job/.venv/lib/python3.9/site-packages/prefect/cli/deploy.py", line 249, in deploy
await _run_single_deploy(
File "/job/.venv/lib/python3.9/site-packages/prefect/client/utilities.py", line 51, in with_injected_client
return await fn(*args, **kwargs)
File "/job/.venv/lib/python3.9/site-packages/prefect/cli/deploy.py", line 511, in _run_single_deploy
await run_steps(push_steps, step_outputs, print_function=app.console.print)
File "/job/.venv/lib/python3.9/site-packages/prefect/deployments/steps/core.py", line 150, in run_steps
raise StepExecutionError(f"Encountered error while running {fqn}") from exc
prefect.deployments.steps.core.StepExecutionError: Encountered error while running prefect_aws.deployments.steps.push_to_s3
An exception occurred.
Jake Kaplan
08/31/2023, 6:02 PMValueError: Invalid endpoint: <https://s3>..<http://amazonaws.com|amazonaws.com>
sqlboi
08/31/2023, 6:02 PMpull:
- prefect_aws.deployments.steps.pull_from_s3:
requires: prefect-aws>=0.3.0
bucket: mybucket-name
folder: "{{ push-code.folder }}"
credentials:
aws_access_key_id: "{{ $AWS_ACCESS_KEY_ID }}"
aws_secret_access_key: "{{ $AWS_SECRET_ACCESS_KEY }}"
region_name: "{{ $AWS_REGION }}"
Jake Kaplan
08/31/2023, 6:10 PM<https://s3>._region-code_.<http://amazonaws.com/|amazonaws.com/>
''
for the region$AWS_REGION
is a valid env varsqlboi
08/31/2023, 6:11 PMServer-side encryption with AWS Key Management Service keys (SSE-KMS)
because i didnt see it in the boto link u sent and got:
Flow could not be retrieved from deployment.
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/prefect/deployments/steps/core.py", line 122, in run_steps
step_output = await run_step(step, upstream_outputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/deployments/steps/core.py", line 93, in run_step
result = await from_async.call_soon_in_new_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py", line 291, in aresult
return await asyncio.wrap_future(self.future)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py", line 315, in _run_sync
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect_aws/deployments/steps.py", line 187, in pull_from_s3
for result in paginator.paginate(Bucket=bucket, Prefix=folder):
File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 269, in __iter__
response = self._make_request(current_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 357, in _make_request
return self._method(**current_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 535, in _api_call
return self._make_api_call(operation_name, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 980, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/prefect/engine.py", line 394, in retrieve_flow_then_begin_flow_run
flow = await load_flow_from_flow_run(flow_run, client=client)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/client/utilities.py", line 51, in with_injected_client
return await fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/deployments/deployments.py", line 206, in load_flow_from_flow_run
output = await run_steps(deployment.pull_steps)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/deployments/steps/core.py", line 150, in run_steps
raise StepExecutionError(f"Encountered error while running {fqn}") from exc
prefect.deployments.steps.core.StepExecutionError: Encountered error while running prefect_aws.deployments.steps.pull_from_s3
Jake Kaplan
08/31/2023, 6:24 PMbotocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied
sqlboi
08/31/2023, 6:25 PMJake Kaplan
08/31/2023, 6:28 PMsqlboi
08/31/2023, 6:37 PM