https://prefect.io logo
Join Slack
Powered by
How to use any *global* object/variable/parameter ...
# prefect-server
f
How to use any global object/variable/parameter in task definitions. For example, my flow fails when I declare the client in the global scope in the below code 
#!/usr/bin/python3
from prefect import task, Flow import boto3 import json client = boto3.client('sagemaker', region_name='us-east-1'                       ) @task def list_training_jobs():     response = client.list_training_jobs()     print(response) with Flow("SageMakerFlow") as flow:     training_jobs = list_training_jobs()     print(training_jobs)      flow.register(project_name="tutorial") flow.run()
Error 
<Task: list_training_jobs>
Traceback (most recent call last): File "first_project.py", line 21, in <module> flow.register(project_name="tutorial") File "/usr/local/lib/python3.7/site-packages/prefect/core/flow.py", line 1669, in register idempotency_key=idempotency_key, File "/usr/local/lib/python3.7/site-packages/prefect/client/client.py", line 782, in register serialized_flow = flow.serialize(build=build) # type: Any File "/usr/local/lib/python3.7/site-packages/prefect/core/flow.py", line 1440, in serialize self.storage.add_flow(self) File "/usr/local/lib/python3.7/site-packages/prefect/storage/local.py", line 143, in add_flow flow_location = flow.save(flow_location) File "/usr/local/lib/python3.7/site-packages/prefect/core/flow.py", line 1533, in save cloudpickle.dump(self, f) File "/usr/local/lib/python3.7/site-packages/cloudpickle/cloudpickle_fast.py", line 88, in dump CloudPickler(file, protocol=protocol).dump(obj) File "/usr/local/lib/python3.7/site-packages/cloudpickle/cloudpickle_fast.py", line 563, in dump return Pickler.dump(self, obj) File "/usr/lib64/python3.7/pickle.py", line 437, in dump self.save(obj) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 914, in save_set save(item) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/local/lib/python3.7/site-packages/cloudpickle/cloudpickle_fast.py", line 745, in save_function *self._dynamic_function_reduce(obj), obj=obj File "/usr/local/lib/python3.7/site-packages/cloudpickle/cloudpickle_fast.py", line 687, in _save_reduce_pickle5 save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple save(element) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 890, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 890, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 634, in save_reduce save(args) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple save(element) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 819, in save_list self._batch_appends(obj) File "/usr/lib64/python3.7/pickle.py", line 846, in _batch_appends save(tmp[0]) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 638, in save_reduce save(args) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple save(element) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 638, in save_reduce save(args) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 774, in save_tuple save(element) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 549, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python3.7/pickle.py", line 662, in save_reduce save(state) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python3.7/pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "/usr/lib64/python3.7/pickle.py", line 885, in _batch_setitems save(v) File "/usr/lib64/python3.7/pickle.py", line 524, in save rv = reduce(self.proto) TypeError: can't pickle SSLContext objects
s
It's trying to pickle the boto client which doesn't work. Instantiate the boto client inside the task.
👍 2
4 Views
Open in Slack
PreviousNext
Powered by