Faheem Khan
07/02/2022, 9:50 AMlistA = ['1', '2', '3']
@flow
def main():
def funcA():
print("A")
def funB():
print("b")
if name == '__main__':
pool = multiprocessing.Pool()
pool.map(funcA, listA)
pool.close()
pool.join()
DeploymentSpec(
name="docker-example-01",
flow=main,
flow_runner=DockerFlowRunner(
#image = 'prefect-orion:beta6',
image = 'prefect-orion-main_prefect-server',
image_pull_policy = 'IF_NOT_PRESENT',
volumes=["C😕Users/data:/data"],
networks = ['prefect-server'],
env = {
"USE_SSL": False,
"AWS_ACCESS_KEY_ID": "abc",
"AWS_SECRET_ACCESS_KEY": "abc",
"ENDPOINT_URL": 'http://minio:9000',
}
),
)Anna Geller
07/02/2022, 10:33 AMFaheem Khan
07/03/2022, 1:30 AMlistA = ['1', '2', '3']
@flow(task_runner=SequentialTaskRunner())
def main():
def funcA():
print("A")
def funB():
print("b")
if name == '__main__':
logger = get_run_logger()
logger.info(f"Hello")
pool = multiprocessing.Pool()
pool.map(funcA, listA)
pool.close()
pool.join()
DeploymentSpec(
name="docker-example-01",
flow=main,
flow_runner=DockerFlowRunner(
#image = 'prefect-orion:beta6',
image = 'prefect-orion-main_prefect-server',
image_pull_policy = 'IF_NOT_PRESENT',
volumes=["C😕Users/data:/data"],
networks = ['prefect-server'],
env = {
"USE_SSL": False,
"AWS_ACCESS_KEY_ID": "abc",
"AWS_SECRET_ACCESS_KEY": "abc",
"ENDPOINT_URL": 'http://minio:9000',
}
),
)Anna Geller
07/04/2022, 1:21 PMimport multiprocessing
from prefect import flow, get_run_logger
from prefect.task_runners import SequentialTaskRunner
from prefect.flow_runners import DockerFlowRunner
listA = ["1", "2", "3"]
def funcA():
print("A")
def funB():
print("b")
@flow(task_runner=SequentialTaskRunner())
def main():
logger = get_run_logger()
<http://logger.info|logger.info>(f"Hello")
pool = multiprocessing.Pool()
pool.map(funcA, listA)
pool.close()
pool.join()
if __name__ == "__main__":
main()
your multiprocessing code doesn't work:
File "/Users/anna/.conda/envs/prefect-2-0/lib/python3.10/multiprocessing/pool.py", line 364, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/Users/anna/.conda/envs/prefect-2-0/lib/python3.10/multiprocessing/pool.py", line 771, in get
raise self._value
TypeError: funcA() takes 0 positional arguments but 1 was given