Nick Coy
09/19/2022, 4:12 PMNate
09/19/2022, 4:17 PMNick Coy
09/19/2022, 4:24 PMTraceback (most recent call last): File "/usr/local/lib/python3.10/site-packages/prefect/agent.py", line 227, in _submit_run_and_capture_errors result = await infrastructure.run(task_status=task_status) File "/usr/local/lib/python3.10/site-packages/prefect/infrastructure/kubernetes.py", line 230, in run job_name = await run_sync_in_worker_thread(self._create_job, manifest) File "/usr/local/lib/python3.10/site-packages/prefect/utilities/asyncutils.py", line 57, in run_sync_in_worker_thread return await anyio.to_thread.run_sync(call, cancellable=True) File "/usr/local/lib/python3.10/site-packages/anyio/to_thread.py", line 31, in run_sync return await get_asynclib().run_sync_in_worker_thread( File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread return await future File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 867, in run result = context.run(func, *args) File "/usr/local/lib/python3.10/site-packages/prefect/infrastructure/kubernetes.py", line 442, in _create_job job = batch_client.create_namespaced_job(self.namespace, job_manifest) File "/usr/local/lib/python3.10/site-packages/kubernetes/client/api/batch_v1_api.py", line 210, in create_namespaced_job return self.create_namespaced_job_with_http_info(namespace, body, **kwargs) # noqa: E501 File "/usr/local/lib/python3.10/site-packages/kubernetes/client/api/batch_v1_api.py", line 309, in create_namespaced_job_with_http_info return self.api_client.call_api( File "/usr/local/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 348, in call_api return self.__call_api(resource_path, method, File "/usr/local/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 180, in __call_api response_data = self.request( File "/usr/local/lib/python3.10/site-packages/kubernetes/client/api_client.py", line 391, in request return <http://self.rest_client.POST|self.rest_client.POST>(url, File "/usr/local/lib/python3.10/site-packages/kubernetes/client/rest.py", line 275, in POST return self.request("POST", url, File "/usr/local/lib/python3.10/site-packages/kubernetes/client/rest.py", line 234, in request raise ApiException(http_resp=r) kubernetes.client.exceptions.ApiException: (500)
Reason: Internal Server Error
HTTP response headers: HTTPHeaderDict({'Audit-Id': '2d947d59-9c20-4c13-b486-ad6dd5d0ca81', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': '533151b7-d867-4fe4-b076-855b6b27cb51', 'X-Kubernetes-Pf-Prioritylevel-Uid': '239c3359-fc1f-4d28-a5a1-0648fe373cef', 'Date': 'Mon, 19 Sep 2022 16:10:35 GMT', 'Content-Length': '264'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"The POST operation against Job.batch could not be completed at this time, please try again.","reason":"ServerTimeout","details":{"name":"POST","group":"batch","kind":"Job"},"code":500}
Nate
09/19/2022, 4:33 PMNick Coy
09/19/2022, 4:59 PMfrom prefect.infrastructure import KubernetesJob
from prefect.blocks.system import Secret
secret_prefect_url = Secret.load("prefect-api-url")
secret_prefect_api = Secret.load("prefect-api-key")
base_manifest = {
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": "k8-job-flow",
"namespace": "prefect",
"labels": {}
},
"spec": {
"ttlSecondsAfterFinished": 180,
"template": {
"spec": {
"serviceAccountName": "prefect2",
"parallelism": 1,
"completions": 1,
"restartPolicy": "Never",
"containers": [
{
"name": "prefect-job",
"env": []
}
]
}
}
}
}
k8_job = KubernetesJob(
image='<http://us.gcr.io/hl-database/prefect_docker:latest|us.gcr.io/hl-database/prefect_docker:latest>',
job=base_manifest,
env={'PREFECT_API_URL':secret_prefect_url.get(), "PREFECT_API_KEY":secret_prefect_api.get()},
image_pull_policy="IfNotPresent",
name="k8-job-flow",
namespace='prefect',
service_account_name='prefect2',
command=["python", "-m", "prefect.engine" ],
pod_watch_timeout_seconds=300
# finished_job_ttl=300, not in current prefect release, should be in next realease after 2.4.0
)
if __name__ == "__main__":
k8_job.save(name='k8-job-flow',overwrite=True)
Nate
09/19/2022, 8:43 PM