M Taufik
11/12/2020, 5:19 AM- name: PREFECT__CLOUD__API
value:
NAME READY STATUS RESTARTS AGE
prefect-poc-agents-k8s-agent-544485858c-6fwzh 2/2 Running 0 2m31s
prefect-poc-apollo-55496847c8-7zwz8 0/1 Running 3 2m31s
prefect-poc-graphql-5ddd4b8bcf-mxv6t 1/1 Running 0 2m15s
prefect-poc-hasura-7cbcd4b57c-kfjj8 1/1 Running 1 2m31s
prefect-poc-postgresql-0 1/1 Running 0 2m31s
prefect-poc-towel-8b554567f-pvj29 1/1 Running 0 2m31s
prefect-poc-ui-5cd448f6b5-8bbrw 1/1 Running 0 2m31s
{{ if .Values.agents.k8s.enabled }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "prefect-server.nameField" . }}-k8s-agent
labels: {{ include "prefect-server.commonLabels" . | nindent 4 }}
<http://app.kubernetes.io/component|app.kubernetes.io/component>: k8s-agent
spec:
replicas: 1
selector:
matchLabels: {{ include "prefect-server.matchLabels" . | nindent 6 }}
<http://app.kubernetes.io/component|app.kubernetes.io/component>: k8s-agent
strategy:
type: RollingUpdate
template:
metadata:
labels: {{ include "prefect-server.commonLabels" . | nindent 8 }}
<http://app.kubernetes.io/component|app.kubernetes.io/component>: k8s-agent
spec:
containers:
- name: k8s-agent
image: "{{ .Values.agents.k8s.image.repository }}:{{ .Values.agents.k8s.image.tag }}"
imagePullPolicy: {{ .Values.agents.k8s.image.pullPolicy }}
command: ["/bin/bash", "-c"]
args:
- "prefect agent start -v kubernetes"
resources: {{ toYaml .Values.agents.k8s.resources | nindent 12 }}
env:
- name: PREFECT__CLOUD__AGENT__AUTH_TOKEN
value: ''
- name: PREFECT__CLOUD__API
value: {{ include "prefect-server.apollo-api-url" . }}
- name: NAMESPACE
value: default
- name: IMAGE_PULL_SECRETS
value: ''
- name: PREFECT__CLOUD__AGENT__LABELS
value: '[]'
- name: JOB_MEM_REQUEST
value: ''
- name: JOB_MEM_LIMIT
value: ''
- name: JOB_CPU_REQUEST
value: ''
- name: JOB_CPU_LIMIT
value: ''
- name: PREFECT__BACKEND
value: server
- name: PREFECT__CLOUD__AGENT__AGENT_ADDRESS
value: <http://127.0.0.1:8080>
{{/* ports:*/}}
{{/* - name: liveness-port*/}}
{{/* containerPort: 8080*/}}
{{/* hostPort: 8080*/}}
{{/* livenessProbe:*/}}
{{/* failureThreshold: 2*/}}
{{/* httpGet:*/}}
{{/* path: /api/health*/}}
{{/* port: liveness-port*/}}
{{/* initialDelaySeconds: 40*/}}
{{/* periodSeconds: 40*/}}
{{ if .Values.agents.k8s.resourceManagerEnabled }}
- name: resource-manager
image: "{{ .Values.agents.k8s.image.repository }}:{{ .Values.agents.k8s.image.tag }}"
imagePullPolicy: {{ .Values.agents.k8s.image.pullPolicy }}
command: ["/bin/bash", "-c"]
args:
[
"python -c 'from prefect.agent.kubernetes import ResourceManager; ResourceManager().start()'",
]
env:
- name: PREFECT__CLOUD__AGENT__AUTH_TOKEN
value: ""
- name: PREFECT__CLOUD__API
value: {{ include "prefect-server.apollo-api-url" . }}
- name: PREFECT__CLOUD__AGENT__RESOURCE_MANAGER__LOOP_INTERVAL
value: "60"
- name: PREFECT__BACKEND
value: server
- name: NAMESPACE
value: "default"
resources:
limits:
memory: 128Mi
cpu: 100m
{{ end }}
restartPolicy: Always
nodeSelector: {{ toYaml .Values.agents.k8s.nodeSelector | nindent 8 }}
affinity: {{ toYaml .Values.agents.k8s.affinity | nindent 8 }}
tolerations: {{ toYaml .Values.agents.k8s.tolerations | nindent 8 }}
---
apiVersion: <http://rbac.authorization.k8s.io/v1|rbac.authorization.k8s.io/v1>
kind: Role
metadata:
namespace: default
name: prefect-agent-rbac
rules:
- apiGroups: ["batch", "extensions"]
resources: ["jobs"]
verbs: ["*"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["*"]
---
kind: Role
apiVersion: <http://rbac.authorization.k8s.io/v1beta1|rbac.authorization.k8s.io/v1beta1>
metadata:
name: daskKubernetes
rules:
- apiGroups:
- "" # indicates the core API group
resources:
- "pods"
verbs:
- "get"
- "list"
- "watch"
- "create"
- "delete"
- apiGroups:
- "" # indicates the core API group
resources:
- "pods/log"
verbs:
- "get"
- "list"
- apiGroups:
- "" # indicates the core API group
resources:
- "services"
verbs:
- "get"
- "list"
- "watch"
- "create"
- "delete"
---
apiVersion: <http://rbac.authorization.k8s.io/v1beta1|rbac.authorization.k8s.io/v1beta1>
kind: RoleBinding
metadata:
namespace: default
name: prefect-agent-rbac
subjects:
- kind: ServiceAccount
name: default
roleRef:
kind: Role
name: prefect-agent-rbac
apiGroup: <http://rbac.authorization.k8s.io|rbac.authorization.k8s.io>
---
apiVersion: <http://rbac.authorization.k8s.io/v1beta1|rbac.authorization.k8s.io/v1beta1>
kind: RoleBinding
metadata:
namespace: default
name: daskKubernetes
subjects:
- kind: ServiceAccount
name: default
roleRef:
kind: Role
name: daskKubernetes
apiGroup: <http://rbac.authorization.k8s.io|rbac.authorization.k8s.io>
{{ end }}
[2020-11-12 05:59:47,604] INFO - agent | Starting KubernetesAgent with labels []
[2020-11-12 05:59:47,605] INFO - agent | Agent documentation can be found at <https://docs.prefect.io/orchestration/>
[2020-11-12 05:59:47,605] INFO - agent | Agent connecting to the Prefect API at <http://prefect-poc-apollo.default.svc.cluster.local:4200>
[2020-11-12 06:00:17,632] ERROR - agent | There was an error connecting to <http://prefect-poc-apollo.default.svc.cluster.local:4200>
[2020-11-12 06:00:17,632] ERROR - agent | HTTPConnectionPool(host='prefect-poc-apollo.default.svc.cluster.local', port=4200): Max retries exceeded with url: /graphql/alpha (
Caused by ConnectTimeoutError(<urllib3.connection.HTTPConnection object at 0x7fb657212130>, 'Connection to prefect-poc-apollo.default.svc.cluster.local timed out. (connect t
imeout=30)'))
[2020-11-12 06:00:17,632] INFO - agent | Waiting for flow runs...
[2020-11-12 06:00:17,633] DEBUG - agent | Agent API server listening on port <http://127.0.0.1:8080>
[2020-11-12 06:00:17,635] DEBUG - agent | Max Workers: 6
Dylan
Zanie
M Taufik
11/13/2020, 1:33 AMprefect-poc-agents-k8s-agent-544485858c-6fwzh 2/2 Running 0 2m31s
prefect-poc-apollo-55496847c8-7zwz8 0/1 Running 3 2m31s
prefect-poc-graphql-5ddd4b8bcf-mxv6t 1/1 Running 0 2m15s
prefect-poc-hasura-7cbcd4b57c-kfjj8 1/1 Running 1 2m31s
prefect-poc-postgresql-0 1/1 Running 0 2m31s
prefect-poc-towel-8b554567f-pvj29 1/1 Running 0 2m31s
prefect-poc-ui-5cd448f6b5-8bbrw 1/1 Running 0 2m31s
2020-11-13 09:24:27,121] INFO - agent | Starting KubernetesAgent with labels []
[2020-11-13 09:24:27,121] INFO - agent | Agent documentation can be found at <https://docs.prefect.io/orchestration/>
[2020-11-13 09:24:27,121] INFO - agent | Agent connecting to the Prefect API at <http://prefect-apollo:4200>
[2020-11-13 09:24:27,132] INFO - agent | Waiting for flow runs...
[2020-11-13 09:24:27,236] ERROR - agent | 400 Client Error: Bad Request for url: <http://prefect-apollo:4200>
This is likely caused by a poorly formatted GraphQL query or mutation. GraphQL sent:
query {
mutation($input: get_runs_in_queue_input!) {
get_runs_in_queue(input: $input) {
flow_run_ids
}
}
}
variables {
{"input": {"before": "2020-11-13T09:24:27.134876+00:00", "labels": [], "tenant_id": null}}
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=4200): Max retries exceeded with url: /graphql (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7ff8c962e890>: Failed to establish a new connection: [Errno 111] Connection refused'))
nicholas
PREFECT_SERVER__ENDPOINT
to the publicly exposed Apollo endpoint of your deployment.M Taufik
11/14/2020, 4:53 AM