diff --git a/charts/airflow/files/pod_template.kubernetes-helm-yaml b/charts/airflow/files/pod_template.kubernetes-helm-yaml index fff65d0b..b27f765b 100644 --- a/charts/airflow/files/pod_template.kubernetes-helm-yaml +++ b/charts/airflow/files/pod_template.kubernetes-helm-yaml @@ -49,8 +49,12 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 4 }} + {{- else }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }} {{- end }} + {{- end }} {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }} {{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }} {{- end }} diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index e17638d9..5387ef96 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -254,8 +254,6 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release name: {{ .Values.dags.gitSync.httpSecret }} key: {{ .Values.dags.gitSync.httpSecretPasswordKey }} {{- end }} - {{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}} - {{- include "airflow.env" . | indent 4 }} volumeMounts: - name: dags-data mountPath: /dags @@ -273,6 +271,26 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release {{- end }} {{- end }} +{{/* +Define a container which copies the contents of the DAG PVC to the DAG directory in the pod +EXAMPLE USAGE: {{ include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) }} +*/}} +{{- define "airflow.container.dag_pvc_copy" }} +- name: dags-copy-from-pvc + image: busybox:1.35 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: {{ .Values.airflow.image.uid }} + runAsGroup: {{ .Values.airflow.image.gid }} + volumeMounts: + - name: original-dag-content + mountPath: /dags/original-dags + - name: dags-data + mountPath: /dags/copied-dags + command: ["/bin/sh","-c"] + args: ["cp -rL /dags/original-dags/repo /dags/copied-dags/"] +{{- end }} + {{/* Define a container which regularly deletes airflow logs older than a retention period. EXAMPLE USAGE: {{ include "airflow.container.log_cleanup" (dict "Release" .Release "Values" .Values "resources" $lc_resources "retention_min" $lc_retention_min "interval_sec" $lc_interval_sec) }} @@ -351,12 +369,17 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value {{- /* dags */ -}} {{- if .Values.dags.persistence.enabled }} +{{- if .Values.dags.gitSync.enabled }} +- name: dags-data + mountPath: {{ .Values.dags.path }} +{{- else }} - name: dags-data mountPath: {{ .Values.dags.path }} subPath: {{ .Values.dags.persistence.subPath }} {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }} readOnly: true {{- end }} +{{- end }} {{- else if .Values.dags.gitSync.enabled }} - name: dags-data mountPath: {{ .Values.dags.path }} @@ -408,6 +431,18 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- /* dags */ -}} {{- if .Values.dags.persistence.enabled }} + +{{- if .Values.dags.gitSync.enabled }} +- name: original-dag-content + persistentVolumeClaim: + {{- if .Values.dags.persistence.existingClaim }} + claimName: {{ .Values.dags.persistence.existingClaim }} + {{- else }} + claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} + {{- end }} +- name: dags-data + emptyDir: {} +{{- else }} - name: dags-data persistentVolumeClaim: {{- if .Values.dags.persistence.existingClaim }} @@ -415,11 +450,14 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- else }} claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} {{- end }} +{{- end }} + {{- else if .Values.dags.gitSync.enabled }} - name: dags-data emptyDir: {} {{- end }} + {{- /* logs */ -}} {{- if .Values.logs.persistence.enabled }} - name: logs-data @@ -435,7 +473,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- end }} {{- /* git-sync */ -}} -{{- if .Values.dags.gitSync.enabled }} +{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- if .Values.dags.gitSync.sshSecret }} - name: git-secret secret: diff --git a/charts/airflow/templates/_helpers/validate-values.tpl b/charts/airflow/templates/_helpers/validate-values.tpl index 42944a98..0ddce053 100644 --- a/charts/airflow/templates/_helpers/validate-values.tpl +++ b/charts/airflow/templates/_helpers/validate-values.tpl @@ -84,10 +84,7 @@ {{- end }} {{/* Checks for `dags.gitSync` */}} -{{- if .Values.dags.gitSync.enabled }} - {{- if .Values.dags.persistence.enabled }} - {{ required "If `dags.gitSync.enabled=true`, then `persistence.enabled` must be disabled!" nil }} - {{- end }} +{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- if not .Values.dags.gitSync.repo }} {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }} {{- end }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml index 1ad60348..8aba39e1 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml @@ -83,9 +83,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: - name: db-migrations @@ -107,7 +111,7 @@ spec: - name: scripts mountPath: /mnt/scripts readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-job.yaml b/charts/airflow/templates/db-migrations/db-migrations-job.yaml index 2e729af6..81e93814 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-job.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-job.yaml @@ -77,9 +77,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: - name: db-migrations diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml index 22f5fa11..dbf88653 100644 --- a/charts/airflow/templates/flower/flower-deployment.yaml +++ b/charts/airflow/templates/flower/flower-deployment.yaml @@ -87,9 +87,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -151,7 +155,7 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/git-sync/git-sync-deployment.yaml b/charts/airflow/templates/git-sync/git-sync-deployment.yaml new file mode 100644 index 00000000..9e489707 --- /dev/null +++ b/charts/airflow/templates/git-sync/git-sync-deployment.yaml @@ -0,0 +1,173 @@ +{{- if and (.Values.dags.persistence.enabled) (.Values.dags.gitSync.enabled) }} +{{- $podNodeSelector := include "airflow.podNodeSelector" (dict "Release" .Release "Values" .Values "nodeSelector" .Values.dags.gitSync.nodeSelector) }} +{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.dags.gitSync.affinity) }} +{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.dags.gitSync.tolerations) }} +{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.dags.gitSync.securityContext) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-dags-gitsync + labels: + app: {{ include "airflow.labels.app" . }} + component: dags-git-sync + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.dags.gitSync.labels }} + {{- toYaml .Values.dags.gitSync.labels | nindent 4 }} + {{- end }} + {{- if .Values.dags.gitSync.annotations }} + annotations: + {{- toYaml .Values.dags.gitSync.annotations | nindent 4 }} + {{- end }} +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + ## multiple gitSync pods can safely run concurrently + maxSurge: 25% + maxUnavailable: 0 + selector: + matchLabels: + app: {{ include "airflow.labels.app" . }} + component: dags-git-sync + release: {{ .Release.Name }} + template: + metadata: + annotations: + {{- if .Values.airflow.podAnnotations }} + {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.dags.gitSync.podAnnotations }} + {{- toYaml .Values.dags.gitSync.podAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.dags.gitSync.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + labels: + app: {{ include "airflow.labels.app" . }} + component: dags-git-sync + release: {{ .Release.Name }} + {{- if .Values.dags.gitSync.podLabels }} + {{- toYaml .Values.dags.gitSync.podLabels | nindent 8 }} + {{- end }} + spec: + restartPolicy: Always + {{- if .Values.airflow.image.pullSecret }} + imagePullSecrets: + - name: {{ .Values.airflow.image.pullSecret }} + {{- end }} + {{- if $podNodeSelector }} + nodeSelector: + {{- $podNodeSelector | nindent 8 }} + {{- end }} + {{- if $podAffinity }} + affinity: + {{- $podAffinity | nindent 8 }} + {{- end }} + {{- if $podTolerations }} + tolerations: + {{- $podTolerations | nindent 8 }} + {{- end }} + {{- if $podSecurityContext }} + securityContext: + {{- $podSecurityContext | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "airflow.serviceAccountName" . }} + containers: + - name: dags-git-sync + image: {{ .Values.dags.gitSync.image.repository }}:{{ .Values.dags.gitSync.image.tag }} + imagePullPolicy: {{ .Values.dags.gitSync.image.pullPolicy }} + securityContext: + runAsUser: {{ .Values.dags.gitSync.image.uid }} + runAsGroup: {{ .Values.dags.gitSync.image.gid }} + resources: + {{- toYaml .Values.dags.gitSync.resources | nindent 12 }} + env: + - name: GIT_SYNC_ROOT + value: "/dags" + - name: GIT_SYNC_DEST + value: "repo" + - name: GIT_SYNC_REPO + value: {{ .Values.dags.gitSync.repo | quote }} + - name: GIT_SYNC_BRANCH + value: {{ .Values.dags.gitSync.branch | quote }} + - name: GIT_SYNC_REV + value: {{ .Values.dags.gitSync.revision | quote }} + - name: GIT_SYNC_DEPTH + value: {{ .Values.dags.gitSync.depth | quote }} + - name: GIT_SYNC_WAIT + value: {{ .Values.dags.gitSync.syncWait | quote }} + - name: GIT_SYNC_TIMEOUT + value: {{ .Values.dags.gitSync.syncTimeout | quote }} + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} + {{- if .Values.dags.gitSync.sshSecret }} + - name: GIT_SYNC_SSH + value: "true" + - name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/id_rsa" + {{- end }} + {{- if .Values.dags.gitSync.sshKnownHosts }} + - name: GIT_KNOWN_HOSTS + value: "true" + - name: GIT_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" + {{- else }} + - name: GIT_KNOWN_HOSTS + value: "false" + {{- end }} + {{- if .Values.dags.gitSync.httpSecret }} + - name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.httpSecret }} + key: {{ .Values.dags.gitSync.httpSecretUsernameKey }} + - name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.httpSecret }} + key: {{ .Values.dags.gitSync.httpSecretPasswordKey }} + {{- end }} + volumeMounts: + - name: dags-data + mountPath: /dags + {{- if .Values.dags.gitSync.sshSecret }} + - name: git-secret + mountPath: /etc/git-secret/id_rsa + readOnly: true + subPath: {{ .Values.dags.gitSync.sshSecretKey }} + {{- end }} + {{- if .Values.dags.gitSync.sshKnownHosts }} + - name: git-known-hosts + mountPath: /etc/git-secret/known_hosts + readOnly: true + subPath: known_hosts + {{- end }} + volumes: + - name: dags-data + persistentVolumeClaim: + {{- if .Values.dags.persistence.existingClaim }} + claimName: {{ .Values.dags.persistence.existingClaim }} + {{- else }} + claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} + {{- end }} + + {{- if .Values.dags.gitSync.sshSecret }} + - name: git-secret + secret: + secretName: {{ .Values.dags.gitSync.sshSecret }} + defaultMode: 0644 + {{- end }} + + {{- if .Values.dags.gitSync.sshKnownHosts }} + - name: git-known-hosts + secret: + secretName: {{ include "airflow.fullname" . }}-known-hosts + defaultMode: 0644 + {{- end }} + +{{- end }} \ No newline at end of file diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml index c43f3904..ecdbe656 100644 --- a/charts/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml @@ -95,8 +95,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} + ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- if .Values.scheduler.extraInitContainers }} @@ -196,7 +201,7 @@ spec: readOnly: true {{- end }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.scheduler.logCleanup.enabled }} diff --git a/charts/airflow/templates/sync/sync-connections-deployment.yaml b/charts/airflow/templates/sync/sync-connections-deployment.yaml index 82734305..57431beb 100644 --- a/charts/airflow/templates/sync/sync-connections-deployment.yaml +++ b/charts/airflow/templates/sync/sync-connections-deployment.yaml @@ -83,9 +83,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -113,7 +117,7 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-connections-job.yaml b/charts/airflow/templates/sync/sync-connections-job.yaml index c77002ad..1a344d40 100644 --- a/charts/airflow/templates/sync/sync-connections-job.yaml +++ b/charts/airflow/templates/sync/sync-connections-job.yaml @@ -77,9 +77,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-pools-deployment.yaml b/charts/airflow/templates/sync/sync-pools-deployment.yaml index 1635a987..a4ac484c 100644 --- a/charts/airflow/templates/sync/sync-pools-deployment.yaml +++ b/charts/airflow/templates/sync/sync-pools-deployment.yaml @@ -83,9 +83,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -108,7 +112,7 @@ spec: - name: scripts mountPath: /mnt/scripts readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-pools-job.yaml b/charts/airflow/templates/sync/sync-pools-job.yaml index cda0f7cd..72e5f978 100644 --- a/charts/airflow/templates/sync/sync-pools-job.yaml +++ b/charts/airflow/templates/sync/sync-pools-job.yaml @@ -77,9 +77,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-users-deployment.yaml b/charts/airflow/templates/sync/sync-users-deployment.yaml index d61d8943..0bce843a 100644 --- a/charts/airflow/templates/sync/sync-users-deployment.yaml +++ b/charts/airflow/templates/sync/sync-users-deployment.yaml @@ -83,9 +83,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -113,7 +117,7 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-users-job.yaml b/charts/airflow/templates/sync/sync-users-job.yaml index d96c8da7..896765d7 100644 --- a/charts/airflow/templates/sync/sync-users-job.yaml +++ b/charts/airflow/templates/sync/sync-users-job.yaml @@ -77,9 +77,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-variables-deployment.yaml b/charts/airflow/templates/sync/sync-variables-deployment.yaml index 21926197..6d583f68 100644 --- a/charts/airflow/templates/sync/sync-variables-deployment.yaml +++ b/charts/airflow/templates/sync/sync-variables-deployment.yaml @@ -83,9 +83,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -113,7 +117,7 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-variables-job.yaml b/charts/airflow/templates/sync/sync-variables-job.yaml index bd50349e..40a58525 100644 --- a/charts/airflow/templates/sync/sync-variables-job.yaml +++ b/charts/airflow/templates/sync/sync-variables-job.yaml @@ -77,9 +77,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml index b079f77e..b6f2de99 100644 --- a/charts/airflow/templates/triggerer/triggerer-deployment.yaml +++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml @@ -87,8 +87,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} + ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -147,7 +152,7 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml index 7af636fb..a0d07644 100644 --- a/charts/airflow/templates/webserver/webserver-deployment.yaml +++ b/charts/airflow/templates/webserver/webserver-deployment.yaml @@ -87,8 +87,13 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} + ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -140,7 +145,7 @@ spec: mountPath: /opt/airflow/webserver_config.py subPath: webserver_config.py readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml index e5147aa1..c5553a77 100644 --- a/charts/airflow/templates/worker/worker-statefulset.yaml +++ b/charts/airflow/templates/worker/worker-statefulset.yaml @@ -92,8 +92,12 @@ spec: {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: @@ -168,7 +172,7 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.workers.logCleanup.enabled }} diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index 569d78ff..0cb4aa2b 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -1385,6 +1385,8 @@ dags: ## maxFailures: 0 + securityContext: {} + ################################### ## CONFIG | Kubernetes Ingress ###################################