Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add s3-sync sidecar #828

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: add s3-sync sidecar #828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ef6a6b8
feat: add s3-sync sidecar
chirichidi Feb 16, 2024
6e6bc87
test: test the "ct lint-and-install --all --target-branch main"
chirichidi Feb 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion charts/airflow/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,8 @@ Parameter | Description | Default
--- | --- | ---
`dags.path` | the airflow dags folder | `/opt/airflow/dags`
`dags.persistence.*` | configs for the dags PVC | `<see values.yaml>`
`dags.gitSync.*` | configs for the git-sync sidecar | `<see values.yaml>`
`dags.gitSync.*` | configs for the git sync sidecars | `<see values.yaml>`
`dags.s3Sync.*` | configs for the s3 sync sidecars | `<see values.yaml>`

</details>

Expand Down
5 changes: 4 additions & 1 deletion charts/airflow/files/pod_template.kubernetes-helm-yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,17 @@ spec:
securityContext:
{{- $podSecurityContext | nindent 4 }}
{{- end }}
{{- if or ($extraPipPackages) (.Values.dags.gitSync.enabled) (.Values.airflow.kubernetesPodTemplate.extraInitContainers) }}
{{- if or ($extraPipPackages) (.Values.dags.gitSync.enabled) (.Values.dags.s3Sync.enabled) (.Values.airflow.kubernetesPodTemplate.extraInitContainers) }}
initContainers:
{{- if $extraPipPackages }}
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }}
{{- end }}
{{- if .Values.dags.gitSync.enabled }}
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
{{- end }}
{{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }}
{{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }}
{{- end }}
Expand Down
4 changes: 4 additions & 0 deletions charts/airflow/sample-values-CeleryExecutor.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,10 @@ dags:
gitSync:
enabled: false

## configs for the s3-sync sidecar
s3Sync:
enabled: false

###################################
## CONFIG | Kubernetes Ingress
###################################
Expand Down
4 changes: 4 additions & 0 deletions charts/airflow/sample-values-CeleryKubernetesExecutor.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,10 @@ dags:
gitSync:
enabled: false

## configs for the s3-sync sidecar
s3Sync:
enabled: false

###################################
## CONFIG | Kubernetes Ingress
###################################
Expand Down
4 changes: 4 additions & 0 deletions charts/airflow/sample-values-KubernetesExecutor.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,10 @@ dags:
gitSync:
enabled: false

## configs for the s3-sync sidecar
s3Sync:
enabled: false

###################################
## CONFIG | Kubernetes Ingress
###################################
Expand Down
2 changes: 2 additions & 0 deletions charts/airflow/templates/_helpers/common.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ The path containing DAG files
{{- define "airflow.dags.path" -}}
{{- if .Values.dags.gitSync.enabled -}}
{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.gitSync.repoSubPath | trimAll "/") -}}
{{- else if .Values.dags.s3Sync.enabled -}}
{{- printf "%s/repo/%s" (.Values.dags.path | trimSuffix "/") (.Values.dags.s3Sync.repoSubPath | trimAll "/") -}}
{{- else -}}
{{- printf .Values.dags.path -}}
{{- end -}}
Expand Down
65 changes: 65 additions & 0 deletions charts/airflow/templates/_helpers/pods.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,65 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
{{- end }}
{{- end }}



{{/*
Define a container which regularly syncs a s3-repo
EXAMPLE USAGE: {{ include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") }}
*/}}
{{- define "airflow.container.s3_sync" }}
{{- if .sync_one_time }}
- name: dags-s3-clone
{{- else }}
- name: dags-s3-sync
{{- end }}
image: {{ .Values.dags.s3Sync.image.repository }}:{{ .Values.dags.s3Sync.image.tag }}
imagePullPolicy: {{ .Values.dags.s3Sync.image.pullPolicy }}
securityContext:
runAsUser: {{ .Values.dags.s3Sync.image.uid }}
runAsGroup: {{ .Values.dags.s3Sync.image.gid }}
resources:
{{- toYaml .Values.dags.s3Sync.resources | nindent 4 }}
envFrom:
{{- include "airflow.envFrom" . | indent 4 }}
env:
{{- if .sync_one_time }}
- name: S3_SYNC_ONE_TIME
value: "true"
{{- end }}
{{- if .Values.dags.s3Sync.profile }}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ .Values.dags.s3Sync.profile }}
key: {{ .Values.dags.s3Sync.accessKey }}
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ .Values.dags.s3Sync.profile }}
key: {{ .Values.dags.s3Sync.secretKey }}
{{- end }}
{{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}}
{{- include "airflow.env" . | indent 4 }}
volumeMounts:
- name: dags-data
mountPath: {{ .Values.dags.path }}
command:
- "/bin/sh"
- "-c"
{{- if $.sync_one_time }}
- |
aws s3 cp --recursive s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} {{ include "airflow.dags.path" . }}
{{- else }}
- |
while true; do
aws s3 sync --delete s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} {{ include "airflow.dags.path" . }}
sleep {{ $.Values.dags.s3Sync.syncWait }}
done
{{- end }}
{{- end }}


{{/*
Define a container which regularly deletes airflow logs older than a retention period.
EXAMPLE USAGE: {{ include "airflow.container.log_cleanup" (dict "Release" .Release "Values" .Values "resources" $lc_resources "retention_min" $lc_retention_min "interval_sec" $lc_interval_sec) }}
Expand Down Expand Up @@ -361,6 +420,9 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value
readOnly: true
{{- end }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags-data
mountPath: {{ .Values.dags.path }}
{{- else if .Values.dags.s3Sync.enabled }}
- name: dags-data
mountPath: {{ .Values.dags.path }}
{{- end }}
Expand Down Expand Up @@ -424,6 +486,9 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V
claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }}
{{- end }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags-data
emptyDir: {}
{{- else if .Values.dags.s3Sync.enabled }}
- name: dags-data
emptyDir: {}
{{- end }}
Expand Down
10 changes: 10 additions & 0 deletions charts/airflow/templates/_helpers/validate-values.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,16 @@
{{- end }}
{{- end }}

{{/* Checks for `dags.s3Sync` */}}
{{- if .Values.dags.s3Sync.enabled }}
{{- if .Values.dags.persistence.enabled }}
{{ required "If `dags.s3Sync.enabled=true`, then `persistence.enabled` must be disabled!" nil }}
{{- end }}
{{- if not .Values.dags.s3Sync.bucket }}
{{ required "If `dags.s3Sync.enabled=true`, then `dags.s3Sync.bucket` must be non-empty!" nil }}
{{- end }}
{{- end }}

{{/* Checks for `ingress` */}}
{{- if .Values.ingress.enabled }}
{{/* Checks for `ingress.apiVersion` */}}
Expand Down
4 changes: 4 additions & 0 deletions charts/airflow/templates/config/secret-config-envs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,10 @@ data:
AIRFLOW__WEBSERVER__WEB_SERVER_PORT: {{ "8080" | b64enc | quote }}
AIRFLOW__CELERY__FLOWER_PORT: {{ "5555" | b64enc | quote }}

{{- if and (.Values.dags.s3Sync.enabled) (not .Values.airflow.config.AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL) }}
## refresh the dags folder at the same frequency as s3-sync
AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: {{ .Values.dags.s3Sync.syncWait | toString | b64enc | quote }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.airflow.config.AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL) }}
## refresh the dags folder at the same frequency as git-sync
AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: {{ .Values.dags.gitSync.syncWait | toString | b64enc | quote }}
Expand Down
6 changes: 6 additions & 0 deletions charts/airflow/templates/db-migrations/db-migrations-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
- name: db-migrations
Expand All @@ -111,6 +114,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
volumes:
{{- $volumes | indent 8 }}
- name: scripts
Expand Down
3 changes: 3 additions & 0 deletions charts/airflow/templates/db-migrations/db-migrations-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
- name: db-migrations
Expand Down
6 changes: 6 additions & 0 deletions charts/airflow/templates/flower/flower-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down Expand Up @@ -155,6 +158,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
{{- if .Values.airflow.extraContainers }}
{{- toYaml .Values.airflow.extraContainers | nindent 8 }}
{{- end }}
Expand Down
6 changes: 6 additions & 0 deletions charts/airflow/templates/scheduler/scheduler-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ spec:
{{- if .Values.dags.gitSync.enabled }}
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- if .Values.scheduler.extraInitContainers }}
Expand Down Expand Up @@ -231,6 +234,9 @@ spec:
{{- if .Values.dags.gitSync.enabled }}
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
{{- if .Values.scheduler.logCleanup.enabled }}
{{- $lc_resources := .Values.scheduler.logCleanup.resources }}
{{- $lc_retention_min := .Values.scheduler.logCleanup.retentionMinutes }}
Expand Down
9 changes: 8 additions & 1 deletion charts/airflow/templates/sync/sync-connections-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
## s3-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down Expand Up @@ -117,6 +121,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
volumes:
{{- $volumes | indent 8 }}
- name: scripts
Expand Down Expand Up @@ -147,4 +154,4 @@ spec:
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
3 changes: 3 additions & 0 deletions charts/airflow/templates/sync/sync-connections-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down
6 changes: 6 additions & 0 deletions charts/airflow/templates/sync/sync-pools-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand All @@ -112,6 +115,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
volumes:
{{- $volumes | indent 8 }}
- name: scripts
Expand Down
3 changes: 3 additions & 0 deletions charts/airflow/templates/sync/sync-pools-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down
7 changes: 7 additions & 0 deletions charts/airflow/templates/sync/sync-users-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
## s3-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down Expand Up @@ -117,6 +121,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
volumes:
{{- $volumes | indent 8 }}
- name: scripts
Expand Down
3 changes: 3 additions & 0 deletions charts/airflow/templates/sync/sync-users-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down
7 changes: 7 additions & 0 deletions charts/airflow/templates/sync/sync-variables-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
## s3-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down Expand Up @@ -117,6 +121,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" . | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" . | indent 8 }}
{{- end }}
volumes:
{{- $volumes | indent 8 }}
- name: scripts
Expand Down
3 changes: 3 additions & 0 deletions charts/airflow/templates/sync/sync-variables-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ spec:
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- if .Values.dags.s3Sync.enabled }}
{{- include "airflow.container.s3_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
{{- end }}
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
containers:
Expand Down
Loading
Loading