#1 - Changes to the airflow helm values configuration for current ver…

…sion of helm chart
AtlasOfLivingAustralia · Dec 12, 2024 · 42d2f83 · 42d2f83
1 parent b8e7ed4
commit 42d2f83
Showing 1 changed file with 133 additions and 43 deletions.
diff --git a/airflow/values.yaml b/airflow/values.yaml
@@ -10,21 +10,21 @@ airflow:
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/airflow-version.md
     image:
       repository: apache/airflow
-      tag: 2.7.3-python3.9
+      tag: 2.9.0-python3.9
       pullPolicy: IfNotPresent
 
     ## the airflow executor type to use
-    executor: KubernetesExecutor
+    executor: CeleryExecutor
 
     ## the fernet encryption key (sets `AIRFLOW__CORE__FERNET_KEY`)
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/security/set-fernet-key.md
     ## [WARNING] change from default value to ensure security
-    fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc="
+    fernetKey: "<REPLACE_ME>"
 
     ## the secret_key for flask (sets `AIRFLOW__WEBSERVER__SECRET_KEY`)
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/security/set-webserver-secret-key.md
     ## [WARNING] change from default value to ensure security
-    webserverSecretKey: "THIS IS UNSAFE!"
+    webserverSecretKey: "<REPLACE_ME>"
 
     ## environment variables for airflow configs
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/airflow-configs.md
@@ -38,10 +38,10 @@ airflow:
     ## a list of users to create
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/security/airflow-users.md
     users:
-      - username: admin
-        password: admin
+      - username: "<REPLACE_ME>"
+        password: "<REPLACE_ME>"
         role: Admin
-        email: [email protected]
+        email: "<REPLACE_ME>"
         firstName: admin
         lastName: admin
 
@@ -140,7 +140,7 @@ airflow:
       - name: "AIRFLOW__API__AUTH_BACKENDS"
         value: 'airflow.api.auth.backend.basic_auth'
       - name: "SQLALCHEMY_SILENCE_UBER_WARNING"
-        value: "1"  
+        value: "1"
 
     ## extra VolumeMounts for the airflow Pods
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
@@ -203,11 +203,18 @@ airflow:
         cpu: 250m
         memory: 2Gi
 
-    ## configs for the log-cleanup sidecar of the scheduler
-    ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-cleanup.md
-    # logCleanup:
-    #   enabled: true
-    #   retentionMinutes: 21600
+    ## configs for the scheduler Pods' liveness probe
+    ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
+    livenessProbe:
+      enabled: true
+
+      ## configs for an additional check that ensures tasks are being created by the scheduler
+      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
+      taskCreationCheck:
+        enabled: false
+        thresholdSeconds: 300
+        schedulerAgeBeforeCheck: 180
+
 
   ###################################
   ## COMPONENT | Airflow Webserver
@@ -259,26 +266,34 @@ airflow:
 
     ## if the airflow workers StatefulSet should be deployed
     ##
-    enabled: false
-
-    ## if the airflow workers StatefulSet should be deployed
-    serviceAccount:
-      ## if a Kubernetes ServiceAccount is created
-      create: false
+    enabled: true
 
-      ## the name of the ServiceAccount
-      name: "airflow"
+    ## the number of worker Pods to run
+    replicas: 2
 
-      ## annotations for the ServiceAccount
-      annotations: {}
+    ## resource requests/limits for the worker Pods
+    ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
+    resources:
+      limits:
+        cpu: 1500m
+        memory: 8Gi
+      requests:
+        cpu: 750m
+        memory: 4Gi
 
+    ## configs for the log-cleanup sidecar of the worker Pods
+    ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-cleanup.md
+    logCleanup:
+      enabled: true
+      retentionMinutes: 21600
 
   ###################################
   ## COMPONENT | Triggerer
   ###################################
   triggerer:
-    ## if the airflow triggerer should be deployed
-    enabled: true
+    ## if the airflow triggerer should be deployed.
+    # Only if using deferrable operators
+    enabled: false
 
     ## the number of triggerer Pods to run
     replicas: 1
@@ -293,19 +308,40 @@ airflow:
         cpu: 1250m
         memory: 4Gi
 
+    ## maximum number of triggers each triggerer will run at once (sets `AIRFLOW__TRIGGERER__DEFAULT_CAPACITY`)
+    capacity: 1000
+
   ###################################
   ## COMPONENT | Flower
   ###################################
   flower:
     ## if the airflow flower UI should be deployed
-    enabled: false
+    enabled: true
+
+    ## the number of flower Pods to run
+    replicas: 1
+
+    ## resource requests/limits for the flower Pod
+    ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
+    resources:
+      limits:
+        cpu: 1500m
+        memory: 8Gi
+      requests:
+        cpu: 750m
+        memory: 4Gi
+
+    ## configs for the Service of the flower Pods
+    service:
+      type: ClusterIP
+      externalPort: 5555
 
   ###################################
   ## CONFIG | Airflow Logs
   ###################################
   logs:
     ## the airflow logs folder
-    # path: /opt/airflow/logs
+    path: /opt/airflow/logs
 
     ## configs for the logs PVC
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-persistence.md
@@ -352,7 +388,19 @@ airflow:
   ingress:
     ## if we should deploy Ingress resources
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/ingress.md
-    enabled: false
+    enabled: true
+
+    web:
+      host: "<REPLACE_ME>"
+      path:
+      ## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
+      ingressClassName: "alb"
+      annotations:
+        alb.ingress.kubernetes.io/scheme: internet-facing
+        alb.ingress.kubernetes.io/target-type: ip
+        alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:<AWS_REGION>:<AWS_ACCOUNT>:certificate/<REPLACE_ME>
+
+
 
   ###################################
   ## CONFIG | Kubernetes ServiceAccount
@@ -365,9 +413,8 @@ airflow:
     name: "airflow"
 
     ## annotations for the ServiceAccount
-    annotations: {}    
-    #annot#ations: 
-        #eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/YourIAMRoleName    
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::<AWS_ACCOUNT>:role/airflow-eks-role
 
   ###################################
   ## CONFIG | Kubernetes Extra Manifests
@@ -377,6 +424,9 @@ airflow:
   ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/extra-manifests.md
   extraManifests: []
 
+  dbMigrations:
+    enabled: false
+
   ###################################
   ## DATABASE | PgBouncer
   ###################################
@@ -430,17 +480,57 @@ airflow:
     ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/external-database.md
     type: postgres
 
-  ###################################
-  ## DATABASE | Embedded Redis
-  ###################################
-  redis:
-    ## if the `stable/redis` chart is used
+###################################
+## DATABASE | Embedded Redis
+###################################
+redis:
+  ## if the `stable/redis` chart is used
+  ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/embedded-redis.md
+  ## [WARNING] consider using an external database with `externalDatabase.*`
+  enabled: true
+
+  ## configs for redis cluster mode
+  cluster:
     enabled: false
+    slaveCount: 1
 
-  ###################################
-  ## DATABASE | External Redis
-  ###################################
-  externalRedis:
-    ## the host of the external redis
-    ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/external-redis.md
-    host: localhost
+  ## configs for the redis master StatefulSet
+  master:
+    ## resource requests/limits for the redis master Pods
+    ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
+    resources: {}
+
+    ## configs for the PVC of the redis master Pods
+    persistence:
+      enabled: true
+      storageClass: efs-sc
+      accessMode: ReadWriteMany
+      size: 50Gi
+      mountPath: /bitnami/redis
+      subPath: airflow/redis
+      existingClaim: "data-airflow-redis-0"
+
+
+  ## configs for the redis slave StatefulSet
+  slave:
+    ## resource requests/limits for the slave Pods
+    ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
+    resources: {}
+
+    ## configs for the PVC of the redis slave Pods
+    persistence:
+      enabled: true
+      storageClass: efs-sc
+      accessMode: ReadWriteMany
+      size: 100Gi
+      mountPath: /bitnami/redis
+      subPath: airflow/redis
+      existingClaim: "data-airflow-redis-1"
+
+###################################
+## DATABASE | External Redis
+###################################
+externalRedis:
+  ## the host of the external redis
+  ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/external-redis.md
+  host: localhost