testing node affinities and tolerations

ExpediaGroup · Sep 23, 2024 · ee67471 · ee67471
1 parent e1d451e
commit ee67471
Show file tree

Hide file tree

Showing 3 changed files with 106 additions and 0 deletions.
diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -42,6 +42,37 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
       spec {
         service_account_name            = kubernetes_service_account_v1.hms_readonly[0].metadata.0.name
         automount_service_account_token = true
+
+        dynamic "tolerations" {
+          for_each = var.hms_ro_tolerations
+          content {
+            effect             = lookup(tolerations.value, "effect", null)
+            key                = lookup(tolerations.value, "key", null)
+            operator           = lookup(tolerations.value, "operator", null)
+            value              = lookup(tolerations.value, "value", null)
+          }
+        }
+
+        dynamic "affinity" {
+          for_each = var.hms_ro_node_affinity
+          content {
+            node_affinity {
+              required_during_scheduling_ignored_during_execution {
+                dynamic "node_selector_terms" {
+                  for_each = lookup(affinity.value, "node_selector_terms", [])
+                  content {
+                    match_expressions {
+                      key      = lookup(node_selector_terms.value, "key", null)
+                      operator = lookup(node_selector_terms.value, "operator", null)
+                      values   = lookup(node_selector_terms.value, "values", [])
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+
         dynamic "security_context"  {
           for_each = var.enable_tcp_keepalive ? ["enabled"] : []
           content {
@@ -59,6 +90,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
             }
           }
         }
+
         dynamic "init_container" {
           for_each = var.external_database_host == "" ? ["enabled"] : []
 

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -42,6 +42,37 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
       spec {
         service_account_name            = kubernetes_service_account_v1.hms_readwrite[0].metadata.0.name
         automount_service_account_token = true
+
+        dynamic "tolerations" {
+          for_each = var.hms_rw_tolerations
+          content {
+            effect             = lookup(tolerations.value, "effect", null)
+            key                = lookup(tolerations.value, "key", null)
+            operator           = lookup(tolerations.value, "operator", null)
+            value              = lookup(tolerations.value, "value", null)
+          }
+        }
+
+        dynamic "affinity" {
+          for_each = var.hms_rw_node_affinity
+          content {
+            node_affinity {
+              required_during_scheduling_ignored_during_execution {
+                dynamic "node_selector_terms" {
+                  for_each = lookup(affinity.value, "node_selector_terms", [])
+                  content {
+                    match_expressions {
+                      key      = lookup(node_selector_terms.value, "key", null)
+                      operator = lookup(node_selector_terms.value, "operator", null)
+                      values   = lookup(node_selector_terms.value, "values", [])
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+
         dynamic "security_context"  {
           for_each = var.enable_tcp_keepalive ? ["enabled"] : []
           content {
@@ -59,6 +90,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
             }
           }
         }
+
         dynamic "init_container" {
           for_each = var.external_database_host == "" ? ["enabled"] : []
           content {

diff --git a/variables.tf b/variables.tf
@@ -394,6 +394,48 @@ variable "hms_ro_k8s_max_replica_count" {
   default     = 10
 }
 
+variable "hms_rw_node_affinity" {
+  type = list(object({
+    node_selector_terms = list(object({
+      key      = string
+      operator = string
+      values   = list(string)
+    }))
+  }))
+  default = []  # Default to an empty list
+}
+
+variable "hms_rw_tolerations" {
+  type = list(object({
+    effect             = string
+    key                = string
+    operator           = string
+    value              = string
+  }))
+  default = []
+}
+
+variable "hms_ro_node_affinity" {
+  type = list(object({
+    node_selector_terms = list(object({
+      key      = string
+      operator = string
+      values   = list(string)
+    }))
+  }))
+  default = []
+}
+
+variable "hms_ro_tolerations" {
+  type = list(object({
+    effect             = string
+    key                = string
+    operator           = string
+    value              = string
+  }))
+  default = []
+}
+
 variable "enable_autoscaling" {
   description = "Enable read only Hive Metastore k8s horizontal pod autoscaling"
   type        = bool