fix: fixing irsa working mode

ExpediaGroup · May 13, 2024 · 5d6a1b1 · 5d6a1b1
1 parent 35fa9ad
commit 5d6a1b1
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.1.5] - 2024-05-13
+### Fixed
+- Fixed k8s IRSA.
+- Changed k8s service account creation to compatible with newer version kubernetes provider.(eks 1.24 and later, create service account no longer create account token automatically)
+
 ## [7.1.4] - 2024-05-06
 ### Fixed
 - Change provider version for `kubernetes`.

diff --git a/k8s-cronjobs.tf b/k8s-cronjobs.tf
@@ -29,7 +29,7 @@ resource "kubernetes_cron_job" "apiary_inventory" {
               name = "${local.instance_alias}-s3-inventory"
             }
             annotations = {
-              "iam.amazonaws.com/role" = aws_iam_role.apiary_s3_inventory.name
+              "iam.amazonaws.com/role" = var.oidc_provider == "" ? aws_iam_role.apiary_s3_inventory.name : null
             }
           }
 

diff --git a/k8s-housekeeper.tf b/k8s-housekeeper.tf
@@ -32,7 +32,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_housekeeper" {
           "ad.datadoghq.com/${local.hms_alias}-housekeeper.check_names" = var.datadog_metrics_enabled ? "[\"prometheus\"]" : null
           "ad.datadoghq.com/${local.hms_alias}-housekeeper.init_configs" = var.datadog_metrics_enabled ? "[{}]" : null
           "ad.datadoghq.com/${local.hms_alias}-housekeeper.instances" = var.datadog_metrics_enabled ? "[{ \"prometheus_url\": \"http://%%host%%:${var.datadog_metrics_port}/actuator/prometheus\", \"namespace\": \"hms_readwrite\", \"metrics\": [ \"${join("\",\"", var.datadog_metrics_hms_readwrite_readonly)}\" ] , \"type_overrides\": { \"${join("\": \"gauge\",\"", var.datadog_metrics_hms_readwrite_readonly)}\": \"gauge\"} }]" : null
-          "iam.amazonaws.com/role" = aws_iam_role.apiary_hms_readwrite.name
+          "iam.amazonaws.com/role" = var.oidc_provider == "" ? aws_iam_role.apiary_hms_readwrite.name : null
           "prometheus.io/path"     = "/metrics"
           "prometheus.io/port"     = "8080"
           "prometheus.io/scrape"   = "true"

diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -32,7 +32,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
           "ad.datadoghq.com/${local.hms_alias}-readonly.check_names" = var.datadog_metrics_enabled ? "[\"prometheus\"]" : null
           "ad.datadoghq.com/${local.hms_alias}-readonly.init_configs" = var.datadog_metrics_enabled ? "[{}]" : null
           "ad.datadoghq.com/${local.hms_alias}-readonly.instances" = var.datadog_metrics_enabled ? "[{ \"prometheus_url\": \"http://%%host%%:${var.datadog_metrics_port}/actuator/prometheus\", \"namespace\": \"hms_readonly\", \"metrics\": [ \"${join("\",\"", var.datadog_metrics_hms_readwrite_readonly)}\" ] , \"type_overrides\": { \"${join("\": \"gauge\",\"", var.datadog_metrics_hms_readwrite_readonly)}\": \"gauge\"} }]"  : null
-          "iam.amazonaws.com/role" = aws_iam_role.apiary_hms_readonly.name
+          "iam.amazonaws.com/role" = var.oidc_provider == "" ? aws_iam_role.apiary_hms_readonly.name : null
           "prometheus.io/path"     = "/metrics"
           "prometheus.io/port"     = "8080"
           "prometheus.io/scrape"   = "true"

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -32,7 +32,7 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
           "ad.datadoghq.com/${local.hms_alias}-readwrite.check_names" = var.datadog_metrics_enabled ? "[\"prometheus\"]" : null
           "ad.datadoghq.com/${local.hms_alias}-readwrite.init_configs" = var.datadog_metrics_enabled ? "[{}]" : null
           "ad.datadoghq.com/${local.hms_alias}-readwrite.instances" = var.datadog_metrics_enabled ? "[{ \"prometheus_url\": \"http://%%host%%:${var.datadog_metrics_port}/actuator/prometheus\", \"namespace\": \"hms_readwrite\", \"metrics\": [ \"${join("\",\"", var.datadog_metrics_hms_readwrite_readonly)}\" ] , \"type_overrides\": { \"${join("\": \"gauge\",\"", var.datadog_metrics_hms_readwrite_readonly)}\": \"gauge\"} }]" : null
-          "iam.amazonaws.com/role" = aws_iam_role.apiary_hms_readwrite.name
+          "iam.amazonaws.com/role" = var.oidc_provider == "" ? aws_iam_role.apiary_hms_readwrite.name : null
           "prometheus.io/path"     = "/metrics"
           "prometheus.io/port"     = "8080"
           "prometheus.io/scrape"   = "true"

diff --git a/k8s-service-accounts.tf b/k8s-service-accounts.tf
@@ -1,4 +1,4 @@
-resource "kubernetes_service_account" "hms_readwrite" {
+resource "kubernetes_service_account_v1" "hms_readwrite" {
   count = var.hms_instance_type == "k8s" ? 1 : 0
   metadata {
     name      = "${local.hms_alias}-readwrite"
@@ -7,10 +7,25 @@ resource "kubernetes_service_account" "hms_readwrite" {
       "eks.amazonaws.com/role-arn" = var.oidc_provider == "" ? "" : aws_iam_role.apiary_hms_readwrite.arn
     }
   }
-  automount_service_account_token = true
 }
 
-resource "kubernetes_service_account" "hms_readonly" {
+resource "kubernetes_secret_v1" "hms_readwrite" {
+  metadata {
+    name        = "${local.hms_alias}-readwrite"
+    namespace   = var.metastore_namespace
+    annotations = {
+      "kubernetes.io/service-account.name"      ="${local.hms_alias}-readwrite"
+      "kubernetes.io/service-account.namespace" = var.metastore_namespace
+    }
+  }
+  type = "kubernetes.io/service-account-token"
+
+  depends_on = [
+    kubernetes_service_account_v1.hms_readwrite
+  ]
+}
+
+resource "kubernetes_service_account_v1" "hms_readonly" {
   count = var.hms_instance_type == "k8s" ? 1 : 0
   metadata {
     name      = "${local.hms_alias}-readonly"
@@ -19,7 +34,22 @@ resource "kubernetes_service_account" "hms_readonly" {
       "eks.amazonaws.com/role-arn" = var.oidc_provider == "" ? "" : aws_iam_role.apiary_hms_readonly.arn
     }
   }
-  automount_service_account_token = true
+}
+
+resource "kubernetes_secret_v1" "hms_readonly" {
+  metadata {
+    name        = "${local.hms_alias}-readonly"
+    namespace   = var.metastore_namespace
+    annotations = {
+      "kubernetes.io/service-account.name"      ="${local.hms_alias}-readonly"
+      "kubernetes.io/service-account.namespace" = var.metastore_namespace
+    }
+  }
+  type = "kubernetes.io/service-account-token"
+
+  depends_on = [
+    kubernetes_service_account_v1.hms_readonly
+  ]
 }
 
 resource "kubernetes_service_account" "s3_inventory" {
@@ -31,5 +61,20 @@ resource "kubernetes_service_account" "s3_inventory" {
       "eks.amazonaws.com/role-arn" = var.oidc_provider == "" ? "" : aws_iam_role.apiary_s3_inventory.arn
     }
   }
-  automount_service_account_token = true
+}
+
+resource "kubernetes_secret_v1" "s3_inventory" {
+  metadata {
+    name        = "${local.hms_alias}-s3-inventory"
+    namespace   = var.metastore_namespace
+    annotations = {
+      "kubernetes.io/service-account.name"      ="${local.hms_alias}-s3-inventory"
+      "kubernetes.io/service-account.namespace" = var.metastore_namespace
+    }
+  }
+  type = "kubernetes.io/service-account-token"
+
+  depends_on = [
+    kubernetes_service_account_v1.s3_inventory
+  ]
 }