kiam to oidc iam/helm kube prometheus stack (#426)

* Modified iam policy document for helm-kube-prometheus-stack to use oidc

* trim identifier for iam policy

* Use preexisting data for oidc value instead of var

* Added aws_subnet_exporter module; Updated oidc for kube-prometheus-stack

* Remove roles from namespace IAM annotation, no longer needed after switch to oidc iam

_sub/compute/helm-kube-prometheus-stack/main.tf

@@ -23,6 +23,7 @@ resource "helm_release" "kube_prometheus_stack" {
       grafana_host            = var.grafana_host
       grafana_root_url        = "https://%(domain)s${var.grafana_ingress_path}"
       grafana_cloudwatch_role = var.grafana_iam_role_arn
+      grafana_serviceaccount_name = var.grafana_serviceaccount_name
     }),
 
     length(var.slack_webhook) > 0 ? templatefile("${path.module}/values/grafana-notifiers.yaml", {

_sub/compute/helm-kube-prometheus-stack/values/grafana.yaml

@@ -20,7 +20,11 @@ grafana:
     path: ${grafana_ingress_path}
   plugins:
     - grafana-polystat-panel
-  podAnnotations:
-    iam.amazonaws.com/role: ${grafana_cloudwatch_role}
   rbac:
     pspEnabled: true
+  serviceAccount:
+    name: ${grafana_serviceaccount_name}
+    create: true
+    annotations:
+      eks.amazonaws.com/role-arn: ${grafana_cloudwatch_role}
+      eks.amazonaws.com/sts-regional-endpoints: "true"

_sub/compute/helm-kube-prometheus-stack/vars.tf

@@ -51,6 +51,10 @@ variable "grafana_iam_role_arn" {
   description = "Grafana IAM role ARN to add as pod annotation"
 }
 
+variable "grafana_serviceaccount_name" {
+  type        = string
+  description = "Grafana serviceaccount to be used for pod"
+}
 
 variable "slack_webhook" {
   type        = string

_sub/compute/k8s-subnet-exporter/dependencies.tf

@@ -0,0 +1,33 @@
+data "aws_iam_policy_document" "subnet_exporter" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:DescribeSubnets",
+    ]
+
+    resources = ["*"]
+  }
+}
+
+data "aws_iam_policy_document" "subnet_exporter_trust" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type = "Federated"
+
+      identifiers = [
+        "arn:aws:iam::${var.aws_account_id}:oidc-provider/${var.oidc_issuer}",
+      ]
+    }
+
+    condition {
+      test = "StringEquals"
+      values = ["system:serviceaccount:${var.namespace_name}:${local.serviceaccount_name}"]
+      variable = "${var.oidc_issuer}:sub"
+    }
+
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+  }
+}

_sub/compute/k8s-subnet-exporter/main.tf

@@ -0,0 +1,122 @@
+locals {
+  serviceaccount_name = "subnet-exporter"
+  deployment_name = "aws-subnet-exporter"
+  iam_role_name = "SubnetExporter"
+}
+
+resource "aws_iam_role" "this" {
+  name = local.iam_role_name
+  path = "/"
+  description = "Role for subnet-exporter to describe ec2 subnets"
+  assume_role_policy = data.aws_iam_policy_document.subnet_exporter_trust.json
+  max_session_duration = 3600
+}
+
+resource "aws_iam_role_policy" "this" {
+  name = local.iam_role_name
+  role = aws_iam_role.this.id
+  policy = data.aws_iam_policy_document.subnet_exporter.json
+}
+
+resource "kubernetes_service_account" "this" {
+  metadata {
+    name = local.serviceaccount_name
+    namespace = var.namespace_name
+    annotations = {
+      "eks.amazonaws.com/role-arn" = aws_iam_role.this.arn
+      "eks.amazonaws.com/sts-regional-endpoints" = "true"
+    }
+  }
+}
+
+resource "kubernetes_deployment" "this" {
+  metadata {
+    name      = local.deployment_name
+    namespace = var.namespace_name
+
+    labels = {
+      app = local.deployment_name
+    }
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = {
+        app = local.deployment_name
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = local.deployment_name
+        }
+      }
+
+      spec {
+        service_account_name = local.serviceaccount_name
+        automount_service_account_token = true
+        container {
+          name  = local.deployment_name
+          image = "dfdsdk/aws-subnet-exporter:${var.image_tag}"
+
+          env {
+            name  = "REGION"
+            value = var.aws_region
+          }
+
+          env {
+            name  = "FILTER"
+            value = "*eks*"
+          }
+
+          env {
+            name  = "PERIOD"
+            value = "30s"
+          }
+
+          env {
+            name  = "PORT"
+            value = ":8080"
+          }
+
+          resources {
+            requests = {
+              cpu = "20m"
+
+              memory = "64Mi"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "this" {
+  metadata {
+    name      = local.deployment_name
+    namespace = var.namespace_name
+
+    labels = {
+      app = local.deployment_name
+
+      scrape-service-metrics = "true"
+    }
+  }
+
+  spec {
+    port {
+      name        = "metrics"
+      protocol    = "TCP"
+      port        = 8080
+      target_port = "8080"
+    }
+
+    selector = {
+      app = local.deployment_name
+    }
+  }
+}

_sub/compute/k8s-subnet-exporter/vars.tf

@@ -0,0 +1,28 @@
+variable "aws_account_id" {
+  type        = string
+  description = "Used for iam policy oidc trust"
+}
+
+variable "aws_region" {
+  type = string
+  description = "Used to filter subnets by AWS region"
+}
+
+variable "oidc_issuer" {
+  type        = string
+  description = "Used for iam policy oidc trust"
+  validation {
+    condition     = substr(var.oidc_issuer, 0, 8) != "https://"
+    error_message = "Oidc_issuer may not contain https:// in the start of the variable."
+  }
+}
+
+variable "namespace_name" {
+  type = string
+  description = "K8s namespace for deployment/iam policy"
+}
+
+variable "image_tag" {
+  type = string
+  description = "K8s subnet-exporter image tag"
+}

_sub/compute/k8s-subnet-exporter/version.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = "~> 1.0"
+}

compute/k8s-services/dependencies.tf

@@ -49,6 +49,14 @@ locals {
   core_dns_zone_name = join(".", local.core_dns_zone_list)
 }
 
+# --------------------------------------------------
+# Monitoring namespace name
+# --------------------------------------------------
+
+locals {
+  monitoring_namespace_name = "monitoring"
+}
+
 
 # --------------------------------------------------
 # Get Route 53 zones and ids
@@ -156,7 +164,6 @@ locals {
 locals {
   grafana_iam_role_name          = "${var.eks_cluster_name}-monitoring-grafana-cloudwatch"
   grafana_iam_role_arn           = "arn:aws:iam::${var.aws_workload_account_id}:role/${local.grafana_iam_role_name}"
-  monitoring_namespace_iam_roles = var.monitoring_kube_prometheus_stack_deploy ? join("|", compact([var.monitoring_namespace_iam_roles, local.grafana_iam_role_arn])) : var.monitoring_namespace_iam_roles
 }
 
 # --------------------------------------------------
@@ -182,20 +189,33 @@ data "aws_iam_policy_document" "cloudwatch_metrics" {
   }
 }
 
+data "aws_caller_identity" "workload_account" {
+}
+
+locals {
+  oidc_issuer = trim(data.aws_eks_cluster.eks.identity[0].oidc[0].issuer, "https://")
+}
+
 data "aws_iam_policy_document" "cloudwatch_metrics_trust" {
   statement {
     effect = "Allow"
 
     principals {
-      type = "AWS"
+      type = "Federated"
 
       identifiers = [
-        module.kiam_deploy.server_role_arn,
+        "arn:aws:iam::${data.aws_caller_identity.workload_account.account_id}:oidc-provider/${local.oidc_issuer}",
       ]
     }
 
-    actions = ["sts:AssumeRole"]
-  }
+    condition {
+      test = "StringEquals"
+      values = ["system:serviceaccount:${local.monitoring_namespace_name}:${var.monitoring_kube_prometheus_stack_grafana_serviceaccount_name}"]
+      variable = "${local.oidc_issuer}:sub"
+    }
+
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+}
 }
 
 # ---------------------------------------------------------------------
@@ -295,4 +315,4 @@ locals {
     local.blackbox_exporter_monitoring_traefik,
     var.blackbox_exporter_monitoring_targets
   )
-}
+}

compute/k8s-services/main.tf

@@ -367,8 +367,7 @@ module "cloudwatch_alarm_alb_targets_health" {
 module "monitoring_namespace" {
   source    = "../../_sub/compute/k8s-namespace"
   count     = var.monitoring_namespace_deploy ? 1 : 0
-  name      = "monitoring"
-  iam_roles = local.monitoring_namespace_iam_roles
+  name      = local.monitoring_namespace_name
 }
 
 
@@ -404,6 +403,7 @@ module "monitoring_kube_prometheus_stack" {
   grafana_host              = "grafana.${var.eks_cluster_name}.${var.workload_dns_zone_name}"
   grafana_notifier_name     = "${var.eks_cluster_name}-alerting"
   grafana_iam_role_arn      = local.grafana_iam_role_arn # Coming from locals to avoid circular dependency between KIAM and Prometheus
+  grafana_serviceaccount_name = var.monitoring_kube_prometheus_stack_grafana_serviceaccount_name
   slack_webhook             = var.monitoring_kube_prometheus_stack_slack_webhook
   prometheus_storageclass   = var.monitoring_kube_prometheus_stack_prometheus_storageclass
   prometheus_storage_size   = var.monitoring_kube_prometheus_stack_prometheus_storage_size
@@ -614,3 +614,18 @@ module "velero_flux_manifests" {
     github = github.fluxcd
   }
 }
+
+
+# --------------------------------------------------
+# aws-subnet-exporter
+# --------------------------------------------------
+
+module "aws_subnet_exporter" {
+  source = "../../_sub/compute/k8s-subnet-exporter"
+  count  = var.monitoring_kube_prometheus_stack_deploy ? 1 : 0
+  namespace_name = module.monitoring_namespace[0].name
+  aws_account_id = var.aws_workload_account_id
+  aws_region = var.aws_region
+  image_tag = "0.2"
+  oidc_issuer = local.oidc_issuer
+}

compute/k8s-services/vars.tf

@@ -236,6 +236,12 @@ variable "monitoring_kube_prometheus_stack_grafana_notifier_name" {
   default     = "notifier1"
 }
 
+variable "monitoring_kube_prometheus_stack_grafana_serviceaccount_name" {
+  type = string
+  description = "Grafana serviceaccount to be used for pod"
+  default = "grafana-cloudwatch"
+}
+
 variable "monitoring_kube_prometheus_stack_slack_webhook" {
   type        = string
   description = "Kube-prometheus-stack alert slack webhook"