From 45e003618acd690c05487963340104b6fe6273b0 Mon Sep 17 00:00:00 2001 From: Emil Hummel Date: Thu, 17 Feb 2022 15:31:31 +0100 Subject: [PATCH] kiam to oidc iam/helm kube prometheus stack (#426) * Modified iam policy document for helm-kube-prometheus-stack to use oidc * trim identifier for iam policy * Use preexisting data for oidc value instead of var * Added aws_subnet_exporter module; Updated oidc for kube-prometheus-stack * Remove roles from namespace IAM annotation, no longer needed after switch to oidc iam --- .../helm-kube-prometheus-stack/main.tf | 1 + .../values/grafana.yaml | 8 +- .../helm-kube-prometheus-stack/vars.tf | 4 + .../k8s-subnet-exporter/dependencies.tf | 33 +++++ _sub/compute/k8s-subnet-exporter/main.tf | 122 ++++++++++++++++++ _sub/compute/k8s-subnet-exporter/vars.tf | 28 ++++ _sub/compute/k8s-subnet-exporter/version.tf | 3 + compute/k8s-services/dependencies.tf | 32 ++++- compute/k8s-services/main.tf | 19 ++- compute/k8s-services/vars.tf | 6 + 10 files changed, 246 insertions(+), 10 deletions(-) create mode 100644 _sub/compute/k8s-subnet-exporter/dependencies.tf create mode 100644 _sub/compute/k8s-subnet-exporter/main.tf create mode 100644 _sub/compute/k8s-subnet-exporter/vars.tf create mode 100644 _sub/compute/k8s-subnet-exporter/version.tf diff --git a/_sub/compute/helm-kube-prometheus-stack/main.tf b/_sub/compute/helm-kube-prometheus-stack/main.tf index d926de348..983bc1cda 100644 --- a/_sub/compute/helm-kube-prometheus-stack/main.tf +++ b/_sub/compute/helm-kube-prometheus-stack/main.tf @@ -23,6 +23,7 @@ resource "helm_release" "kube_prometheus_stack" { grafana_host = var.grafana_host grafana_root_url = "https://%(domain)s${var.grafana_ingress_path}" grafana_cloudwatch_role = var.grafana_iam_role_arn + grafana_serviceaccount_name = var.grafana_serviceaccount_name }), length(var.slack_webhook) > 0 ? templatefile("${path.module}/values/grafana-notifiers.yaml", { diff --git a/_sub/compute/helm-kube-prometheus-stack/values/grafana.yaml b/_sub/compute/helm-kube-prometheus-stack/values/grafana.yaml index a7df0f290..44b770d69 100644 --- a/_sub/compute/helm-kube-prometheus-stack/values/grafana.yaml +++ b/_sub/compute/helm-kube-prometheus-stack/values/grafana.yaml @@ -20,7 +20,11 @@ grafana: path: ${grafana_ingress_path} plugins: - grafana-polystat-panel - podAnnotations: - iam.amazonaws.com/role: ${grafana_cloudwatch_role} rbac: pspEnabled: true + serviceAccount: + name: ${grafana_serviceaccount_name} + create: true + annotations: + eks.amazonaws.com/role-arn: ${grafana_cloudwatch_role} + eks.amazonaws.com/sts-regional-endpoints: "true" \ No newline at end of file diff --git a/_sub/compute/helm-kube-prometheus-stack/vars.tf b/_sub/compute/helm-kube-prometheus-stack/vars.tf index 51f674e5c..849f65d64 100644 --- a/_sub/compute/helm-kube-prometheus-stack/vars.tf +++ b/_sub/compute/helm-kube-prometheus-stack/vars.tf @@ -51,6 +51,10 @@ variable "grafana_iam_role_arn" { description = "Grafana IAM role ARN to add as pod annotation" } +variable "grafana_serviceaccount_name" { + type = string + description = "Grafana serviceaccount to be used for pod" +} variable "slack_webhook" { type = string diff --git a/_sub/compute/k8s-subnet-exporter/dependencies.tf b/_sub/compute/k8s-subnet-exporter/dependencies.tf new file mode 100644 index 000000000..a45ebb6a9 --- /dev/null +++ b/_sub/compute/k8s-subnet-exporter/dependencies.tf @@ -0,0 +1,33 @@ +data "aws_iam_policy_document" "subnet_exporter" { + statement { + effect = "Allow" + + actions = [ + "ec2:DescribeSubnets", + ] + + resources = ["*"] + } +} + +data "aws_iam_policy_document" "subnet_exporter_trust" { + statement { + effect = "Allow" + + principals { + type = "Federated" + + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:oidc-provider/${var.oidc_issuer}", + ] + } + + condition { + test = "StringEquals" + values = ["system:serviceaccount:${var.namespace_name}:${local.serviceaccount_name}"] + variable = "${var.oidc_issuer}:sub" + } + + actions = ["sts:AssumeRoleWithWebIdentity"] + } +} diff --git a/_sub/compute/k8s-subnet-exporter/main.tf b/_sub/compute/k8s-subnet-exporter/main.tf new file mode 100644 index 000000000..e6c00ed7d --- /dev/null +++ b/_sub/compute/k8s-subnet-exporter/main.tf @@ -0,0 +1,122 @@ +locals { + serviceaccount_name = "subnet-exporter" + deployment_name = "aws-subnet-exporter" + iam_role_name = "SubnetExporter" +} + +resource "aws_iam_role" "this" { + name = local.iam_role_name + path = "/" + description = "Role for subnet-exporter to describe ec2 subnets" + assume_role_policy = data.aws_iam_policy_document.subnet_exporter_trust.json + max_session_duration = 3600 +} + +resource "aws_iam_role_policy" "this" { + name = local.iam_role_name + role = aws_iam_role.this.id + policy = data.aws_iam_policy_document.subnet_exporter.json +} + +resource "kubernetes_service_account" "this" { + metadata { + name = local.serviceaccount_name + namespace = var.namespace_name + annotations = { + "eks.amazonaws.com/role-arn" = aws_iam_role.this.arn + "eks.amazonaws.com/sts-regional-endpoints" = "true" + } + } +} + +resource "kubernetes_deployment" "this" { + metadata { + name = local.deployment_name + namespace = var.namespace_name + + labels = { + app = local.deployment_name + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = local.deployment_name + } + } + + template { + metadata { + labels = { + app = local.deployment_name + } + } + + spec { + service_account_name = local.serviceaccount_name + automount_service_account_token = true + container { + name = local.deployment_name + image = "dfdsdk/aws-subnet-exporter:${var.image_tag}" + + env { + name = "REGION" + value = var.aws_region + } + + env { + name = "FILTER" + value = "*eks*" + } + + env { + name = "PERIOD" + value = "30s" + } + + env { + name = "PORT" + value = ":8080" + } + + resources { + requests = { + cpu = "20m" + + memory = "64Mi" + } + } + } + } + } + } +} + +resource "kubernetes_service" "this" { + metadata { + name = local.deployment_name + namespace = var.namespace_name + + labels = { + app = local.deployment_name + + scrape-service-metrics = "true" + } + } + + spec { + port { + name = "metrics" + protocol = "TCP" + port = 8080 + target_port = "8080" + } + + selector = { + app = local.deployment_name + } + } +} \ No newline at end of file diff --git a/_sub/compute/k8s-subnet-exporter/vars.tf b/_sub/compute/k8s-subnet-exporter/vars.tf new file mode 100644 index 000000000..b83fec7b8 --- /dev/null +++ b/_sub/compute/k8s-subnet-exporter/vars.tf @@ -0,0 +1,28 @@ +variable "aws_account_id" { + type = string + description = "Used for iam policy oidc trust" +} + +variable "aws_region" { + type = string + description = "Used to filter subnets by AWS region" +} + +variable "oidc_issuer" { + type = string + description = "Used for iam policy oidc trust" + validation { + condition = substr(var.oidc_issuer, 0, 8) != "https://" + error_message = "Oidc_issuer may not contain https:// in the start of the variable." + } +} + +variable "namespace_name" { + type = string + description = "K8s namespace for deployment/iam policy" +} + +variable "image_tag" { + type = string + description = "K8s subnet-exporter image tag" +} diff --git a/_sub/compute/k8s-subnet-exporter/version.tf b/_sub/compute/k8s-subnet-exporter/version.tf new file mode 100644 index 000000000..ab789a34e --- /dev/null +++ b/_sub/compute/k8s-subnet-exporter/version.tf @@ -0,0 +1,3 @@ +terraform { + required_version = "~> 1.0" +} diff --git a/compute/k8s-services/dependencies.tf b/compute/k8s-services/dependencies.tf index 49fff7493..ba6364955 100644 --- a/compute/k8s-services/dependencies.tf +++ b/compute/k8s-services/dependencies.tf @@ -49,6 +49,14 @@ locals { core_dns_zone_name = join(".", local.core_dns_zone_list) } +# -------------------------------------------------- +# Monitoring namespace name +# -------------------------------------------------- + +locals { + monitoring_namespace_name = "monitoring" +} + # -------------------------------------------------- # Get Route 53 zones and ids @@ -156,7 +164,6 @@ locals { locals { grafana_iam_role_name = "${var.eks_cluster_name}-monitoring-grafana-cloudwatch" grafana_iam_role_arn = "arn:aws:iam::${var.aws_workload_account_id}:role/${local.grafana_iam_role_name}" - monitoring_namespace_iam_roles = var.monitoring_kube_prometheus_stack_deploy ? join("|", compact([var.monitoring_namespace_iam_roles, local.grafana_iam_role_arn])) : var.monitoring_namespace_iam_roles } # -------------------------------------------------- @@ -182,20 +189,33 @@ data "aws_iam_policy_document" "cloudwatch_metrics" { } } +data "aws_caller_identity" "workload_account" { +} + +locals { + oidc_issuer = trim(data.aws_eks_cluster.eks.identity[0].oidc[0].issuer, "https://") +} + data "aws_iam_policy_document" "cloudwatch_metrics_trust" { statement { effect = "Allow" principals { - type = "AWS" + type = "Federated" identifiers = [ - module.kiam_deploy.server_role_arn, + "arn:aws:iam::${data.aws_caller_identity.workload_account.account_id}:oidc-provider/${local.oidc_issuer}", ] } - actions = ["sts:AssumeRole"] - } + condition { + test = "StringEquals" + values = ["system:serviceaccount:${local.monitoring_namespace_name}:${var.monitoring_kube_prometheus_stack_grafana_serviceaccount_name}"] + variable = "${local.oidc_issuer}:sub" + } + + actions = ["sts:AssumeRoleWithWebIdentity"] +} } # --------------------------------------------------------------------- @@ -295,4 +315,4 @@ locals { local.blackbox_exporter_monitoring_traefik, var.blackbox_exporter_monitoring_targets ) -} +} \ No newline at end of file diff --git a/compute/k8s-services/main.tf b/compute/k8s-services/main.tf index 651942fd4..6a70a10c5 100644 --- a/compute/k8s-services/main.tf +++ b/compute/k8s-services/main.tf @@ -367,8 +367,7 @@ module "cloudwatch_alarm_alb_targets_health" { module "monitoring_namespace" { source = "../../_sub/compute/k8s-namespace" count = var.monitoring_namespace_deploy ? 1 : 0 - name = "monitoring" - iam_roles = local.monitoring_namespace_iam_roles + name = local.monitoring_namespace_name } @@ -404,6 +403,7 @@ module "monitoring_kube_prometheus_stack" { grafana_host = "grafana.${var.eks_cluster_name}.${var.workload_dns_zone_name}" grafana_notifier_name = "${var.eks_cluster_name}-alerting" grafana_iam_role_arn = local.grafana_iam_role_arn # Coming from locals to avoid circular dependency between KIAM and Prometheus + grafana_serviceaccount_name = var.monitoring_kube_prometheus_stack_grafana_serviceaccount_name slack_webhook = var.monitoring_kube_prometheus_stack_slack_webhook prometheus_storageclass = var.monitoring_kube_prometheus_stack_prometheus_storageclass prometheus_storage_size = var.monitoring_kube_prometheus_stack_prometheus_storage_size @@ -614,3 +614,18 @@ module "velero_flux_manifests" { github = github.fluxcd } } + + +# -------------------------------------------------- +# aws-subnet-exporter +# -------------------------------------------------- + +module "aws_subnet_exporter" { + source = "../../_sub/compute/k8s-subnet-exporter" + count = var.monitoring_kube_prometheus_stack_deploy ? 1 : 0 + namespace_name = module.monitoring_namespace[0].name + aws_account_id = var.aws_workload_account_id + aws_region = var.aws_region + image_tag = "0.2" + oidc_issuer = local.oidc_issuer +} \ No newline at end of file diff --git a/compute/k8s-services/vars.tf b/compute/k8s-services/vars.tf index 16a2389d0..63669538b 100644 --- a/compute/k8s-services/vars.tf +++ b/compute/k8s-services/vars.tf @@ -236,6 +236,12 @@ variable "monitoring_kube_prometheus_stack_grafana_notifier_name" { default = "notifier1" } +variable "monitoring_kube_prometheus_stack_grafana_serviceaccount_name" { + type = string + description = "Grafana serviceaccount to be used for pod" + default = "grafana-cloudwatch" +} + variable "monitoring_kube_prometheus_stack_slack_webhook" { type = string description = "Kube-prometheus-stack alert slack webhook"