Replace KIAM with IRSA for Velero (#427)

* Replace KIAM with IRSA for Velero. Now supports cross account setup

Signed-off-by: Audun Nes <[email protected]>

* Remove unused kiam_server_role_arn variable

Signed-off-by: Audun Nes <[email protected]>

* Use aws_iam_oidc_provider from sub module

Signed-off-by: Audun Nes <[email protected]>

* Cleanup QA more controlled due to dependency between EKS account and Velero S3 bucket account

Signed-off-by: Audun Nes <[email protected]>

* Fix typo in function name

Signed-off-by: Audun Nes <[email protected]>

_sub/storage/velero-flux/dependencies.tf

@@ -66,8 +66,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: velero
-  annotations:
-    iam.amazonaws.com/permitted: "${var.role_arn}"
 
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
@@ -78,12 +76,16 @@ metadata:
 spec:
   values:
     snapshotsEnabled: ${var.snapshots_enabled}
-    podAnnotations:
-      iam.amazonaws.com/role: "${var.role_arn}"
     configuration:
       logLevel: ${var.log_level}
       backupStorageLocation:
         bucket: ${var.bucket_name}
+    serviceAccount:
+      server:
+        create: true
+        annotations:
+          eks.amazonaws.com/role-arn: "${var.role_arn}"
+          eks.amazonaws.com/sts-regional-endpoints: "true"
     schedules:
       ${var.cluster_name}-cluster-backup:
         schedule: "${var.cron_schedule}"

azure-pipelines.yaml

@@ -69,22 +69,32 @@ stages:
           - bash: ./src/qa-test-eks.sh apply-shared _global/s3-bucket-velero
             displayName: "Provision Velero S3 bucket"
 
+      - job: destroy_velero_bucket
+        displayName: Destroy Velero S3 Bucket
+        container: prime
+        dependsOn:
+          - init_shared_velero
+        condition: ne(variables['NO_DESTROY'], 'true')
+        steps:
+          - bash: ./src/qa-test-eks.sh destroy-velero-bucket eu-west-1 _global/s3-bucket-velero
+            displayName: "Terraform Destroy Velero S3 Bucket (post)"
+
       - job: destroy_eks_1_21
         displayName: Destroy EKS 1.21
         container: prime
         dependsOn:
-          - init_shared_velero
+          - destroy_velero_bucket
         condition: ne(variables['NO_DESTROY'], 'true')
         steps:
           - bash: ./src/qa-test-eks.sh destroy-cluster eu-west-1 qa21
             displayName: "Terraform Destroy (post)"
 
       - job: destroy_shared
-        displayName: Destroy EKS shared resources
+        displayName: Destroy Public S3 bucket
         container: prime
         dependsOn:
           - destroy_eks_1_21
         condition: ne(variables['NO_DESTROY'], 'true')
         steps:
-          - bash: ./src/qa-test-eks.sh destroy-shared eu-west-1 _global
-            displayName: "Destroy shared all resources"
+          - bash: ./src/qa-test-eks.sh destroy-public-bucket eu-west-1 _global/eks-public-s3-bucket
+            displayName: "Terraform Destroy Public S3 bucket (post)"

src/qa-test-eks.sh

@@ -161,7 +161,22 @@ if [ "$ACTION" = "destroy-cluster" ]; then
 fi
 
 
-if [ "$ACTION" = "destroy-shared" ]; then
+if [ "$ACTION" = "destroy-public-bucket" ]; then
+    RETURN=0
+    REGION=$2
+    SUBPATH=$3
+    WORKDIR="${BASEPATH}/${SUBPATH}"
+
+    # Cleanup
+    terragrunt run-all destroy --terragrunt-working-dir "$WORKDIR" --terragrunt-source-update --terragrunt-non-interactive -input=false -auto-approve || RETURN=1
+
+    # Return false, if any *eseential* commands failed
+    if [ $RETURN -ne 0 ]; then
+        false
+    fi
+fi
+
+if [ "$ACTION" = "destroy-velero-bucket" ]; then
     RETURN=0
     REGION=$2
     SUBPATH=$3

storage/s3-velero-backup/dependencies.tf

@@ -0,0 +1,44 @@
+# --------------------------------------------------
+# OIDC Provider URL
+# --------------------------------------------------
+
+# When var.eks_cluster_name is supplied, we will use
+# the EKS data provider to fetch the oidc_provider_url
+# Since data providers don't work across accounts,
+# using var.eks_cluster_name only make sense if the
+# EKS cluster and the S3 bucket for Velero are in
+# the same AWS account.
+#
+# Hence for our sandbox environments and QA:
+# 1. ONLY provide var.eks_cluster_name AND var.bucket_name
+#
+# For our production environments:
+# 1. ONLY provide var.oidc_provider_url AND var.bucket_name
+
+data "aws_eks_cluster" "eks" {
+  count = var.eks_cluster_name != null ? 1 : 0
+  name  = var.eks_cluster_name
+}
+
+locals {
+  oidc_provider_url = var.oidc_provider_url == null ? (
+    data.aws_eks_cluster.eks[0].identity[0].oidc[0].issuer) : (
+    var.oidc_provider_url
+  )
+}
+
+# --------------------------------------------------
+# Caller identity and additional OIDC properties
+# --------------------------------------------------
+
+data "aws_caller_identity" "current" {}
+
+data "tls_certificate" "oidc_provider" {
+  url = local.oidc_provider_url
+}
+
+locals {
+  account_id              = data.aws_caller_identity.current.account_id
+  oidc_provider_server_id = trim(local.oidc_provider_url, "https://")
+  oidc_provider_arn       = "arn:aws:iam::${local.account_id}:oidc-provider/${local.oidc_provider_server_id}"
+}

storage/s3-velero-backup/main.tf

@@ -4,13 +4,26 @@ terraform {
 }
 
 provider "aws" {
-  region  = var.aws_region
+  region = var.aws_region
 
   assume_role {
     role_arn = var.aws_assume_role_arn
   }
 }
 
+# --------------------------------------------------
+# AWS IAM Open ID Connect Provider
+# --------------------------------------------------
+
+# Only create a IAM Provider for OIDC if var.oidc_provider_url is supplied.
+# If var.oidc_provider_url is not supplied, it means that one already exist,
+# and information is fetched through the EKS data source.
+module "aws_iam_oidc_provider" {
+  count                           = var.oidc_provider_url != null ? 1 : 0
+  source                          = "../../_sub/security/iam-oidc-provider"
+  eks_openid_connect_provider_url = local.oidc_provider_url
+}
+
 resource "aws_s3_bucket" "velero_storage" {
   bucket        = var.bucket_name
   acl           = "private"
@@ -79,10 +92,15 @@ data "aws_iam_policy_document" "assume_role" {
   statement {
     sid     = ""
     effect  = "Allow"
-    actions = ["sts:AssumeRole"]
+    actions = ["sts:AssumeRoleWithWebIdentity"]
     principals {
-      type        = "AWS"
-      identifiers = var.kiam_server_role_arn
+      type        = "Federated"
+      identifiers = ["${local.oidc_provider_arn}"]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "${local.oidc_provider_server_id}:sub"
+      values   = ["system:serviceaccount:${var.namespace}:${var.service_account}"]
     }
   }
 }

storage/s3-velero-backup/vars.tf

@@ -11,21 +11,15 @@ variable "bucket_name" {
   description = "Velero storage bucket name"
 }
 
-variable "kiam_server_role_arn" {
-  type        = list(string)
-  default     = [""]
-  description = "Role to allow for trust relationship to KIAM "
-}
-
 variable "versioning" {
   type        = bool
   default     = true
   description = "Enable S3 bucket versioning"
 }
 
 variable "velero_iam_role_name" {
-  type = string
-  default = "VeleroBackup"
+  type        = string
+  default     = "VeleroBackup"
   description = "Velero role for S3 actions"
 }
 
@@ -34,3 +28,27 @@ variable "force_bucket_destroy" {
   default     = true
   description = "Destroy bucket without error"
 }
+
+variable "namespace" {
+  type        = string
+  default     = "velero"
+  description = "The namespace that Velero will be installed to"
+}
+
+variable "service_account" {
+  type        = string
+  default     = "velero-server"
+  description = "The service account to be used by Velero"
+}
+
+variable "eks_cluster_name" {
+  type        = string
+  default     = null
+  description = "The AWS EKS cluster name. Only supply this if Velero S3 bucket and EKS cluster exist in the SAME account"
+}
+
+variable "oidc_provider_url" {
+  type        = string
+  default     = null
+  description = "The OIDC provider URL. Only supply this if Velero S3 bucket and EKS cluster exist in the DIFFERENT accounts"
+}

test/integration/_global/s3-bucket-velero/terragrunt.hcl

@@ -13,5 +13,5 @@ dependencies {
 
 inputs = {
   bucket_name = "dfds-velero-qa"
-  kiam_server_role_arn = ["arn:aws:iam::266901158286:role/eks-qa21-kiam-server"]
+  eks_cluster_name = "qa21"
 }