From c465295c037a6d7d31e5d0b1292a4c087b8b06c4 Mon Sep 17 00:00:00 2001 From: "Audun V. Nes" Date: Fri, 18 Feb 2022 09:55:09 +0100 Subject: [PATCH] Replace KIAM with IRSA for Velero (#427) * Replace KIAM with IRSA for Velero. Now supports cross account setup Signed-off-by: Audun Nes * Remove unused kiam_server_role_arn variable Signed-off-by: Audun Nes * Use aws_iam_oidc_provider from sub module Signed-off-by: Audun Nes * Cleanup QA more controlled due to dependency between EKS account and Velero S3 bucket account Signed-off-by: Audun Nes * Fix typo in function name Signed-off-by: Audun Nes --- _sub/storage/velero-flux/dependencies.tf | 10 +++-- azure-pipelines.yaml | 18 ++++++-- src/qa-test-eks.sh | 17 ++++++- storage/s3-velero-backup/dependencies.tf | 44 +++++++++++++++++++ storage/s3-velero-backup/main.tf | 26 +++++++++-- storage/s3-velero-backup/vars.tf | 34 ++++++++++---- .../_global/s3-bucket-velero/terragrunt.hcl | 2 +- 7 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 storage/s3-velero-backup/dependencies.tf diff --git a/_sub/storage/velero-flux/dependencies.tf b/_sub/storage/velero-flux/dependencies.tf index 0d3fe64c4..173b5e8db 100644 --- a/_sub/storage/velero-flux/dependencies.tf +++ b/_sub/storage/velero-flux/dependencies.tf @@ -66,8 +66,6 @@ apiVersion: v1 kind: Namespace metadata: name: velero - annotations: - iam.amazonaws.com/permitted: "${var.role_arn}" --- apiVersion: helm.toolkit.fluxcd.io/v2beta1 @@ -78,12 +76,16 @@ metadata: spec: values: snapshotsEnabled: ${var.snapshots_enabled} - podAnnotations: - iam.amazonaws.com/role: "${var.role_arn}" configuration: logLevel: ${var.log_level} backupStorageLocation: bucket: ${var.bucket_name} + serviceAccount: + server: + create: true + annotations: + eks.amazonaws.com/role-arn: "${var.role_arn}" + eks.amazonaws.com/sts-regional-endpoints: "true" schedules: ${var.cluster_name}-cluster-backup: schedule: "${var.cron_schedule}" diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 6d01062a2..db19b8b44 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -69,22 +69,32 @@ stages: - bash: ./src/qa-test-eks.sh apply-shared _global/s3-bucket-velero displayName: "Provision Velero S3 bucket" + - job: destroy_velero_bucket + displayName: Destroy Velero S3 Bucket + container: prime + dependsOn: + - init_shared_velero + condition: ne(variables['NO_DESTROY'], 'true') + steps: + - bash: ./src/qa-test-eks.sh destroy-velero-bucket eu-west-1 _global/s3-bucket-velero + displayName: "Terraform Destroy Velero S3 Bucket (post)" + - job: destroy_eks_1_21 displayName: Destroy EKS 1.21 container: prime dependsOn: - - init_shared_velero + - destroy_velero_bucket condition: ne(variables['NO_DESTROY'], 'true') steps: - bash: ./src/qa-test-eks.sh destroy-cluster eu-west-1 qa21 displayName: "Terraform Destroy (post)" - job: destroy_shared - displayName: Destroy EKS shared resources + displayName: Destroy Public S3 bucket container: prime dependsOn: - destroy_eks_1_21 condition: ne(variables['NO_DESTROY'], 'true') steps: - - bash: ./src/qa-test-eks.sh destroy-shared eu-west-1 _global - displayName: "Destroy shared all resources" + - bash: ./src/qa-test-eks.sh destroy-public-bucket eu-west-1 _global/eks-public-s3-bucket + displayName: "Terraform Destroy Public S3 bucket (post)" diff --git a/src/qa-test-eks.sh b/src/qa-test-eks.sh index dbf64b1ea..654bfdeac 100755 --- a/src/qa-test-eks.sh +++ b/src/qa-test-eks.sh @@ -161,7 +161,22 @@ if [ "$ACTION" = "destroy-cluster" ]; then fi -if [ "$ACTION" = "destroy-shared" ]; then +if [ "$ACTION" = "destroy-public-bucket" ]; then + RETURN=0 + REGION=$2 + SUBPATH=$3 + WORKDIR="${BASEPATH}/${SUBPATH}" + + # Cleanup + terragrunt run-all destroy --terragrunt-working-dir "$WORKDIR" --terragrunt-source-update --terragrunt-non-interactive -input=false -auto-approve || RETURN=1 + + # Return false, if any *eseential* commands failed + if [ $RETURN -ne 0 ]; then + false + fi +fi + +if [ "$ACTION" = "destroy-velero-bucket" ]; then RETURN=0 REGION=$2 SUBPATH=$3 diff --git a/storage/s3-velero-backup/dependencies.tf b/storage/s3-velero-backup/dependencies.tf new file mode 100644 index 000000000..8989bd280 --- /dev/null +++ b/storage/s3-velero-backup/dependencies.tf @@ -0,0 +1,44 @@ +# -------------------------------------------------- +# OIDC Provider URL +# -------------------------------------------------- + +# When var.eks_cluster_name is supplied, we will use +# the EKS data provider to fetch the oidc_provider_url +# Since data providers don't work across accounts, +# using var.eks_cluster_name only make sense if the +# EKS cluster and the S3 bucket for Velero are in +# the same AWS account. +# +# Hence for our sandbox environments and QA: +# 1. ONLY provide var.eks_cluster_name AND var.bucket_name +# +# For our production environments: +# 1. ONLY provide var.oidc_provider_url AND var.bucket_name + +data "aws_eks_cluster" "eks" { + count = var.eks_cluster_name != null ? 1 : 0 + name = var.eks_cluster_name +} + +locals { + oidc_provider_url = var.oidc_provider_url == null ? ( + data.aws_eks_cluster.eks[0].identity[0].oidc[0].issuer) : ( + var.oidc_provider_url + ) +} + +# -------------------------------------------------- +# Caller identity and additional OIDC properties +# -------------------------------------------------- + +data "aws_caller_identity" "current" {} + +data "tls_certificate" "oidc_provider" { + url = local.oidc_provider_url +} + +locals { + account_id = data.aws_caller_identity.current.account_id + oidc_provider_server_id = trim(local.oidc_provider_url, "https://") + oidc_provider_arn = "arn:aws:iam::${local.account_id}:oidc-provider/${local.oidc_provider_server_id}" +} diff --git a/storage/s3-velero-backup/main.tf b/storage/s3-velero-backup/main.tf index b86e98b11..89f613d52 100644 --- a/storage/s3-velero-backup/main.tf +++ b/storage/s3-velero-backup/main.tf @@ -4,13 +4,26 @@ terraform { } provider "aws" { - region = var.aws_region + region = var.aws_region assume_role { role_arn = var.aws_assume_role_arn } } +# -------------------------------------------------- +# AWS IAM Open ID Connect Provider +# -------------------------------------------------- + +# Only create a IAM Provider for OIDC if var.oidc_provider_url is supplied. +# If var.oidc_provider_url is not supplied, it means that one already exist, +# and information is fetched through the EKS data source. +module "aws_iam_oidc_provider" { + count = var.oidc_provider_url != null ? 1 : 0 + source = "../../_sub/security/iam-oidc-provider" + eks_openid_connect_provider_url = local.oidc_provider_url +} + resource "aws_s3_bucket" "velero_storage" { bucket = var.bucket_name acl = "private" @@ -79,10 +92,15 @@ data "aws_iam_policy_document" "assume_role" { statement { sid = "" effect = "Allow" - actions = ["sts:AssumeRole"] + actions = ["sts:AssumeRoleWithWebIdentity"] principals { - type = "AWS" - identifiers = var.kiam_server_role_arn + type = "Federated" + identifiers = ["${local.oidc_provider_arn}"] + } + condition { + test = "StringEquals" + variable = "${local.oidc_provider_server_id}:sub" + values = ["system:serviceaccount:${var.namespace}:${var.service_account}"] } } } diff --git a/storage/s3-velero-backup/vars.tf b/storage/s3-velero-backup/vars.tf index b2295299d..1acb0d141 100644 --- a/storage/s3-velero-backup/vars.tf +++ b/storage/s3-velero-backup/vars.tf @@ -11,12 +11,6 @@ variable "bucket_name" { description = "Velero storage bucket name" } -variable "kiam_server_role_arn" { - type = list(string) - default = [""] - description = "Role to allow for trust relationship to KIAM " -} - variable "versioning" { type = bool default = true @@ -24,8 +18,8 @@ variable "versioning" { } variable "velero_iam_role_name" { - type = string - default = "VeleroBackup" + type = string + default = "VeleroBackup" description = "Velero role for S3 actions" } @@ -34,3 +28,27 @@ variable "force_bucket_destroy" { default = true description = "Destroy bucket without error" } + +variable "namespace" { + type = string + default = "velero" + description = "The namespace that Velero will be installed to" +} + +variable "service_account" { + type = string + default = "velero-server" + description = "The service account to be used by Velero" +} + +variable "eks_cluster_name" { + type = string + default = null + description = "The AWS EKS cluster name. Only supply this if Velero S3 bucket and EKS cluster exist in the SAME account" +} + +variable "oidc_provider_url" { + type = string + default = null + description = "The OIDC provider URL. Only supply this if Velero S3 bucket and EKS cluster exist in the DIFFERENT accounts" +} diff --git a/test/integration/_global/s3-bucket-velero/terragrunt.hcl b/test/integration/_global/s3-bucket-velero/terragrunt.hcl index 45b3990d9..d7a776d4b 100644 --- a/test/integration/_global/s3-bucket-velero/terragrunt.hcl +++ b/test/integration/_global/s3-bucket-velero/terragrunt.hcl @@ -13,5 +13,5 @@ dependencies { inputs = { bucket_name = "dfds-velero-qa" - kiam_server_role_arn = ["arn:aws:iam::266901158286:role/eks-qa21-kiam-server"] + eks_cluster_name = "qa21" }