From 49740eb868632a09769055e058c37042573909fd Mon Sep 17 00:00:00 2001 From: Garland Kan Date: Thu, 29 Apr 2021 12:26:49 -0700 Subject: [PATCH] Cluster autoscaler (#131) * Adding initial cluster autoscaler Signed-off-by: gar * running tf fmt Signed-off-by: gar * Filling out input param to the policy Signed-off-by: gar * Test commit Signed-off-by: gar * Removing debug test output Signed-off-by: gar * scoping the role to a cluster Signed-off-by: gar --- .../aws/cluster-autoscaler/README.md | 3 + .../cluster-autoscaler/helm_values.yaml.tpl | 14 +++ .../aws/cluster-autoscaler/main.tf | 87 +++++++++++++++++++ .../aws/cluster-autoscaler/variables.tf | 36 ++++++++ terraform-modules/aws/eks/outputs.tf | 4 + 5 files changed, 144 insertions(+) create mode 100644 terraform-modules/aws/cluster-autoscaler/README.md create mode 100644 terraform-modules/aws/cluster-autoscaler/helm_values.yaml.tpl create mode 100644 terraform-modules/aws/cluster-autoscaler/main.tf create mode 100644 terraform-modules/aws/cluster-autoscaler/variables.tf diff --git a/terraform-modules/aws/cluster-autoscaler/README.md b/terraform-modules/aws/cluster-autoscaler/README.md new file mode 100644 index 000000000..c1e839f82 --- /dev/null +++ b/terraform-modules/aws/cluster-autoscaler/README.md @@ -0,0 +1,3 @@ +# EKS cluster autoscaler + +source: https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/irsa diff --git a/terraform-modules/aws/cluster-autoscaler/helm_values.yaml.tpl b/terraform-modules/aws/cluster-autoscaler/helm_values.yaml.tpl new file mode 100644 index 000000000..5eb57c7f8 --- /dev/null +++ b/terraform-modules/aws/cluster-autoscaler/helm_values.yaml.tpl @@ -0,0 +1,14 @@ +awsRegion: ${awsRegion} + +rbac: + create: true + serviceAccount: + # This value should match local.k8s_service_account_name in locals.tf + name: ${serviceAccountName} + annotations: + # This value should match the ARN of the role created by module.iam_assumable_role_admin in irsa.tf + eks.amazonaws.com/role-arn: "arn:aws:iam::${awsAccountID}:role/cluster-autoscaler-${clusterName}" + +autoDiscovery: + clusterName: ${clusterName} + enabled: true diff --git a/terraform-modules/aws/cluster-autoscaler/main.tf b/terraform-modules/aws/cluster-autoscaler/main.tf new file mode 100644 index 000000000..5c47aa147 --- /dev/null +++ b/terraform-modules/aws/cluster-autoscaler/main.tf @@ -0,0 +1,87 @@ +module "iam_assumable_role_admin" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "3.6.0" + create_role = true + role_name = "cluster-autoscaler-${var.cluster_name}" + provider_url = replace(var.eks_cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.cluster_autoscaler.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:${var.k8s_service_account_namespace}:${var.k8s_service_account_name}"] +} + +resource "aws_iam_policy" "cluster_autoscaler" { + name_prefix = "cluster-autoscaler-${var.cluster_name}" + description = "EKS cluster-autoscaler policy for cluster ${var.eks_cluster_id}" + policy = data.aws_iam_policy_document.cluster_autoscaler.json +} + +data "aws_iam_policy_document" "cluster_autoscaler" { + statement { + sid = "clusterAutoscalerAll" + effect = "Allow" + + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions", + ] + + resources = ["*"] + } + + statement { + sid = "clusterAutoscalerOwn" + effect = "Allow" + + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${var.eks_cluster_id}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled" + values = ["true"] + } + } +} + +data "aws_caller_identity" "current" {} + +# +# Helm - cluster-autoscaler +# +data "template_file" "helm_values" { + template = file("${path.module}/helm_values.yaml.tpl") + vars = { + awsAccountID = data.aws_caller_identity.current.account_id + awsRegion = var.aws_region + clusterName = var.cluster_name + serviceAccountName = var.k8s_service_account_name + } +} + +module "cluster-autoscaler" { + source = "github.com/ManagedKube/kubernetes-ops//terraform-modules/aws/helm/helm_generic?ref=v1.0.9" + + repository = "https://kubernetes.github.io/autoscaler" + official_chart_name = "cluster-autoscaler" + user_chart_name = "cluster-autoscaler" + helm_version = "9.9.2" + namespace = "kube-system" + helm_values = data.template_file.helm_values.rendered + + depends_on = [ + module.iam_assumable_role_admin + ] +} diff --git a/terraform-modules/aws/cluster-autoscaler/variables.tf b/terraform-modules/aws/cluster-autoscaler/variables.tf new file mode 100644 index 000000000..1615ad09f --- /dev/null +++ b/terraform-modules/aws/cluster-autoscaler/variables.tf @@ -0,0 +1,36 @@ +variable "aws_region" { + type = string + default = "us-east-1" + description = "AWS region" +} + +variable "cluster_name" { + type = string + default = "cluster" + description = "EKS cluster name" +} + + +variable "eks_cluster_id" { + type = string + default = "" + description = "EKS cluster ID" +} + +variable "eks_cluster_oidc_issuer_url" { + type = string + default = "" + description = "EKS cluster oidc issuer url" +} + +variable "k8s_service_account_namespace" { + type = string + default = "kube-system" + description = "Namespace to place the service account into" +} + +variable "k8s_service_account_name" { + type = string + default = "cluster-autoscaler-aws-cluster-autoscaler" + description = "Service account name" +} diff --git a/terraform-modules/aws/eks/outputs.tf b/terraform-modules/aws/eks/outputs.tf index dc89dc223..7ed2824b4 100644 --- a/terraform-modules/aws/eks/outputs.tf +++ b/terraform-modules/aws/eks/outputs.tf @@ -14,3 +14,7 @@ output "cluster_certificate_authority_data" { output "cluster_id" { value = module.eks.cluster_id } + +output "cluster_oidc_issuer_url" { + value = module.eks.cluster_oidc_issuer_url +}