feat: Multi-cluster architecture to increase resiliency and reduce inter-az data transfer charges

docs/patterns/cell-based-eks.md

@@ -0,0 +1,7 @@
+---
+title: Cell-Based Architecture for Amazon EKS
+---
+
+{%
+ include-markdown "../../patterns/cell-based-eks/README.md"
+%}

patterns/cell-based-eks/0.vpc.tf

@@ -0,0 +1,47 @@
+provider "aws" {
+ region = local.region
+}
+
+data "aws_availability_zones" "available" {}
+
+locals {
+ name = basename(path.cwd)
+ region = "us-west-2"
+
+ vpc_cidr = "10.0.0.0/16"
+ azs = slice(data.aws_availability_zones.available.names, 0, 3)
+
+ tags = {
+ Blueprint = local.name
+ GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints"
+ }
+}
+
+################################################################################
+# VPC
+################################################################################
+
+module "vpc" {
+ source = "terraform-aws-modules/vpc/aws"
+ version = "~> 5.0"
+
+ name = local.name
+ cidr = local.vpc_cidr
+
+ azs = local.azs
+ private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
+ public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)]
+
+ enable_nat_gateway = true
+ single_nat_gateway = true
+
+ public_subnet_tags = {
+ "kubernetes.io/role/elb" = 1
+ }
+
+ private_subnet_tags = {
+ "kubernetes.io/role/internal-elb" = 1
+ }
+
+ tags = local.tags
+}

patterns/cell-based-eks/1.az1.tf

@@ -0,0 +1,133 @@
+# Required for public ECR where Karpenter artifacts are hosted
+provider "aws" {
+ region = "us-east-1"
+ alias = "virginia"
+}
+
+data "aws_ecrpublic_authorization_token" "token" {
+ provider = aws.virginia
+}
+
+provider "kubernetes" {
+ alias = "k8s-az1"
+ host = module.eks_az1.cluster_endpoint
+ cluster_ca_certificate = base64decode(module.eks_az1.cluster_certificate_authority_data)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ command = "aws"
+ # This requires the awscli to be installed locally where Terraform is executed
+ args = ["eks", "get-token", "--cluster-name", module.eks_az1.cluster_name]
+ }
+}
+
+provider "helm" {
+ alias = "helm-az1"
+ kubernetes {
+ host = module.eks_az1.cluster_endpoint
+ cluster_ca_certificate = base64decode(module.eks_az1.cluster_certificate_authority_data)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ command = "aws"
+ # This requires the awscli to be installed locally where Terraform is executed
+ args = ["eks", "get-token", "--cluster-name", module.eks_az1.cluster_name]
+ }
+ }
+}
+
+locals {
+ cell1_name = format("%s-%s", local.name, "az1")
+}
+
+################################################################################
+# Cluster
+################################################################################
+
+module "eks_az1" {
+ source = "terraform-aws-modules/eks/aws"
+ version = "~> 19.18"
+
+ providers = {
+ kubernetes = kubernetes.k8s-az1
+ }
+
+ cluster_name = local.cell1_name
+ cluster_version = "1.28"
+ cluster_endpoint_public_access = true
+
+ vpc_id = module.vpc.vpc_id
+ subnet_ids = module.vpc.private_subnets
+
+ manage_aws_auth_configmap = true
+ aws_auth_roles = [
+ # We need to add in the Karpenter node IAM role for nodes launched by Karpenter
+ {
+ rolearn = module.eks_blueprints_addons_az1.karpenter.node_iam_role_arn
+ username = "system:node:{{EC2PrivateDNSName}}"
+ groups = [
+ "system:bootstrappers",
+ "system:nodes",
+ ]
+ },
+ ]
+
+ eks_managed_node_groups = {
+ cell1 = {
+ instance_types = ["m5.large"]
+
+ min_size = 1
+ max_size = 5
+ desired_size = 2
+
+ subnet_ids = [module.vpc.private_subnets[0]]
+ }
+ }
+
+ tags = merge(local.tags, {
+ # NOTE - if creating multiple security groups with this module, only tag the
+ # security group that Karpenter should utilize with the following tag
+ # (i.e. - at most, only one security group should have this tag in your account)
+ "karpenter.sh/discovery" = local.cell1_name
+ })
+}
+
+################################################################################
+# EKS Blueprints Addons
+################################################################################
+
+module "eks_blueprints_addons_az1" {
+ source = "aws-ia/eks-blueprints-addons/aws"
+ version = "~> 1.11"
+
+ providers = {
+ helm = helm.helm-az1
+ kubernetes = kubernetes.k8s-az1
+ }
+
+ cluster_name = module.eks_az1.cluster_name
+ cluster_endpoint = module.eks_az1.cluster_endpoint
+ cluster_version = module.eks_az1.cluster_version
+ oidc_provider_arn = module.eks_az1.oidc_provider_arn
+
+ # We want to wait for the EKS Managed Nodegroups to be deployed first
+ create_delay_dependencies = [for group in module.eks_az1.eks_managed_node_groups : group.node_group_arn]
+
+ eks_addons = {
+ coredns = {}
+ vpc-cni = {}
+ kube-proxy = {}
+ }
+
+ enable_karpenter = true
+ karpenter = {
+ repository_username = data.aws_ecrpublic_authorization_token.token.user_name
+ repository_password = data.aws_ecrpublic_authorization_token.token.password
+ }
+ karpenter_node = {
+ # Use static name so that it matches what is defined in `az1.yaml` example manifest
+ iam_role_use_name_prefix = false
+ }
+
+ tags = local.tags
+}

patterns/cell-based-eks/2.az2.tf

@@ -0,0 +1,123 @@
+provider "kubernetes" {
+ alias = "k8s-az2"
+ host = module.eks_az2.cluster_endpoint
+ cluster_ca_certificate = base64decode(module.eks_az2.cluster_certificate_authority_data)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ command = "aws"
+ # This requires the awscli to be installed locally where Terraform is executed
+ args = ["eks", "get-token", "--cluster-name", module.eks_az2.cluster_name]
+ }
+}
+
+provider "helm" {
+ alias = "helm-az2"
+ kubernetes {
+ host = module.eks_az2.cluster_endpoint
+ cluster_ca_certificate = base64decode(module.eks_az2.cluster_certificate_authority_data)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ command = "aws"
+ # This requires the awscli to be installed locally where Terraform is executed
+ args = ["eks", "get-token", "--cluster-name", module.eks_az2.cluster_name]
+ }
+ }
+}
+
+locals {
+ cell2_name = format("%s-%s", local.name, "az2")
+}
+
+################################################################################
+# Cluster
+################################################################################
+
+module "eks_az2" {
+ source = "terraform-aws-modules/eks/aws"
+ version = "~> 19.18"
+
+ providers = {
+ kubernetes = kubernetes.k8s-az2
+ }
+
+ cluster_name = local.cell2_name
+ cluster_version = "1.28"
+ cluster_endpoint_public_access = true
+
+ vpc_id = module.vpc.vpc_id
+ subnet_ids = module.vpc.private_subnets
+
+ manage_aws_auth_configmap = true
+ aws_auth_roles = [
+ # We need to add in the Karpenter node IAM role for nodes launched by Karpenter
+ {
+ rolearn = module.eks_blueprints_addons_az2.karpenter.node_iam_role_arn
+ username = "system:node:{{EC2PrivateDNSName}}"
+ groups = [
+ "system:bootstrappers",
+ "system:nodes",
+ ]
+ },
+ ]
+
+ eks_managed_node_groups = {
+ cell1 = {
+ instance_types = ["m5.large"]
+
+ min_size = 1
+ max_size = 5
+ desired_size = 2
+
+ subnet_ids = [module.vpc.private_subnets[1]]
+ }
+ }
+
+ tags = merge(local.tags, {
+ # NOTE - if creating multiple security groups with this module, only tag the
+ # security group that Karpenter should utilize with the following tag
+ # (i.e. - at most, only one security group should have this tag in your account)
+ "karpenter.sh/discovery" = local.cell2_name
+ })
+}
+
+################################################################################
+# EKS Blueprints Addons
+################################################################################
+
+module "eks_blueprints_addons_az2" {
+ source = "aws-ia/eks-blueprints-addons/aws"
+ version = "~> 1.11"
+
+ providers = {
+ helm = helm.helm-az2
+ kubernetes = kubernetes.k8s-az2
+ }
+
+ cluster_name = module.eks_az2.cluster_name
+ cluster_endpoint = module.eks_az2.cluster_endpoint
+ cluster_version = module.eks_az2.cluster_version
+ oidc_provider_arn = module.eks_az2.oidc_provider_arn
+
+ # We want to wait for the EKS Managed Nodegroups to be deployed first
+ create_delay_dependencies = [for group in module.eks_az2.eks_managed_node_groups : group.node_group_arn]
+
+ eks_addons = {
+ coredns = {}
+ vpc-cni = {}
+ kube-proxy = {}
+ }
+
+ enable_karpenter = true
+ karpenter = {
+ repository_username = data.aws_ecrpublic_authorization_token.token.user_name
+ repository_password = data.aws_ecrpublic_authorization_token.token.password
+ }
+ karpenter_node = {
+ # Use static name so that it matches what is defined in `az2.yaml` example manifest
+ iam_role_use_name_prefix = false
+ }
+
+ tags = local.tags
+}