diff --git a/docs/patterns/cell-based-eks.md b/docs/patterns/cell-based-eks.md new file mode 100644 index 0000000000..2a9bc2b46b --- /dev/null +++ b/docs/patterns/cell-based-eks.md @@ -0,0 +1,7 @@ +--- +title: Cell-Based Architecture for Amazon EKS +--- + +{% + include-markdown "../../patterns/cell-based-eks/README.md" +%} diff --git a/patterns/cell-based-eks/0.vpc.tf b/patterns/cell-based-eks/0.vpc.tf new file mode 100644 index 0000000000..81528caba1 --- /dev/null +++ b/patterns/cell-based-eks/0.vpc.tf @@ -0,0 +1,47 @@ +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" {} + +locals { + name = basename(path.cwd) + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +################################################################################ +# VPC +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/patterns/cell-based-eks/1.az1.tf b/patterns/cell-based-eks/1.az1.tf new file mode 100644 index 0000000000..f95c1a3bdf --- /dev/null +++ b/patterns/cell-based-eks/1.az1.tf @@ -0,0 +1,133 @@ +# Required for public ECR where Karpenter artifacts are hosted +provider "aws" { + region = "us-east-1" + alias = "virginia" +} + +data "aws_ecrpublic_authorization_token" "token" { + provider = aws.virginia +} + +provider "kubernetes" { + alias = "k8s-az1" + host = module.eks_az1.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_az1.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks_az1.cluster_name] + } +} + +provider "helm" { + alias = "helm-az1" + kubernetes { + host = module.eks_az1.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_az1.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks_az1.cluster_name] + } + } +} + +locals { + cell1_name = format("%s-%s", local.name, "az1") +} + +################################################################################ +# Cluster +################################################################################ + +module "eks_az1" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.18" + + providers = { + kubernetes = kubernetes.k8s-az1 + } + + cluster_name = local.cell1_name + cluster_version = "1.28" + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.eks_blueprints_addons_az1.karpenter.node_iam_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + + eks_managed_node_groups = { + cell1 = { + instance_types = ["m5.large"] + + min_size = 1 + max_size = 5 + desired_size = 2 + + subnet_ids = [module.vpc.private_subnets[0]] + } + } + + tags = merge(local.tags, { + # NOTE - if creating multiple security groups with this module, only tag the + # security group that Karpenter should utilize with the following tag + # (i.e. - at most, only one security group should have this tag in your account) + "karpenter.sh/discovery" = local.cell1_name + }) +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ + +module "eks_blueprints_addons_az1" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.11" + + providers = { + helm = helm.helm-az1 + kubernetes = kubernetes.k8s-az1 + } + + cluster_name = module.eks_az1.cluster_name + cluster_endpoint = module.eks_az1.cluster_endpoint + cluster_version = module.eks_az1.cluster_version + oidc_provider_arn = module.eks_az1.oidc_provider_arn + + # We want to wait for the EKS Managed Nodegroups to be deployed first + create_delay_dependencies = [for group in module.eks_az1.eks_managed_node_groups : group.node_group_arn] + + eks_addons = { + coredns = {} + vpc-cni = {} + kube-proxy = {} + } + + enable_karpenter = true + karpenter = { + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + } + karpenter_node = { + # Use static name so that it matches what is defined in `az1.yaml` example manifest + iam_role_use_name_prefix = false + } + + tags = local.tags +} diff --git a/patterns/cell-based-eks/2.az2.tf b/patterns/cell-based-eks/2.az2.tf new file mode 100644 index 0000000000..0535bf2ce4 --- /dev/null +++ b/patterns/cell-based-eks/2.az2.tf @@ -0,0 +1,123 @@ +provider "kubernetes" { + alias = "k8s-az2" + host = module.eks_az2.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_az2.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks_az2.cluster_name] + } +} + +provider "helm" { + alias = "helm-az2" + kubernetes { + host = module.eks_az2.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_az2.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks_az2.cluster_name] + } + } +} + +locals { + cell2_name = format("%s-%s", local.name, "az2") +} + +################################################################################ +# Cluster +################################################################################ + +module "eks_az2" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.18" + + providers = { + kubernetes = kubernetes.k8s-az2 + } + + cluster_name = local.cell2_name + cluster_version = "1.28" + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.eks_blueprints_addons_az2.karpenter.node_iam_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + + eks_managed_node_groups = { + cell1 = { + instance_types = ["m5.large"] + + min_size = 1 + max_size = 5 + desired_size = 2 + + subnet_ids = [module.vpc.private_subnets[1]] + } + } + + tags = merge(local.tags, { + # NOTE - if creating multiple security groups with this module, only tag the + # security group that Karpenter should utilize with the following tag + # (i.e. - at most, only one security group should have this tag in your account) + "karpenter.sh/discovery" = local.cell2_name + }) +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ + +module "eks_blueprints_addons_az2" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.11" + + providers = { + helm = helm.helm-az2 + kubernetes = kubernetes.k8s-az2 + } + + cluster_name = module.eks_az2.cluster_name + cluster_endpoint = module.eks_az2.cluster_endpoint + cluster_version = module.eks_az2.cluster_version + oidc_provider_arn = module.eks_az2.oidc_provider_arn + + # We want to wait for the EKS Managed Nodegroups to be deployed first + create_delay_dependencies = [for group in module.eks_az2.eks_managed_node_groups : group.node_group_arn] + + eks_addons = { + coredns = {} + vpc-cni = {} + kube-proxy = {} + } + + enable_karpenter = true + karpenter = { + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + } + karpenter_node = { + # Use static name so that it matches what is defined in `az2.yaml` example manifest + iam_role_use_name_prefix = false + } + + tags = local.tags +} diff --git a/patterns/cell-based-eks/3.az3.tf b/patterns/cell-based-eks/3.az3.tf new file mode 100644 index 0000000000..2797e3700d --- /dev/null +++ b/patterns/cell-based-eks/3.az3.tf @@ -0,0 +1,123 @@ +provider "kubernetes" { + alias = "k8s-az3" + host = module.eks_az3.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_az3.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks_az3.cluster_name] + } +} + +provider "helm" { + alias = "helm-az3" + kubernetes { + host = module.eks_az3.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_az3.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks_az3.cluster_name] + } + } +} + +locals { + cell3_name = format("%s-%s", local.name, "az3") +} + +################################################################################ +# Cluster +################################################################################ + +module "eks_az3" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.18" + + providers = { + kubernetes = kubernetes.k8s-az3 + } + + cluster_name = local.cell3_name + cluster_version = "1.28" + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.eks_blueprints_addons_az3.karpenter.node_iam_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + + eks_managed_node_groups = { + cell1 = { + instance_types = ["m5.large"] + + min_size = 1 + max_size = 5 + desired_size = 2 + + subnet_ids = [module.vpc.private_subnets[2]] + } + } + + tags = merge(local.tags, { + # NOTE - if creating multiple security groups with this module, only tag the + # security group that Karpenter should utilize with the following tag + # (i.e. - at most, only one security group should have this tag in your account) + "karpenter.sh/discovery" = local.cell3_name + }) +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ + +module "eks_blueprints_addons_az3" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.11" + + providers = { + helm = helm.helm-az3 + kubernetes = kubernetes.k8s-az3 + } + + cluster_name = module.eks_az3.cluster_name + cluster_endpoint = module.eks_az3.cluster_endpoint + cluster_version = module.eks_az3.cluster_version + oidc_provider_arn = module.eks_az3.oidc_provider_arn + + # We want to wait for the EKS Managed Nodegroups to be deployed first + create_delay_dependencies = [for group in module.eks_az3.eks_managed_node_groups : group.node_group_arn] + + eks_addons = { + coredns = {} + vpc-cni = {} + kube-proxy = {} + } + + enable_karpenter = true + karpenter = { + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + } + karpenter_node = { + # Use static name so that it matches what is defined in `az3.yaml` example manifest + iam_role_use_name_prefix = false + } + + tags = local.tags +} diff --git a/patterns/cell-based-eks/README.md b/patterns/cell-based-eks/README.md new file mode 100644 index 0000000000..4d4b3c9fd4 --- /dev/null +++ b/patterns/cell-based-eks/README.md @@ -0,0 +1,95 @@ +# Cell-Based Architecture for Amazon EKS + +This pattern demonstrates how to configure a cell-based architecture for Amazon Elastic Kubernetes Service (Amazon EKS) workloads. It moves away from typical multiple Availability Zone (AZ) clusters to a single Availability Zone cluster. These single AZ clusters are called cells, and the aggregation of these cells in each Region is called a supercell. These cells help to ensure that a failure in one cell doesn't affect the cells in another, reducing data transfer costs and improving both the availability and resiliency against AZ wide failures for Amazon EKS workloads. + +Refer to the [AWS Solution Guidance](https://aws.amazon.com/solutions/guidance/cell-based-architecture-for-amazon-eks/) for more details. + +## Deploy + +See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites. This pattern consists 1 VPC and 3 public & private subnets across 3 AZs. Also 3 Amazon EKS clusters are deployed, each in single AZ. + +```bash +terraform init +terraform apply -target="module.vpc" -auto-approve +terraform apply -target="module.eks_az1" -auto-approve +terraform apply -target="module.eks_az2" -auto-approve +terraform apply -target="module.eks_az3" -auto-approve +terraform apply -auto-approve +``` + +## Validate + +1. Export the necessary environment variables and update the local kubeconfig file. + +```bash +export CELL_1=cell-based-eks-az1 +export CELL_2=cell-based-eks-az2 +export CELL_3=cell-based-eks-az3 +export AWS_REGION=$(aws configure get region) #AWS region of the EKS clusters +export AWS_ACCOUNT_NUMBER=$(aws sts get-caller-identity --query "Account" --output text) +export SUBNET_ID_CELL1=$(terraform output -raw subnet_id_az1) +export SUBNET_ID_CELL2=$(terraform output -raw subnet_id_az2) +export SUBNET_ID_CELL3=$(terraform output -raw subnet_id_az3) +alias kgn="kubectl get node -o custom-columns='NODE_NAME:.metadata.name,READY:.status.conditions[?(@.type==\"Ready\")].status,INSTANCE-TYPE:.metadata.labels.node\.kubernetes\.io/instance-type,AZ:.metadata.labels.topology\.kubernetes\.io/zone,CAPACITY-TYPE:.metadata.labels.karpenter\.sh/capacity-type,VERSION:.status.nodeInfo.kubeletVersion,OS-IMAGE:.status.nodeInfo.osImage,INTERNAL-IP:.metadata.annotations.alpha\.kubernetes\.io/provided-node-ip'" +``` + +```bash +aws eks update-kubeconfig --name $CELL_1 --region $AWS_REGION --alias $CELL_1 +aws eks update-kubeconfig --name $CELL_2 --region $AWS_REGION --alias $CELL_2 +aws eks update-kubeconfig --name $CELL_3 --region $AWS_REGION --alias $CELL_3 +``` + +2. Lets start our validation using Cell 1 which is running in AZ1. Verify the existing nodes are deployed to AZ1 (us-west-2a) + +```bash +kgn --context ${CELL_1} +``` +```output +NODE_NAME READY INSTANCE-TYPE AZ CAPACITY-TYPE VERSION OS-IMAGE INTERNAL-IP +ip-10-0-12-83.us-west-2.compute.internal True m5.large us-west-2a v1.28.3-eks-e71965b Amazon Linux 2 10.0.12.83 +ip-10-0-7-191.us-west-2.compute.internal True m5.large us-west-2a v1.28.3-eks-e71965b Amazon Linux 2 10.0.7.191 +``` + +3. Deploy the necessary Karpenter resources like `EC2NodeClass`, `NodePool` and configure them to use AZ1 to launch any EC2 resources + +```bash +sed -i'.bak' -e 's/SUBNET_ID_CELL1/'"${SUBNET_ID_CELL1}"'/g' az1.yaml + +kubectl apply -f az1.yaml --context ${CELL_1} +``` + +4. Deploy a sample application `inflate` with 20 replicas and watch for Karpenter to launch the EC2 worker nodes in AZ1 + +```bash +kubectl apply -f inflate.yaml --context ${CELL_1} + +kubectl wait --for=condition=ready pods --all --timeout 2m --context ${CELL_1} +``` + +5. List the EKS worker nodes to verify all of them are deployed to AZ1 + +```bash +kgn --context ${CELL_1} +``` +```output +NODE_NAME READY INSTANCE-TYPE AZ CAPACITY-TYPE VERSION OS-IMAGE INTERNAL-IP +ip-10-0-11-154.us-west-2.compute.internal True c7g.8xlarge us-west-2a spot v1.28.3-eks-e71965b Amazon Linux 2 10.0.11.154 +ip-10-0-12-83.us-west-2.compute.internal True m5.large us-west-2a v1.28.3-eks-e71965b Amazon Linux 2 10.0.12.83 +ip-10-0-7-191.us-west-2.compute.internal True m5.large us-west-2a v1.28.3-eks-e71965b Amazon Linux 2 10.0.7.191 +``` + +6. Repeat the steps from 2 to 5 for Cell 2 and Cell 3 using --context $CELL_2, $CELL_3 respectively. + +## Destroy + +To teardown and remove the resources created in the pattern, the typical steps of execution are as follows: + +```bash +terraform destroy -target="module.eks_blueprints_addons_az1" -auto-approve +terraform destroy -target="module.eks_blueprints_addons_az2" -auto-approve +terraform destroy -target="module.eks_blueprints_addons_az3" -auto-approve +terraform destroy -target="module.eks_az1" -auto-approve +terraform destroy -target="module.eks_az2" -auto-approve +terraform destroy -target="module.eks_az3" -auto-approve +terraform destroy -auto-approve +``` diff --git a/patterns/cell-based-eks/az1.yaml b/patterns/cell-based-eks/az1.yaml new file mode 100644 index 0000000000..4c821d91e8 --- /dev/null +++ b/patterns/cell-based-eks/az1.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + role: karpenter-cell-based-eks-az1 + subnetSelectorTerms: + - id: SUBNET_ID_CELL1 + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: cell-based-eks-az1 + tags: + karpenter.sh/discovery: cell-based-eks-az1 +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s diff --git a/patterns/cell-based-eks/az2.yaml b/patterns/cell-based-eks/az2.yaml new file mode 100644 index 0000000000..484fe169ad --- /dev/null +++ b/patterns/cell-based-eks/az2.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + role: karpenter-cell-based-eks-az2 + subnetSelectorTerms: + - id: SUBNET_ID_CELL2 + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: cell-based-eks-az2 + tags: + karpenter.sh/discovery: cell-based-eks-az2 +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s diff --git a/patterns/cell-based-eks/az3.yaml b/patterns/cell-based-eks/az3.yaml new file mode 100644 index 0000000000..ecc1d7c696 --- /dev/null +++ b/patterns/cell-based-eks/az3.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + role: karpenter-cell-based-eks-az3 + subnetSelectorTerms: + - id: SUBNET_ID_CELL3 + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: cell-based-eks-az3 + tags: + karpenter.sh/discovery: cell-based-eks-az3 +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s diff --git a/patterns/cell-based-eks/inflate.yaml b/patterns/cell-based-eks/inflate.yaml new file mode 100644 index 0000000000..f755e5f244 --- /dev/null +++ b/patterns/cell-based-eks/inflate.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 20 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 diff --git a/patterns/cell-based-eks/outputs.tf b/patterns/cell-based-eks/outputs.tf new file mode 100644 index 0000000000..a87926d032 --- /dev/null +++ b/patterns/cell-based-eks/outputs.tf @@ -0,0 +1,34 @@ +output "vpc_id" { + description = "Amazon EKS VPC ID" + value = module.vpc.vpc_id +} + +output "subnet_id_az1" { + description = "Amazon EKS AZ1 Cluster Subnet ID" + value = module.vpc.private_subnets[0] +} + +output "subnet_id_az2" { + description = "Amazon EKS AZ2 Cluster Subnet ID" + value = module.vpc.private_subnets[1] +} + +output "subnet_id_az3" { + description = "Amazon EKS AZ3 Cluster Subnet ID" + value = module.vpc.private_subnets[2] +} + +output "configure_kubectl_az1" { + description = "Configure kubectl for AZ1 Cluster: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks_az1.cluster_name}" +} + +output "configure_kubectl_az2" { + description = "Configure kubectl for AZ2 Cluster: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks_az2.cluster_name}" +} + +output "configure_kubectl_az3" { + description = "Configure kubectl for AZ3 Cluster: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks_az3.cluster_name}" +} diff --git a/patterns/cell-based-eks/test_setup.sh b/patterns/cell-based-eks/test_setup.sh new file mode 100755 index 0000000000..41e6390aec --- /dev/null +++ b/patterns/cell-based-eks/test_setup.sh @@ -0,0 +1,87 @@ +export CELL_1=cell-based-eks-az1 +export CELL_2=cell-based-eks-az2 +export CELL_3=cell-based-eks-az3 +export AWS_DEFAULT_REGION=$(aws configure get region) +export AWS_ACCOUNT_NUMBER=$(aws sts get-caller-identity --query "Account" --output text) +export SUBNET_ID_CELL1=$(terraform output -raw subnet_id_az1) +export SUBNET_ID_CELL2=$(terraform output -raw subnet_id_az2) +export SUBNET_ID_CELL3=$(terraform output -raw subnet_id_az3) + +aws eks update-kubeconfig --name $CELL_1 --region $AWS_DEFAULT_REGION +aws eks update-kubeconfig --name $CELL_2 --region $AWS_DEFAULT_REGION +aws eks update-kubeconfig --name $CELL_3 --region $AWS_DEFAULT_REGION + +export CTX_CELL_1=arn:aws:eks:$AWS_DEFAULT_REGION:${AWS_ACCOUNT_NUMBER}:cluster/$CELL_1 +export CTX_CELL_2=arn:aws:eks:$AWS_DEFAULT_REGION:${AWS_ACCOUNT_NUMBER}:cluster/$CELL_2 +export CTX_CELL_3=arn:aws:eks:$AWS_DEFAULT_REGION:${AWS_ACCOUNT_NUMBER}:cluster/$CELL_3 + +bold=$(tput bold) +normal=$(tput sgr0) + +alias kgn="kubectl get node -o custom-columns='NODE_NAME:.metadata.name,READY:.status.conditions[?(@.type==\"Ready\")].status,INSTANCE-TYPE:.metadata.labels.node\.kubernetes\.io/instance-type,CAPACITY-TYPE:.metadata.labels.karpenter\.sh/capacity-type,AZ:.metadata.labels.topology\.kubernetes\.io/zone,VERSION:.status.nodeInfo.kubeletVersion,OS-IMAGE:.status.nodeInfo.osImage,INTERNAL-IP:.metadata.annotations.alpha\.kubernetes\.io/provided-node-ip'" + +echo "------------${bold}Test the Cell-1 Setup${normal}-------------" + +echo "${bold}Cell-1: Nodes before the scaling event${normal}" + +kgn --context="${CTX_CELL_1}" + +sed -i'.bak' -e 's/SUBNET_ID_CELL1/'"${SUBNET_ID_CELL1}"'/g' az1.yaml + +kubectl apply -f az1.yaml,inflate.yaml --context="${CTX_CELL_1}" + +echo "${bold}Cell-1: Scaling the inflate deployment to 50 replicas${normal}" + +kubectl scale deployment inflate --replicas 20 --context="${CTX_CELL_1}" + +echo "${bold}Cell-1: Wait for karpenter to launch the worker nodes and pods become ready......${normal}" + +kubectl wait --for=condition=ready pods --all --timeout 2m --context="${CTX_CELL_1}" + +echo "${bold}Cell-1: Nodes after the scaling event${normal}" + +kgn --context="${CTX_CELL_1}" + +echo "------------${bold}Test the Cell-2 Setup${normal}-------------" + +echo "${bold}Cell-2: Nodes before the scaling event${normal}" + +kgn --context="${CTX_CELL_2}" + +sed -i'.bak' -e 's/SUBNET_ID_CELL2/'"${SUBNET_ID_CELL2}"'/g' az2.yaml + +kubectl apply -f az2.yaml,inflate.yaml --context="${CTX_CELL_2}" + +echo "${bold}Cell-2: Scaling the inflate deployment to 50 replicas${normal}" + +kubectl scale deployment inflate --replicas 20 --context="${CTX_CELL_2}" + +echo "${bold}Cell-2: Wait for karpenter to launch the worker nodes and pods become ready......${normal}" + +kubectl wait --for=condition=ready pods --all --timeout 2m --context="${CTX_CELL_2}" + +echo "${bold}Cell-2: Nodes after the scaling event${normal}" + +kgn --context="${CTX_CELL_2}" + +echo "------------${bold}Test the Cell-3 Setup${normal}-------------" + +echo "${bold}Cell-3: Nodes before the scaling event${normal}" + +kgn --context="${CTX_CELL_3}" + +sed -i'.bak' -e 's/SUBNET_ID_CELL3/'"${SUBNET_ID_CELL3}"'/g' az3.yaml + +kubectl apply -f az3.yaml,inflate.yaml --context="${CTX_CELL_3}" + +echo "${bold}Cell-3: Scaling the inflate deployment to 50 replicas${normal}" + +kubectl scale deployment inflate --replicas 20 --context="${CTX_CELL_3}" + +echo "${bold}Cell-3: Wait for karpenter to launch the worker nodes and pods become ready......${normal}" + +kubectl wait --for=condition=ready pods --all --timeout 2m --context="${CTX_CELL_3}" + +echo "${bold}Cell-3: Nodes after the scaling event${normal}" + +kgn --context="${CTX_CELL_3}" diff --git a/patterns/cell-based-eks/variables.tf b/patterns/cell-based-eks/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/patterns/cell-based-eks/versions.tf b/patterns/cell-based-eks/versions.tf new file mode 100644 index 0000000000..e11e084c1d --- /dev/null +++ b/patterns/cell-based-eks/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.47" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.9" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "" + # region = "" + # key = "e2e/cell-based-eks/terraform.tfstate" + # } +}