diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index cab935af62..c35f235b2c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,33 +1,23 @@ -## :bangbang: PLEASE READ THIS FIRST :bangbang: - -The direction for EKS Blueprints will soon shift from providing an all-encompassing, monolithic "framework" and instead focus more on how users can organize a set of modular components to create the desired solution on Amazon EKS. We have updated the [examples](https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/main/examples) to show how we use the https://github.com/terraform-aws-modules/terraform-aws-eks for EKS cluster and node group creation. We will not be accepting any PRs that apply to EKS cluster or node group creation process. Any such PR may be closed by the maintainers. - -We are hitting also the pause button on new add-on creations at this time until a future roadmap for add-ons is finalized. Please do not submit new add-on PRs. Any such PR may be closed by the maintainers. - -Please track progress, learn what's new and how the migration path would look like to upgrade your current Terraform deployments. We welcome the EKS Blueprints community to continue the discussion in issue https://github.com/aws-ia/terraform-aws-eks-blueprints/issues/1421 - -### What does this PR do? +# Description + +A brief description of the change being made with this pull request. +--> -### Motivation +### Motivation and Context - Resolves # -### More +### How was this change tested? - [ ] Yes, I have tested the PR using my local account setup (Provide any test evidence report under Additional Notes) - [ ] Yes, I have updated the [docs](https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/main/docs) for this feature - [ ] Yes, I ran `pre-commit run -a` with this PR -### For Moderators - -- [ ] E2E Test successfully complete before merge? - ### Additional Notes diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..253bcb76ba --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000000..8692d30d51 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,27 @@ +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a Pull Request, +# surfacing known-vulnerable versions of the packages declared or updated in the PR. +# Once installed, if the workflow run is marked as required, +# PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 + with: + egress-policy: audit + + - name: 'Checkout Repository' + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - name: 'Dependency Review' + uses: actions/dependency-review-action@1360a344ccb0ab6e9475edef90ad2f46bf8003b1 # v3.0.6 diff --git a/.github/workflows/e2e-parallel-destroy.yml b/.github/workflows/e2e-parallel-destroy.yml index 529248aade..a09f422418 100644 --- a/.github/workflows/e2e-parallel-destroy.yml +++ b/.github/workflows/e2e-parallel-destroy.yml @@ -10,6 +10,9 @@ on: concurrency: e2e-parallel-destroy +permissions: + contents: read + jobs: deploy: name: Run e2e test @@ -34,6 +37,11 @@ jobs: - example_path: examples/vpc-cni-custom-networking steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout uses: actions/checkout@v3 @@ -42,7 +50,7 @@ jobs: run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf - name: Auth AWS - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v2.2.0 with: role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/e2e-parallel-full.yml b/.github/workflows/e2e-parallel-full.yml index e0d2fdb0af..6aa007cf42 100644 --- a/.github/workflows/e2e-parallel-full.yml +++ b/.github/workflows/e2e-parallel-full.yml @@ -14,6 +14,9 @@ env: IAMLIVE_VERSION: v0.48.0 BUCKET_NAME: terraform-eks-blueprints-iam-policies-examples +permissions: + contents: read + jobs: prereq-cleanup: name: Prerequisite Cleanup @@ -23,11 +26,16 @@ jobs: id-token: write contents: read steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout uses: actions/checkout@v3 - name: Auth AWS - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v2.2.0 with: role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} aws-region: us-west-2 @@ -62,6 +70,11 @@ jobs: - example_path: examples/stateful - example_path: examples/vpc-cni-custom-networking steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout uses: actions/checkout@v3 @@ -70,7 +83,7 @@ jobs: run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf - name: Auth AWS - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v2.2.0 with: role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} aws-region: us-west-2 @@ -147,11 +160,16 @@ jobs: runs-on: ubuntu-latest steps: # Be careful not to change this to explicit checkout from PR ref/code, as below we run a python code that may change from the PR code. + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout uses: actions/checkout@v3 - name: Configure AWS credentials from Test account - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v2.2.0 with: role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index ddaec1eb3b..7fc91bb630 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -13,10 +13,18 @@ on: paths: - "**/*.md" +permissions: + contents: read + jobs: markdown-link-check: runs-on: ubuntu-latest steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: diff --git a/.github/workflows/plan-examples.yml b/.github/workflows/plan-examples.yml index c28e77ca89..eb95350454 100644 --- a/.github/workflows/plan-examples.yml +++ b/.github/workflows/plan-examples.yml @@ -11,6 +11,9 @@ concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true +permissions: + contents: read + jobs: getExampleDirectories: name: Get example directories @@ -23,6 +26,11 @@ jobs: directories: ${{ steps.dirs.outputs.directories }} steps: # Be careful not to change this to explicit checkout from PR ref/code, as below we run a python code that may change from the PR code. + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout uses: actions/checkout@v3 @@ -49,6 +57,11 @@ jobs: directory: ${{ fromJson(needs.getExampleDirectories.outputs.directories) }} steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Remove default Terraform run: rm -rf $(which terraform) @@ -75,7 +88,7 @@ jobs: - '*.tf' - name: Configure AWS credentials from Test account - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v2.2.0 if: steps.changes.outputs.src== 'true' with: role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index b4b2c9b59c..04153efe94 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -7,12 +7,23 @@ on: - edited - synchronize +permissions: + contents: read + jobs: main: + permissions: + pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs + statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5.0.2 + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + + - uses: amannn/action-semantic-pull-request@v5.2.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 4e4ac382d1..834f6350ec 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -26,12 +26,17 @@ jobs: outputs: directories: ${{ steps.dirs.outputs.directories }} steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout uses: actions/checkout@v3 - name: Get root directories id: dirs - uses: clowdhaus/terraform-composite-actions/directories@v1.8.0 + uses: clowdhaus/terraform-composite-actions/directories@v1.8.3 preCommitMinVersions: name: Min TF pre-commit @@ -41,6 +46,11 @@ jobs: matrix: directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Remove default Terraform run: rm -rf $(which terraform) @@ -70,7 +80,7 @@ jobs: restore-keys: ${{ runner.os }}-terraform- - name: Terraform min/max versions - uses: clowdhaus/terraform-min-max@v1.2.0 + uses: clowdhaus/terraform-min-max@v1.2.7 if: steps.changes.outputs.src== 'true' id: minMax with: @@ -99,6 +109,11 @@ jobs: runs-on: ubuntu-latest needs: collectInputs steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Remove default Terraform run: rm -rf $(which terraform) @@ -130,7 +145,7 @@ jobs: - name: Terraform min/max versions id: minMax - uses: clowdhaus/terraform-min-max@v1.2.0 + uses: clowdhaus/terraform-min-max@v1.2.7 if: steps.changes.outputs.src== 'true' - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 9a806ec576..5366bc1e30 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -3,30 +3,32 @@ on: push: branches: - main - paths: - - 'docs/**' - - mkdocs.yml - - README.md - - '.github/workflows/publish-docs.yml' - release: - types: - - published env: PYTHON_VERSION: 3.x +permissions: + contents: read + jobs: build: name: Deploy docs runs-on: ubuntu-latest + permissions: + contents: write steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout main uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_VERSION }} @@ -34,7 +36,7 @@ jobs: run: | python -m pip install --upgrade pip pip install mike==1.1.2 \ - mkdocs-material==9.1.4 \ + mkdocs-material==9.1.19 \ mkdocs-include-markdown-plugin==4.0.4 \ mkdocs-awesome-pages-plugin==2.9.1 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml new file mode 100644 index 0000000000..04aca8d5c6 --- /dev/null +++ b/.github/workflows/scorecards.yml @@ -0,0 +1,76 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '20 7 * * 2' + push: + branches: ["main"] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + contents: read + actions: read + + steps: + - name: Harden Runner + uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 + with: + egress-policy: audit + + - name: "Checkout code" + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecards on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@6ca1aa8c195c3ca3e77c174fe0356db1bce3b319 # v2.21.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/stale-issue-pr.yml b/.github/workflows/stale-issue-pr.yml index 035a69fb69..be3cdefb22 100644 --- a/.github/workflows/stale-issue-pr.yml +++ b/.github/workflows/stale-issue-pr.yml @@ -4,6 +4,9 @@ on: schedule: - cron: '0 0 * * *' +permissions: + contents: read + jobs: stale: runs-on: ubuntu-latest @@ -11,6 +14,11 @@ jobs: issues: write pull-requests: write steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - uses: actions/stale@main id: stale with: diff --git a/.gitignore b/.gitignore index 2c3661dc57..dc36ca0831 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ terraform.rc # local env *.envrc *kube-config.yaml + +# Generated files +builds diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 600b1ef432..08b92035ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.80.0 + rev: v1.81.0 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/docs/blueprints/istio.md b/docs/blueprints/istio.md new file mode 100644 index 0000000000..c4473f99eb --- /dev/null +++ b/docs/blueprints/istio.md @@ -0,0 +1,7 @@ +--- +title: Istio +--- + +{% + include-markdown "../../examples/istio/README.md" +%} diff --git a/docs/blueprints/privatelink-access.md b/docs/blueprints/privatelink-access.md new file mode 100644 index 0000000000..a58bc3f5c9 --- /dev/null +++ b/docs/blueprints/privatelink-access.md @@ -0,0 +1,7 @@ +--- +title: PrivateLink Access +--- + +{% + include-markdown "../../examples/privatelink-access/README.md" +%} diff --git a/examples/blue-green-upgrade/environment/versions.tf b/examples/blue-green-upgrade/environment/versions.tf index bfb0f2ee9a..967337612f 100644 --- a/examples/blue-green-upgrade/environment/versions.tf +++ b/examples/blue-green-upgrade/environment/versions.tf @@ -1,13 +1,14 @@ terraform { - required_version = ">= 1.0.0" + required_version = ">= 1.0" required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.72.0" + version = ">= 4.67" } random = { - version = ">= 3" + source = "hashicorp/random" + version = ">= 3.0" } } } diff --git a/examples/fargate-serverless/README.md b/examples/fargate-serverless/README.md index fa30269973..703cc2416d 100644 --- a/examples/fargate-serverless/README.md +++ b/examples/fargate-serverless/README.md @@ -8,7 +8,7 @@ This example solution provides: - AWS EKS Fargate Profiles for the `kube-system` namespace which is used by the `coredns`, `vpc-cni`, and `kube-proxy` addons, as well as profile that will match on `app-*` namespaces using a wildcard pattern. - AWS EKS managed addons `coredns`, `vpc-cni` and `kube-proxy` - AWS Load Balancer Controller add-on deployed through a Helm chart. The default AWS Load Balancer Controller add-on configuration is overridden so that it can be deployed on Fargate compute. -- A [sample-app](./sample-app) is provided to demonstrates how to configure the Ingress so that application can be accessed over the internet. +- A sample-app is provided (in-line) to demonstrate how to configure the Ingress so that application can be accessed over the internet. ## Prerequisites: @@ -41,7 +41,7 @@ Apply complete! Resources: 63 added, 0 changed, 0 destroyed. Outputs: -configure_kubectl = "aws eks --region us-west-2 update-kubeconfig --name fully-private-cluster" +configure_kubectl = "aws eks --region us-west-2 update-kubeconfig --name fargate-serverless" ``` 2. Run `update-kubeconfig` command, using the Terraform provided Output, replace with your `$AWS_REGION` and your `$CLUSTER_NAME` variables. diff --git a/examples/fargate-serverless/main.tf b/examples/fargate-serverless/main.tf index ef2678874d..a718f822cd 100644 --- a/examples/fargate-serverless/main.tf +++ b/examples/fargate-serverless/main.tf @@ -100,6 +100,9 @@ module "eks_blueprints_addons" { cluster_version = module.eks.cluster_version oidc_provider_arn = module.eks.oidc_provider_arn + # We want to wait for the Fargate profiles to be deployed first + create_delay_dependencies = [for prof in module.eks.fargate_profiles : prof.fargate_profile_arn] + # EKS Add-ons eks_addons = { coredns = { diff --git a/examples/fully-private-cluster/main.tf b/examples/fully-private-cluster/main.tf index 7490e5e534..dd2116caf0 100644 --- a/examples/fully-private-cluster/main.tf +++ b/examples/fully-private-cluster/main.tf @@ -77,39 +77,22 @@ module "vpc" { tags = local.tags } -module "vpc_endpoints_sg" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 5.0" - - name = "${local.name}-vpc-endpoints" - description = "Security group for VPC endpoint access" - vpc_id = module.vpc.vpc_id - - ingress_with_cidr_blocks = [ - { - rule = "https-443-tcp" - description = "VPC CIDR HTTPS" - cidr_blocks = join(",", module.vpc.private_subnets_cidr_blocks) - }, - ] - - egress_with_cidr_blocks = [ - { - rule = "https-443-tcp" - description = "All egress HTTPS" - cidr_blocks = "0.0.0.0/0" - }, - ] - - tags = local.tags -} - module "vpc_endpoints" { source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 5.0" - - vpc_id = module.vpc.vpc_id - security_group_ids = [module.vpc_endpoints_sg.security_group_id] + version = "~> 5.1" + + vpc_id = module.vpc.vpc_id + + # Security group + create_security_group = true + security_group_name_prefix = "${local.name}-vpc-endpoints-" + security_group_description = "VPC endpoint security group" + security_group_rules = { + ingress_https = { + description = "HTTPS from VPC" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + } endpoints = merge({ s3 = { diff --git a/examples/istio/README.md b/examples/istio/README.md new file mode 100644 index 0000000000..45f0bb03bd --- /dev/null +++ b/examples/istio/README.md @@ -0,0 +1,267 @@ +# Amazon EKS Cluster w/ Istio + +This example shows how to provision an EKS cluster with Istio. + +* Deploy EKS Cluster with one managed node group in an VPC +* Add node_security_group rules for port access required for Istio communication +* Install Istio using Helm resources in Terraform +* Install Istio Ingress Gateway using Helm resources in Terraform +* Deploy/Validate Istio communication using sample application + +Refer to the [documentation](https://istio.io/latest/docs/concepts/) for `Istio` concepts. + +## Prerequisites: + +Ensure that you have the following tools installed locally: + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) + +## Deploy + +To provision this example: + +```sh +terraform init +terraform apply +``` + +Enter `yes` at command prompt to apply + +## Validate + +The following command will update the `kubeconfig` on your local machine and allow you to interact with your EKS Cluster using `kubectl` to validate the deployment. + +1. Run `update-kubeconfig` command: + +```sh +aws eks --region update-kubeconfig --name +``` + +2. List the nodes running currently + +```sh +kubectl get nodes +``` + +``` +# Output should look like below +NAME STATUS ROLES AGE VERSION +ip-10-0-22-173.ec2.internal Ready 48m v1.27.3-eks-a5565ad +``` + +3. List out the pods running currently: + +```sh +kubectl get pods,svc -n istio-system +``` + +``` +# Output should look like below +NAME READY STATUS RESTARTS AGE +pod/istio-ingress-6f7c5dffd8-chkww 1/1 Running 0 48m +pod/istiod-ff577f8b8-t9ww2 1/1 Running 0 48m + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/istio-ingress LoadBalancer 172.20.100.3 a59363808e78d46d59bf3378cafffcec-a12f9c78cb607b6b.elb.us-east-1.amazonaws.com 15021:32118/TCP,80:32740/TCP,443:30624/TCP 48m +service/istiod ClusterIP 172.20.249.63 15010/TCP,15012/TCP,443/TCP,15014/TCP 48m +``` + +4. Verify all the helm releases installed for Istio: + +```sh +helm list -n istio-system +``` + +``` +# Output should look like below +NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION +istio-base istio-system 1 2023-07-19 11:05:41.599921 -0700 PDT deployed base-1.18.1 1.18.1 +istio-ingress istio-system 1 2023-07-19 11:06:03.41609 -0700 PDT deployed gateway-1.18.1 1.18.1 +istiod istio-system 1 2023-07-19 11:05:48.087616 -0700 PDT deployed istiod-1.18.1 1.18.1 +``` + +## Test + +1. Create the sample namespace and enable the sidecar injection for this namespace + +```sh +kubectl create namespace sample +kubectl label namespace sample istio-injection=enabled +``` + +``` +namespace/sample created +namespace/sample labeled +``` + +2. Deploy helloworld app + +```sh +cat < helloworld.yaml +apiVersion: v1 +kind: Service +metadata: + name: helloworld + labels: + app: helloworld + service: helloworld +spec: + ports: + - port: 5000 + name: http + selector: + app: helloworld +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helloworld-v1 + labels: + app: helloworld + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: helloworld + version: v1 + template: + metadata: + labels: + app: helloworld + version: v1 + spec: + containers: + - name: helloworld + image: docker.io/istio/examples-helloworld-v1 + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent #Always + ports: + - containerPort: 5000 +EOF + +kubectl apply -f helloworld.yaml -n sample +``` + +``` +service/helloworld created +deployment.apps/helloworld-v1 created +``` + +3. Deploy sleep app that we will use to connect to helloworld app + +```sh +cat < sleep.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sleep +--- +apiVersion: v1 +kind: Service +metadata: + name: sleep + labels: + app: sleep + service: sleep +spec: + ports: + - port: 80 + name: http + selector: + app: sleep +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sleep +spec: + replicas: 1 + selector: + matchLabels: + app: sleepdocs/blueprints/argocd.md + template: + metadata: + labels: + app: sleep + spec: + terminationGracePeriodSeconds: 0 + serviceAccountName: sleep + containers: + - name: sleep + image: curlimages/curl + command: ["/bin/sleep", "infinity"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /etc/sleep/tls + name: secret-volume + volumes: + - name: secret-volume + secret: + secretName: sleep-secret + optional: true +EOF + +kubectl apply -f sleep.yaml -n sample +``` + +``` +serviceaccount/sleep created +service/sleep created +deployment.apps/sleep created +``` + +4. Check all the pods in the `sample` namespace + +```sh +kubectl get pods -n sample +``` +``` +NAME READY STATUS RESTARTS AGE +helloworld-v1-b6c45f55-bx2xk 2/2 Running 0 50s +sleep-9454cc476-p2zxr 2/2 Running 0 15s +``` +5. Connect to helloworld app from sleep app and see the connectivity is using envoy proxy + +```sh +kubectl exec -n sample -c sleep \ + "$(kubectl get pod -n sample -l \ + app=sleep -o jsonpath='{.items[0].metadata.name}')" \ + -- curl -v helloworld.sample:5000/hello +``` +``` +* processing: helloworld.sample:5000/hello + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 172.20.26.38:5000... +* Connected to helloworld.sample (172.20.26.38) port 5000 +> GET /hello HTTP/1.1 +> Host: helloworld.sample:5000 +> User-Agent: curl/8.2.0 +> Accept: */* +> +< HTTP/1.1 200 OK +< server: envoy +< date: Fri, 21 Jul 2023 18:56:09 GMT +< content-type: text/html; charset=utf-8 +< content-length: 58 +< x-envoy-upstream-service-time: 142 +< +{ [58 bytes data] +100 58 100 58 Hello version: v1, instance: helloworld-v1-b6c45f55-h592c + 0 0 392 0 --:--:-- --:--:-- --:--:-- 394 +* Connection #0 to host helloworld.sample left intact +``` + +## Destroy + +To teardown and remove the resources created in this example: + +```sh +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -auto-approve +``` diff --git a/examples/istio/main.tf b/examples/istio/main.tf new file mode 100644 index 0000000000..9d0096ca8c --- /dev/null +++ b/examples/istio/main.tf @@ -0,0 +1,217 @@ +provider "aws" { + region = local.region +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} + +data "aws_availability_zones" "available" {} + +locals { + name = basename(path.cwd) + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + istio_chart_url = "https://istio-release.storage.googleapis.com/charts" + istio_chart_version = "1.18.1" + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +################################################################################ +# Cluster +################################################################################ + +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.15" + + cluster_name = local.name + cluster_version = "1.27" + cluster_endpoint_public_access = true + + # EKS Addons + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["m5.large"] + + min_size = 1 + max_size = 5 + desired_size = 2 + } + } + + # EKS K8s API cluster needs to be able to talk with the EKS worker nodes with port 15017/TCP and 15012/TCP which is used by Istio + # Istio in order to create sidecar needs to be able to communicate with webhook and for that network passage to EKS is needed. + node_security_group_additional_rules = { + ingress_15017 = { + description = "Cluster API - Istio Webhook namespace.sidecar-injector.istio.io" + protocol = "TCP" + from_port = 15017 + to_port = 15017 + type = "ingress" + source_cluster_security_group = true + } + ingress_15012 = { + description = "Cluster API to nodes ports/protocols" + protocol = "TCP" + from_port = 15012 + to_port = 15012 + type = "ingress" + source_cluster_security_group = true + } + } + + tags = local.tags +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # This is required to expose Istio Ingress Gateway + enable_aws_load_balancer_controller = true + + tags = local.tags +} + +################################################################################ +# Istio +################################################################################ + +resource "kubernetes_namespace" "istio_system" { + metadata { + name = "istio-system" + labels = { + istio-injection = "enabled" + } + } +} + +resource "helm_release" "istio_base" { + repository = local.istio_chart_url + chart = "base" + name = "istio-base" + namespace = kubernetes_namespace.istio_system.metadata[0].name + version = local.istio_chart_version + wait = false + + depends_on = [ + module.eks_blueprints_addons + ] +} + +resource "helm_release" "istiod" { + repository = local.istio_chart_url + chart = "istiod" + name = "istiod" + namespace = helm_release.istio_base.metadata[0].namespace + version = local.istio_chart_version + wait = false + + set { + name = "meshConfig.accessLogFile" + value = "/dev/stdout" + } +} + +resource "helm_release" "istio_ingress" { + repository = local.istio_chart_url + chart = "gateway" + name = "istio-ingress" + namespace = helm_release.istiod.metadata[0].namespace + version = local.istio_chart_version + wait = false + + values = [ + yamlencode( + { + labels = { + istio = "ingressgateway" + } + service = { + annotations = { + "service.beta.kubernetes.io/aws-load-balancer-type" = "nlb" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + } + } + } + ) + ] +} + +################################################################################ +# Supporting Resources +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/istio/outputs.tf b/examples/istio/outputs.tf new file mode 100644 index 0000000000..c624023e90 --- /dev/null +++ b/examples/istio/outputs.tf @@ -0,0 +1,4 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" +} diff --git a/examples/istio/variables.tf b/examples/istio/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/istio/versions.tf b/examples/istio/versions.tf new file mode 100644 index 0000000000..20c550ab61 --- /dev/null +++ b/examples/istio/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.47" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.9" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/istio/terraform.tfstate" + # } +} diff --git a/examples/multi-tenancy-with-teams/main.tf b/examples/multi-tenancy-with-teams/main.tf index 37bc47777f..fbc22b96f0 100644 --- a/examples/multi-tenancy-with-teams/main.tf +++ b/examples/multi-tenancy-with-teams/main.tf @@ -126,10 +126,10 @@ module "eks_blueprints_dev_teams" { } namespaces = { - "blue-${each.key}" = { + "team-${each.key}" = { labels = { - appName = "blue-team-app", - projectName = "project-blue", + appName = "${each.key}-team-app", + projectName = "project-${each.key}", } resource_quota = { diff --git a/examples/privatelink-access/README.md b/examples/privatelink-access/README.md new file mode 100644 index 0000000000..30d2c1d285 --- /dev/null +++ b/examples/privatelink-access/README.md @@ -0,0 +1,140 @@ +# Private EKS cluster access via AWS PrivateLink + +This example demonstrates how to access a private EKS cluster using AWS PrivateLink. + +Refer to the [documentation](https://docs.aws.amazon.com/vpc/latest/privatelink/concepts.html) +for further details on `AWS PrivateLink`. + +## Prerequisites: + +Ensure that you have the following tools installed locally: + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) + +## Deploy + +To provision this example, first deploy the Lambda function that responds to +`CreateNetworkInterface` API calls. This needs to exist before the cluster is +created so that it can respond to the ENIs created by the EKS control plane: + +```sh +terraform init +terraform apply -target=module.create_eni_lambda -target=module.nlb +``` + +Enter `yes` at command prompt to apply + +Next, deploy the remaining resources: + +```sh +terraform apply +``` + +Enter `yes` at command prompt to apply + +## Validate + +### Network Connectivity + +An output `ssm_test` has been provided to aid in quickly testing the +connectivity from the client EC2 instance to the private EKS cluster via AWS +PrivateLink. Copy the output value and paste it into your terminal to execute +and check the connectivity. If configured correctly, the value returned should +be `ok`. + +```sh +COMMAND="curl -ks https://9A85B21811733524E3ABCDFEA8714642.gr7.us-west-2.eks.amazonaws.com/readyz" + +COMMAND_ID=$(aws ssm send-command --region us-west-2 \ +--document-name "AWS-RunShellScript" \ +--parameters "commands=[$COMMAND]" \ +--targets "Key=instanceids,Values=i-0a45eff73ba408575" \ +--query 'Command.CommandId' \ +--output text) + +aws ssm get-command-invocation --region us-west-2 \ +--command-id $COMMAND_ID \ +--instance-id i-0a45eff73ba408575 \ +--query 'StandardOutputContent' \ +--output text +``` + +### Cluster Access + +To test access to the cluster, you will need to execute Kubernetes API calls +from within the private network to access the cluster. An EC2 instance has been +deployed into a "client" VPC to simulate this scenario. However, since the EKS +cluster was created with your local IAM identity, the `aws-auth` ConfigMap will +only have your local identity that is permitted to access the cluster. Since +cluster's API endpoint is private, we cannot use Terraform to reach it to +add additional entries to the ConfigMap; we can only access the cluster from +within the private network of the cluster's VPC or from the client VPC using AWS +PrivateLink access. + +> :warning: The "client" EC2 instance provided and copying of AWS credentials to + that instance are merely for demonstration purposes only. Please consider + alternate methods of network access such as AWS Client VPN to provide more + secure access. + +Perform the following steps to access the cluster with `kubectl` from the +provided "client" EC2 instance. + +1. Execute the command below on your local machine to get temporary credentials +that will be used on the "client" EC2 instance: + + ```sh + aws sts get-session-token --duration-seconds 3600 --output yaml + ``` + +2. Start a new SSM session on the "client" EC2 instance using the provided +`ssm_start_session` output value. Copy the output value and paste it into your +terminal to execute. Your terminal will now be connected to the "client" EC2 +instance. + + ```sh + aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44 + ``` + +3. Once logged in, export the following environment variables from the output +of step #1: + + > :exclamation: The session credentials are only valid for 1 hour; you can + adjust the session duration in the command provided in step #1 + + ```sh + export AWS_ACCESS_KEY_ID=XXXX + export AWS_SECRET_ACCESS_KEY=YYYY + export AWS_SESSION_TOKEN=ZZZZ + ``` + +4. Run the following command to update the local `~/.kube/config` file to enable +access to the cluster: + + ```sh + aws eks update-kubeconfig --region us-west-2 --name privatelink-access + ``` + +5. Test access by listing the pods running on the cluster: + + ```sh + kubectl get pods -A + ``` + + The test succeeded if you see an output like the one shown below: + + NAMESPACE NAME READY STATUS RESTARTS AGE + kube-system aws-node-4f8g8 1/1 Running 0 1m + kube-system coredns-6ff9c46cd8-59sqp 1/1 Running 0 1m + kube-system coredns-6ff9c46cd8-svnpb 1/1 Running 0 2m + kube-system kube-proxy-mm2zc 1/1 Running 0 1m + + +## Destroy + +Run the following command to destroy all the resources created by Terraform: + +```sh +terraform destroy --auto-approve +``` diff --git a/examples/privatelink-access/client.tf b/examples/privatelink-access/client.tf new file mode 100644 index 0000000000..0793c4804f --- /dev/null +++ b/examples/privatelink-access/client.tf @@ -0,0 +1,88 @@ +# The resources defined in this file are only used for demonstrating private connectivity +# They are not required for the solution + +locals { + client_name = "${local.name}-client" +} + +################################################################################ +# VPC +################################################################################ + +module "client_vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.client_name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k + 10)] + + enable_nat_gateway = true + single_nat_gateway = true + + manage_default_network_acl = true + default_network_acl_tags = { Name = "${local.client_name}-default" } + manage_default_route_table = true + default_route_table_tags = { Name = "${local.client_name}-default" } + manage_default_security_group = true + default_security_group_tags = { Name = "${local.client_name}-default" } + + tags = local.tags +} + +################################################################################ +# EC2 Instance +################################################################################ + +module "client_ec2_instance" { + source = "terraform-aws-modules/ec2-instance/aws" + + name = local.client_name + + create_iam_instance_profile = true + iam_role_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + vpc_security_group_ids = [module.client_security_group.security_group_id] + subnet_id = element(module.client_vpc.private_subnets, 0) + + user_data = <<-EOT + #!/bin/bash + + # Install kubectl + curl -LO https://dl.k8s.io/release/v${module.eks.cluster_version}.0/bin/linux/amd64/kubectl + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + + # Remove default awscli which is v1 - we want latest v2 + yum remove awscli -y + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -qq awscliv2.zip + ./aws/install + EOT + + tags = local.tags +} + +module "client_security_group" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 5.0" + + name = local.client_name + description = "Security group for SSM access to private cluster" + vpc_id = module.client_vpc.vpc_id + + egress_with_cidr_blocks = [ + { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = "0.0.0.0/0" + }, + ] + + tags = local.tags +} diff --git a/examples/privatelink-access/eks.tf b/examples/privatelink-access/eks.tf new file mode 100644 index 0000000000..2536c8b79c --- /dev/null +++ b/examples/privatelink-access/eks.tf @@ -0,0 +1,121 @@ +################################################################################ +# EKS Cluster +################################################################################ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.15" + + cluster_name = local.name + cluster_version = "1.27" + + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = {} + } + + cluster_security_group_additional_rules = { + # Allow tcp/443 from the NLB IP addresses + for ip_addr in data.dns_a_record_set.nlb.addrs : "nlb_ingress_${replace(ip_addr, ".", "")}" => { + description = "Allow ingress from NLB" + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["${ip_addr}/32"] + } + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + default = { + instance_types = ["c5.large"] + + min_size = 1 + max_size = 3 + desired_size = 1 + } + } + + tags = local.tags +} + +################################################################################ +# VPC +################################################################################ + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + + # Disable NAT gateway for fully private networking + enable_nat_gateway = false + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} + +# Explicitly create a Internet Gateway here in the Private EKS VPC as without an +# internet gateway, a NLB cannot be created. Config option of create_igw = true +# (default) did not work during the VPC creation as it requires public subnets +# and the related routes that connect them to IGW +resource "aws_internet_gateway" "igw" { + vpc_id = module.vpc.vpc_id + tags = local.tags +} + +################################################################################ +# VPC Endpoints +################################################################################ + +module "vpc_endpoints" { + source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" + version = "~> 5.1" + + vpc_id = module.vpc.vpc_id + + # Security group + create_security_group = true + security_group_name_prefix = "${local.name}-vpc-endpoints-" + security_group_description = "VPC endpoint security group" + security_group_rules = { + ingress_https = { + description = "HTTPS from VPC" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + } + + endpoints = merge({ + s3 = { + service = "s3" + service_type = "Gateway" + route_table_ids = module.vpc.private_route_table_ids + tags = { + Name = "${local.name}-s3" + } + } + }, + { for service in toset(["autoscaling", "ecr.api", "ecr.dkr", "ec2", "ec2messages", "elasticloadbalancing", "sts", "kms", "logs", "ssm", "ssmmessages"]) : + replace(service, ".", "_") => + { + service = service + subnet_ids = module.vpc.private_subnets + private_dns_enabled = true + tags = { Name = "${local.name}-${service}" } + } + }) + + tags = local.tags +} diff --git a/examples/privatelink-access/lambdas/create_eni.py b/examples/privatelink-access/lambdas/create_eni.py new file mode 100644 index 0000000000..66f23908fb --- /dev/null +++ b/examples/privatelink-access/lambdas/create_eni.py @@ -0,0 +1,40 @@ +import boto3 +import logging +import os +import json + +ELBV2_CLIENT = boto3.client('elbv2') + +TARGET_GROUP_ARN = os.environ['TARGET_GROUP_ARN'] + + +class StructuredMessage: + def __init__(self, message, /, **kwargs): + self.message = message + self.kwargs = kwargs + + def __str__(self): + return '%s >>> %s' % (self.message, json.dumps(self.kwargs)) + +_ = StructuredMessage # optional, to improve readability +logging.basicConfig(level=logging.DEBUG, format='%(message)s') + + +def handler(event, context): + # Only modify on CreateNetworkInterface events + if event["detail"]["eventName"] == "CreateNetworkInterface": + ip = event['detail']['responseElements']['networkInterface']['privateIpAddress'] + + # Add the extracted private IP address of the ENI as an IP target in the target group + try: + response = ELBV2_CLIENT.register_targets( + TargetGroupArn = TARGET_GROUP_ARN, + Targets=[{ + 'Id': ip, + 'Port': 443 + }] + ) + logging.info(_(response)) + except Exception as e: + logging.error(_(e)) + raise(e) diff --git a/examples/privatelink-access/lambdas/delete_eni.py b/examples/privatelink-access/lambdas/delete_eni.py new file mode 100644 index 0000000000..206f502790 --- /dev/null +++ b/examples/privatelink-access/lambdas/delete_eni.py @@ -0,0 +1,87 @@ +import boto3 +import logging +import os +import json + +ELBV2_CLIENT = boto3.client('elbv2') +EC2_CLIENT = boto3.client('ec2') + +TARGET_GROUP_ARN = os.environ['TARGET_GROUP_ARN'] + + +class StructuredMessage: + def __init__(self, message, /, **kwargs): + self.message = message + self.kwargs = kwargs + + def __str__(self): + return '%s >>> %s' % (self.message, json.dumps(self.kwargs)) + +_ = StructuredMessage # optional, to improve readability +logging.basicConfig(level=logging.DEBUG, format='%(message)s') + + +def handler(event, context): + + unhealthyTargetIPAddresses = [] + eksApiEndpointEniIPAddresses = [] + unhealthyTargetsToDeregister = [] + + targetHealthDescriptions = ELBV2_CLIENT.describe_target_health( + TargetGroupArn=TARGET_GROUP_ARN, + )['TargetHealthDescriptions'] + + if not targetHealthDescriptions: + logging.info("Did not find any TargetHealthDescriptions, quitting!") + return + + # Iterate over the list of TargetHealthDescriptions and extract the list of + # unhealthy targets + for targetHealthDescription in targetHealthDescriptions: + if targetHealthDescription["TargetHealth"]["State"] == "unhealthy": + unhealthyTargetIPAddress = targetHealthDescription["Target"]["Id"] + unhealthyTargetIPAddresses.append(unhealthyTargetIPAddress) + + networkInterfaces = EC2_CLIENT.describe_network_interfaces( + Filters=[ + { + 'Name': 'description', + 'Values': [ + f'Amazon EKS {os.environ["EKS_CLUSTER_NAME"]}', + ] + }, + ], + )['NetworkInterfaces'] + + if not networkInterfaces: + logging.info("Did not find any EKS API ENIs to compare with, quitting!") + return + + for networkInterface in networkInterfaces: + eksApiEndpointEniIPAddresses.append( + networkInterface["PrivateIpAddress"] + ) + + for unhealthyTargetIPAddress in unhealthyTargetIPAddresses: + if unhealthyTargetIPAddress not in eksApiEndpointEniIPAddresses: + unhealthyTarget = { + 'Id': unhealthyTargetIPAddress, + 'Port': 443 + } + unhealthyTargetsToDeregister.append(unhealthyTarget) + + if not unhealthyTargetsToDeregister: + logging.info("There are no unhealthy targets to deregister, quitting!") + return + + logging.info("Targets are to be deregistered: %s", unhealthyTargetsToDeregister) + + try: + response = ELBV2_CLIENT.deregister_targets( + TargetGroupArn = TARGET_GROUP_ARN, + Targets=unhealthyTargetsToDeregister + ) + logging.info(_(response)) + except Exception as e: + logging.error(_(e)) + raise(e) diff --git a/examples/privatelink-access/main.tf b/examples/privatelink-access/main.tf new file mode 100644 index 0000000000..62af1323cf --- /dev/null +++ b/examples/privatelink-access/main.tf @@ -0,0 +1,19 @@ +provider "aws" { + region = local.region +} + +data "aws_availability_zones" "available" {} + +locals { + name = basename(path.cwd) + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Name = local.name + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} diff --git a/examples/privatelink-access/outputs.tf b/examples/privatelink-access/outputs.tf new file mode 100644 index 0000000000..3130bf693f --- /dev/null +++ b/examples/privatelink-access/outputs.tf @@ -0,0 +1,24 @@ +output "ssm_start_session" { + description = "SSM start session command to connect to remote host created" + value = "aws ssm start-session --region ${local.region} --target ${module.client_ec2_instance.id}" +} + +output "ssm_test" { + description = "SSM commands to test connectivity from client EC2 instance to the private EKS cluster" + value = <<-EOT + COMMAND="curl -ks ${module.eks.cluster_endpoint}/readyz" + + COMMAND_ID=$(aws ssm send-command --region ${local.region} \ + --document-name "AWS-RunShellScript" \ + --parameters "commands=[$COMMAND]" \ + --targets "Key=instanceids,Values=${module.client_ec2_instance.id}" \ + --query 'Command.CommandId' \ + --output text) + + aws ssm get-command-invocation --region ${local.region} \ + --command-id $COMMAND_ID \ + --instance-id ${module.client_ec2_instance.id} \ + --query 'StandardOutputContent' \ + --output text + EOT +} diff --git a/examples/privatelink-access/privatelink.tf b/examples/privatelink-access/privatelink.tf new file mode 100644 index 0000000000..33588087d6 --- /dev/null +++ b/examples/privatelink-access/privatelink.tf @@ -0,0 +1,286 @@ +################################################################################ +# Internal NLB +################################################################################ + +module "nlb" { + source = "terraform-aws-modules/alb/aws" + version = "~> 8.6" + + name = local.name + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + internal = true + load_balancer_type = "network" + + target_groups = [{ + name = local.name + backend_protocol = "TCP" + backend_port = 443 + target_type = "ip" + health_check = { + enabled = true + path = "/readyz" + protocol = "HTTPS" + matcher = "200" + } + }] + + http_tcp_listeners = [{ + port = 443 + protocol = "TCP" + target_group_index = 0 + }] + + tags = local.tags +} + +data "dns_a_record_set" "nlb" { + host = module.nlb.lb_dns_name +} + +# VPC Endpoint Service that can be shared with other services in other VPCs. +# This Service Endpoint is created in the VPC where the LB exists; the client +# VPC Endpoint will connect to this service to reach the cluster via AWS PrivateLink +resource "aws_vpc_endpoint_service" "this" { + acceptance_required = true + network_load_balancer_arns = [module.nlb.lb_arn] + + tags = merge(local.tags, + { Name = local.name }, + ) +} + +resource "aws_vpc_endpoint_connection_accepter" "this" { + vpc_endpoint_service_id = aws_vpc_endpoint_service.this.id + vpc_endpoint_id = aws_vpc_endpoint.client.id +} + +################################################################################ +# VPC Endpoint +# This allows resources in the client VPC to connect to the EKS cluster API +# endpoint in the EKS VPC without going over the internet, using a VPC peering +# connection, or a transit gateway attachment between VPCs +################################################################################ + +locals { + # Get patterns for subdomain name (index 1) and domain name (index 2) + api_server_url_pattern = regex("(https://)([[:alnum:]]+\\.)(.*)", module.eks.cluster_endpoint) + + # Retrieve the subdomain and domain name of the API server endpoint URL + cluster_endpoint_subdomain = local.api_server_url_pattern[1] + cluster_endpoint_domain = local.api_server_url_pattern[2] +} + +resource "aws_vpc_endpoint" "client" { + vpc_id = module.client_vpc.vpc_id + service_name = resource.aws_vpc_endpoint_service.this.service_name + vpc_endpoint_type = "Interface" + subnet_ids = module.client_vpc.private_subnets + security_group_ids = [resource.aws_security_group.client_vpc_endpoint.id] + + tags = merge(local.tags, + { Name = local.name }, + ) +} + +# Create a new security group that allows TLS traffic on port 443 and let the +# client applications in Client VPC connect to the VPC Endpoint on port 443 +resource "aws_security_group" "client_vpc_endpoint" { + name_prefix = "${local.name}-" + description = "Allow TLS inbound traffic" + vpc_id = module.client_vpc.vpc_id + + ingress { + description = "TLS from VPC" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = [module.client_vpc.vpc_cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = local.tags +} + +resource "aws_route53_zone" "client" { + name = local.cluster_endpoint_domain + comment = "Private hosted zone for EKS API server endpoint" + + vpc { + vpc_id = module.client_vpc.vpc_id + } + + tags = local.tags +} + +resource "aws_route53_record" "client" { + zone_id = resource.aws_route53_zone.client.zone_id + name = "${local.cluster_endpoint_subdomain}${local.cluster_endpoint_domain}" + type = "A" + + alias { + name = resource.aws_vpc_endpoint.client.dns_entry[0].dns_name + zone_id = resource.aws_vpc_endpoint.client.dns_entry[0].hosted_zone_id + evaluate_target_health = true + } +} + +################################################################################ +# Lambda - Create ENI IPs to NLB Target Group +################################################################################ + +module "create_eni_lambda" { + source = "terraform-aws-modules/lambda/aws" + version = "~> 5.0" + + function_name = "${local.name}-add-eni-ips" + description = "Add ENI IPs to NLB target group when EKS API endpoint is created" + handler = "create_eni.handler" + runtime = "python3.10" + publish = true + source_path = "lambdas" + + attach_policy_json = true + policy_json = <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets" + ], + "Resource": ["${module.nlb.target_group_arns[0]}"] + } + ] + } + EOT + + environment_variables = { + TARGET_GROUP_ARN = module.nlb.target_group_arns[0] + } + + allowed_triggers = { + eventbridge = { + principal = "events.amazonaws.com" + source_arn = module.eventbridge.eventbridge_rule_arns["eks-api-endpoint-create"] + } + } + + tags = local.tags +} + +################################################################################ +# Lambda - Delete ENI IPs from NLB Target Group +################################################################################ + +module "delete_eni_lambda" { + source = "terraform-aws-modules/lambda/aws" + version = "~> 5.0" + + function_name = "${local.name}-delete-eni-ips" + description = "Deletes ENI IPs from NLB target group when EKS API endpoint is deleted" + handler = "delete_eni.handler" + runtime = "python3.10" + publish = true + source_path = "lambdas" + + attach_policy_json = true + policy_json = <<-EOT + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeNetworkInterfaces", + "elasticloadbalancing:Describe*" + ], + "Resource": ["*"] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": ["${module.nlb.target_group_arns[0]}"] + } + ] + } + EOT + + environment_variables = { + TARGET_GROUP_ARN = module.nlb.target_group_arns[0] + EKS_CLUSTER_NAME = module.eks.cluster_name + } + + allowed_triggers = { + eventbridge = { + principal = "events.amazonaws.com" + source_arn = module.eventbridge.eventbridge_rule_arns["eks-api-endpoint-delete"] + } + } + + tags = local.tags +} + +################################################################################ +# EventBridge Rules +################################################################################ + +module "eventbridge" { + source = "terraform-aws-modules/eventbridge/aws" + version = "~> 2.0" + + # Use the existing default event bus + create_bus = false + + rules = { + eks-api-endpoint-create = { + event_pattern = jsonencode({ + "source" : ["aws.ec2"], + "detail-type" : ["AWS API Call via CloudTrail"], + "detail" : { + "eventSource" : ["ec2.amazonaws.com"], + "eventName" : ["CreateNetworkInterface"], + "sourceIPAddress" : ["eks.amazonaws.com"], + "responseElements" : { + "networkInterface" : { + "description" : ["Amazon EKS ${local.name}"] + } + } + } + }) + enabled = true + } + + eks-api-endpoint-delete = { + description = "Trigger for a Lambda" + schedule_expression = "rate(15 minutes)" + } + } + + targets = { + eks-api-endpoint-create = [ + { + name = module.create_eni_lambda.lambda_function_name + arn = module.create_eni_lambda.lambda_function_arn + } + ] + eks-api-endpoint-delete = [ + { + name = module.delete_eni_lambda.lambda_function_name + arn = module.delete_eni_lambda.lambda_function_arn + } + ] + } + + tags = local.tags +} diff --git a/examples/privatelink-access/variables.tf b/examples/privatelink-access/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/privatelink-access/versions.tf b/examples/privatelink-access/versions.tf new file mode 100644 index 0000000000..4cbe90687e --- /dev/null +++ b/examples/privatelink-access/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + dns = { + source = "hashicorp/dns" + version = ">= 3.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/privatelink-access/terraform.tfstate" + # } +} diff --git a/examples/vpc-cni-custom-networking/main.tf b/examples/vpc-cni-custom-networking/main.tf index 07bc0e2c54..04b02d234a 100644 --- a/examples/vpc-cni-custom-networking/main.tf +++ b/examples/vpc-cni-custom-networking/main.tf @@ -127,7 +127,6 @@ resource "kubectl_manifest" "eni_config" { } spec = { securityGroups = [ - module.eks.cluster_primary_security_group_id, module.eks.node_security_group_id, ] subnet = each.value