Skip to content

Commit

Permalink
fix: ECR pull through cache pattern (#1980)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rodrigobersa
rodrigobersa authored Aug 4, 2024
1 parent afd006d commit 0921b88
Show file tree
Hide file tree
Showing 12 changed files with 143 additions and 343 deletions.
29 changes: 21 additions & 8 deletions patterns/ecr-pull-through-cache/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,25 @@
This pattern demonstrates how to set up ECR cache pull-through for public images. The Terraform code creates four cache pull-through rules for public image repositories: Docker, Kubernetes, Quay, and ECR. It also configures basic scanning on push for all repositories and includes a creation template. Additionally, it configures the EC2 node role with permissions to pull through images. The setup then installs ALB Controller, Metrics Server, Gatekeeper, ArgoCD, and Prometheus Operator, with their respective Helm charts configured in the values files to pull images through the pull-through cache.

## Deploy

Follow the instructions [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and steps to deploy this pattern.

```
```sh
terraform init
terraform apply -var='docker_secret={"username":"your-docker-username", "accessToken":"your-docker-password"}'
```

## Validate

Validate the pull trough cache rules connectivity:
```

```sh
for i in docker-hub ecr k8s quay ; do aws ecr validate-pull-through-cache-rule --ecr-repository-prefix $i --region us-east-1; done
```

Expected output:
```

```json
{
"ecrRepositoryPrefix": "docker-hub",
"registryId": "111122223333",
Expand Down Expand Up @@ -47,12 +52,16 @@ Expected output:
"failure": ""
}
```

Validate pods are pulling the images and in Running state:
```

```sh
kubectl get pods -A
```

Expected output:
```

```sh
NAMESPACE NAME READY STATUS RESTARTS AGE
argocd argo-cd-argocd-application-controller-0 1/1 Running 0 2m26s
argocd argo-cd-argocd-applicationset-controller-78ccd75cfb-7zfs5 1/1 Running 0 2m28s
Expand Down Expand Up @@ -97,11 +106,15 @@ kube-system metrics-server-5d6489d58d-pbrxv
```

## Destroy

ECR repositories are automatically created via pull through cache and can be deleted using the following command.
NOTE: This commands deletes all the ecr repositories in a region.
```

> NOTE: This commands deletes all the ECR repositories in the specified region.
```sh
for REPO in $(aws ecr describe-repositories --query 'repositories[].repositoryName' --output text); do aws ecr delete-repository --repository-name $REPO --force ; done
```
```

{%
include-markdown "../../docs/_partials/destroy.md"
%}
131 changes: 94 additions & 37 deletions patterns/ecr-pull-through-cache/addons.tf
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
locals {
ecr_url = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${local.region}.amazonaws.com"
}

module "eks_blueprints_addons" {
source = "aws-ia/eks-blueprints-addons/aws"
version = "~> 1.16"
Expand All @@ -9,56 +13,92 @@ module "eks_blueprints_addons" {

enable_argocd = true
argocd = {
namespace = "argocd"
chart_version = "7.1.0" # ArgoCD v2.11.2
values = [
templatefile("${path.module}/values/argocd.yaml", {
ecr_account_id = local.ecr_account_id
ecr_region = local.ecr_region
})]
values = [<<-EOT
global:
image:
repository: ${local.ecr_url}/quay/argoproj/argocd
dex:
image:
repository: ${local.ecr_url}/docker-hub/dexidp/dex
haproxy:
image:
repository: ${local.ecr_url}/docker-hub/library/haproxy
EOT
]
}

enable_metrics_server = true
metrics_server = {
values = [
templatefile("${path.module}/values/metrics-server.yaml", {
ecr_account_id = local.ecr_account_id
ecr_region = local.ecr_region
})]
set = [{
name = "image.repository"
value = "${local.ecr_url}/k8s/metrics-server/metrics-server"
}
]
}

enable_aws_load_balancer_controller = true
aws_load_balancer_controller = {
values = [
templatefile("${path.module}/values/aws-load-balancer-controller.yaml", {
ecr_account_id = local.ecr_account_id
ecr_region = local.ecr_region
})]
set = [{
name = "image.repository"
value = "${local.ecr_url}/ecr/eks/aws-load-balancer-controller"
}
]
}

enable_kube_prometheus_stack = true
kube_prometheus_stack = {
values = [
templatefile("${path.module}/values/prometheus.yaml", {
ecr_account_id = local.ecr_account_id
ecr_region = local.ecr_region
})]
values = [<<-EOT
global:
imageRegistry: ${local.ecr_url}
prometheusOperator:
image:
repository: quay/prometheus-operator/prometheus-operator
admissionWebhooks:
deployment:
image:
repository: quay/prometheus-operator/admission-webhook
patch:
image:
repository: k8s/ingress-nginx/kube-webhook-certgen
prometheusConfigReloader:
image:
repository: quay/prometheus-operator/prometheus-config-reloader
alertmanager:
alertmanagerSpec:
image:
repository: quay/prometheus/alertmanager
prometheus:
prometheusSpec:
image:
repository: quay/prometheus/prometheus
prometheus-node-exporter:
image:
repository: quay/prometheus/node-exporter
kube-state-metrics:
image:
repository: k8s/kube-state-metrics/kube-state-metrics
grafana:
global:
imageRegistry: ${local.ecr_url}
downloadDashboardsImage:
repository: ${local.ecr_url}/docker-hub/curlimages/curl
image:
repository: ${local.ecr_url}/quay/grafana/grafana
imageRenderer:
repository: ${local.ecr_url}/quay/grafana/grafana-image-renderer
sidecar:
image:
repository: ${local.ecr_url}/quay/kiwigrid/k8s-sidecar
EOT
]
}

depends_on = [module.eks.cluster_addons]
}

resource "time_sleep" "addons_wait_60_seconds" {
create_duration = "60s"
depends_on = [module.eks_blueprints_addons]
}

#---------------------------------------------------------------
# Gatekeeper
#---------------------------------------------------------------
module "gatekeeper" {
source = "aws-ia/eks-blueprints-addon/aws"
version = "1.1.1"
version = "~> 1.1"

name = "gatekeeper"
description = "A Helm chart to deploy gatekeeper project"
Expand All @@ -67,11 +107,28 @@ module "gatekeeper" {
chart = "gatekeeper"
chart_version = "3.16.3"
repository = "https://open-policy-agent.github.io/gatekeeper/charts"
values = [
templatefile("${path.module}/values/gatekeeper.yaml", {
ecr_account_id = local.ecr_account_id
ecr_region = local.ecr_region
})]
values = [<<-EOT
image:
repository: ${local.ecr_url}/docker-hub/openpolicyagent/gatekeeper
crdRepository: ${local.ecr_url}/docker-hub/openpolicyagent/gatekeeper-crds
postUpgrade:
labelNamespace:
image:
repository: ${local.ecr_url}/docker-hub/openpolicyagent/gatekeeper-crds
postInstall:
labelNamespace:
image:
repository: ${local.ecr_url}/docker-hub/openpolicyagent/gatekeeper-crds
probeWebhook:
image:
repository: ${local.ecr_url}/docker-hub/curlimages/curl
preUninstall:
deleteWebhookConfigurations:
image:
repository: ${local.ecr_url}/docker-hub/openpolicyagent/gatekeeper-crds
EOT
]

depends_on = [time_sleep.addons_wait_60_seconds]
depends_on = [module.eks.cluster_addons]
}
20 changes: 8 additions & 12 deletions patterns/ecr-pull-through-cache/ecr.tf
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
locals {
ecr_account_id = var.ecr_account_id != "" ? var.ecr_account_id : data.aws_caller_identity.current.account_id
ecr_region = var.ecr_region != "" ? var.ecr_region : local.region
}

module "secrets-manager" {
module "secrets_manager" {
source = "terraform-aws-modules/secrets-manager/aws"
version = "1.1.2"
version = "~> 1.1"

name = "ecr-pullthroughcache/docker"
secret_string = jsonencode(var.docker_secret)
name = "ecr-pullthroughcache/docker"
secret_string = jsonencode(var.docker_secret)
recovery_window_in_days = 0 # Set to 0 for testing purposes, this will immediatly delete the secret. This action is irreversible. https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_DeleteSecret.html
}

module "ecr" {
source = "terraform-aws-modules/ecr/aws"
version = "2.2.1"
version = "~> 2.2"

create_repository = false

Expand All @@ -33,12 +29,12 @@ module "ecr" {
dockerhub = {
ecr_repository_prefix = "docker-hub"
upstream_registry_url = "registry-1.docker.io"
credential_arn = module.secrets-manager.secret_arn
credential_arn = module.secrets_manager.secret_arn
}
}

manage_registry_scanning_configuration = true
registry_scan_type = "BASIC"
registry_scan_type = "ENHANCED"
registry_scan_rules = [
{
scan_frequency = "SCAN_ON_PUSH"
Expand Down
20 changes: 10 additions & 10 deletions patterns/ecr-pull-through-cache/eks.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

module "eks" {
source = "terraform-aws-modules/eks/aws"
version = "~> 20.11"
version = "~> 20.20"

cluster_name = local.name
cluster_version = local.cluster_version
Expand Down Expand Up @@ -36,12 +36,14 @@ module "eks" {

eks_managed_node_groups = {
initial = {
instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"]
capacity_type = var.capacity_type # defaults to SPOT
min_size = 1
max_size = 5
desired_size = 3
subnet_ids = module.vpc.private_subnets
instance_types = ["m5.large", "m5a.large"]

min_size = 1
max_size = 5
desired_size = 3

subnet_ids = module.vpc.private_subnets

iam_role_additional_policies = {
AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
additional = aws_iam_policy.ecrpullthroughcache.arn
Expand All @@ -54,9 +56,8 @@ module "eks" {
depends_on = [module.vpc]
}


resource "aws_iam_policy" "ecrpullthroughcache" {
name = "ECRPullThroughCache"
name = "ECRPullThroughCache"

policy = jsonencode({
Version = "2012-10-17"
Expand Down Expand Up @@ -102,4 +103,3 @@ output "configure_kubectl" {
description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig"
value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias ${module.eks.cluster_name}"
}

16 changes: 8 additions & 8 deletions patterns/ecr-pull-through-cache/main.tf
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

terraform {
required_version = ">= 1.8"

Expand All @@ -11,10 +8,13 @@ terraform {
}
helm = {
source = "hashicorp/helm"
version = ">= 2.13"
version = ">= 2.9"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = ">= 2.20"
}
}

}

provider "aws" {
Expand Down Expand Up @@ -51,10 +51,10 @@ data "aws_caller_identity" "current" {}
data "aws_availability_zones" "available" {}

locals {
name = var.name != "" ? var.name : "${basename(path.cwd)}"
region = var.region
name = basename(path.cwd)
region = "us-west-2"

cluster_version = var.cluster_version
cluster_version = "1.30"

vpc_cidr = "10.0.0.0/16"
azs = slice(data.aws_availability_zones.available.names, 0, 3)
Expand Down
Loading

0 comments on commit 0921b88

Please sign in to comment.