diff --git a/.github/scripts/plan-examples.py b/.github/scripts/plan-examples.py index 2be720a15b..e05d8b1a9d 100644 --- a/.github/scripts/plan-examples.py +++ b/.github/scripts/plan-examples.py @@ -3,23 +3,23 @@ import re -def get_examples(): +def get_patterns(): """ - Get all Terraform example root directories using their respective `versions.tf`; + Get all pattern root directories using their respective `main.tf`; returning a string formatted json array of the example directories minus those that are excluded """ exclude = { - 'examples/appmesh-mtls', # excluded until Rout53 is setup - 'examples/blue-green-upgrade/environment', - 'examples/blue-green-upgrade/modules/eks_cluster', - 'examples/istio-multi-cluster/1.cluster1', # relies on remote state - 'examples/istio-multi-cluster/2.cluster2', # relies on remote state - 'examples/privatelink-access', + 'patterns/appmesh-mtls', # excluded until Rout53 is setup + 'patterns/blue-green-upgrade/environment', + 'patterns/blue-green-upgrade/modules/eks_cluster', + 'patterns/istio-multi-cluster/1.cluster1', # relies on remote state + 'patterns/istio-multi-cluster/2.cluster2', # relies on remote state + 'patterns/privatelink-access', } projects = { - x.replace('/versions.tf', '') - for x in glob.glob('patterns/**/versions.tf', recursive=True) + x.replace('/main.tf', '') + for x in glob.glob('patterns/**/main.tf', recursive=True) if not re.match(r'^.+/_', x) } @@ -27,4 +27,4 @@ def get_examples(): if __name__ == '__main__': - get_examples() + get_patterns() diff --git a/patterns/wireguard-with-cilium/README.md b/patterns/wireguard-with-cilium/README.md index 877b58ab79..d799203f41 100644 --- a/patterns/wireguard-with-cilium/README.md +++ b/patterns/wireguard-with-cilium/README.md @@ -1,9 +1,16 @@ # Transparent Encryption with Cilium and Wireguard -This pattern demonstrates Cilium configured in CNI chaining mode with VPC CNI and with Wireguard transparent encryption enabled on an Amazon EKS cluster. +This pattern demonstrates Cilium configured in CNI chaining mode with the VPC CNI and with Wireguard transparent encryption enabled on an Amazon EKS cluster. -- [Cilium CNI Chaining Documentation](https://docs.cilium.io/en/v1.12/gettingstarted/cni-chaining-aws-cni/) -- [Cilium Wireguard Encryption Documentation](https://docs.cilium.io/en/v1.12/gettingstarted/encryption-wireguard/) +- [Cilium CNI Chaining Documentation](https://docs.cilium.io/en/stable/installation/cni-chaining-aws-cni/) +- [Cilium Wireguard Encryption Documentation](https://docs.cilium.io/en/stable/security/network/encryption-wireguard/) + +## Areas of Interest + +- `eks.tf` contains the cluster configuration and the deployment of Cilium. + - There are no specific requirements from an EKS perspective, other than the Linux Kernel version used by the OS must be 5.10+. + On Amazon EKS, this is available starting with EKS 1.24, or users can utilize the Bottlerocket OS for EKS < 1.23 +- `sample.tf` provides a sample application used to demonstrate the encrypted connectivity. This is optional and not required for the pattern. ## Deploy @@ -11,17 +18,39 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started ## Validate -1. List the daemonsets +1. Get the Cilium status from one of the Cilium pods. + + Under the `Encryption` field, it should state `Wireguard` with a PubKey. + `NodeEncryption: Disabled` is expected since `NodeEncryption` was not enabled + via the Helm values provided. ```sh - kubectl get ds -n kube-system + kubectl -n kube-system exec -ti ds/cilium -- cilium status ``` ```text - NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE - aws-node 2 2 2 2 2 156m - cilium 2 2 2 2 2 kubernetes.io/os=linux 152m - kube-proxy 2 2 2 2 2 156m + Defaulted container "cilium-agent" out of: cilium-agent, config (init), mount-cgroup (init), apply-sysctl-overwrites (init), mount-bpf-fs (init), clean-cilium-state (init), install-cni-binaries (init) + KVStore: Ok Disabled + Kubernetes: Ok 1.27+ (v1.27.4-eks-2d98532) [linux/amd64] + Kubernetes APIs: ["EndpointSliceOrEndpoint", "cilium/v2::CiliumClusterwideNetworkPolicy", "cilium/v2::CiliumEndpoint", "cilium/v2::CiliumNetworkPolicy", "cilium/v2::CiliumNode", "cilium/v2alpha1::CiliumCIDRGroup", "core/v1::Namespace", "core/v1::Pods", "core/v1::Service", "networking.k8s.io/v1::NetworkPolicy"] + KubeProxyReplacement: False [eth0 10.0.45.128 (Direct Routing), eth1 10.0.40.206] + Host firewall: Disabled + CNI Chaining: aws-cni + Cilium: Ok 1.14.1 (v1.14.1-c191ef6f) + NodeMonitor: Listening for events on 2 CPUs with 64x4096 of shared memory + Cilium health daemon: Ok + IPAM: IPv4: 1/254 allocated from 10.0.1.0/24, + IPv4 BIG TCP: Disabled + IPv6 BIG TCP: Disabled + BandwidthManager: Disabled + Host Routing: Legacy + Masquerading: Disabled + Controller Status: 20/20 healthy + Proxy Status: No managed proxy redirect + Global Identity Range: min 256, max 65535 + Hubble: Ok Current/Max Flows: 4095/4095 (100.00%), Flows/s: 1.58 Metrics: Disabled + Encryption: Wireguard [NodeEncryption: Disabled, cilium_wg0 (Pubkey: Es25c2idJtRzE0/FKAOvKPJ7ybRmZ23KrufK3HOuZTY=, Port: 51871, Peers: 1)] + Cluster health: Probe disabled ``` 2. Open a shell inside the cilium container @@ -30,39 +59,114 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started kubectl -n kube-system exec -ti ds/cilium -- bash ``` -3. Verify Encryption is enabled - - ```sh - cilium status | grep Encryption - ``` - - ```text - Encryption: Wireguard [cilium_wg0 (Pubkey: b2krgbHgaCsVWALMnFLiS/RekhhcE36PXEjQ7T8+mW0=, Port: 51871, Peers: 1)] - ``` - -4. Install [`tcpdump`](https://www.tcpdump.org/) +3. Install [`tcpdump`](https://www.tcpdump.org/) ```sh apt-get update apt-get install -y tcpdump ``` -5. Start a packet capture on `cilium_wg0` and verify you see payload in clear text, it means the traffic is encrypted with wireguard +4. Start a packet capture on `cilium_wg0` and verify you see payload in clear text, it means the traffic is encrypted with wireguard ```sh tcpdump -A -c 40 -i cilium_wg0 | grep "Welcome to nginx!" ``` ```text + tcpdump: verbose output suppressed, use -v[v]... for full protocol decode + listening on cilium_wg0, link-type RAW (Raw IP), snapshot length 262144 bytes Welcome to nginx!

Welcome to nginx!

- ... - 40 packets captured 40 packets received by filter 0 packets dropped by kernel ``` +5. Deploy the Cilium connectivity resources to check and evaluate connectivity: + + ```sh + kubectl create ns cilium-test + kubectl apply -n cilium-test -f https://raw.githubusercontent.com/cilium/cilium/v1.14.1/examples/kubernetes/connectivity-check/connectivity-check.yaml + ``` + + ```text + deployment.apps/echo-a created + deployment.apps/echo-b created + deployment.apps/echo-b-host created + deployment.apps/pod-to-a created + deployment.apps/pod-to-external-1111 created + deployment.apps/pod-to-a-denied-cnp created + deployment.apps/pod-to-a-allowed-cnp created + deployment.apps/pod-to-external-fqdn-allow-google-cnp created + deployment.apps/pod-to-b-multi-node-clusterip created + deployment.apps/pod-to-b-multi-node-headless created + deployment.apps/host-to-b-multi-node-clusterip created + deployment.apps/host-to-b-multi-node-headless created + deployment.apps/pod-to-b-multi-node-nodeport created + deployment.apps/pod-to-b-intra-node-nodeport created + service/echo-a created + service/echo-b created + service/echo-b-headless created + service/echo-b-host-headless created + ciliumnetworkpolicy.cilium.io/pod-to-a-denied-cnp created + ciliumnetworkpolicy.cilium.io/pod-to-a-allowed-cnp created + ciliumnetworkpolicy.cilium.io/pod-to-external-fqdn-allow-google-cnp created + ``` + +6. View the logs of any of the connectivity tests to view the results: + + ```sh + kubectl logs echo-a-6575c98b7d-xknsv -n cilium-test + ``` + + ```text + \{^_^}/ hi! + + Loading /default.json + Done + + Resources + http://:8080/private + http://:8080/public + + Home + http://:8080 + + Type s + enter at any time to create a snapshot of the database + Watching... + + GET /public 200 7.063 ms - 57 + GET /public 200 3.126 ms - 57 + GET /public 200 3.039 ms - 57 + GET /public 200 2.776 ms - 57 + GET /public 200 3.087 ms - 57 + GET /public 200 2.781 ms - 57 + GET /public 200 2.977 ms - 57 + GET /public 200 2.596 ms - 57 + GET /public 200 2.991 ms - 57 + GET /public 200 2.708 ms - 57 + GET /public 200 3.066 ms - 57 + GET /public 200 2.616 ms - 57 + GET /public 200 2.875 ms - 57 + GET /public 200 2.689 ms - 57 + GET /public 200 2.800 ms - 57 + GET /public 200 2.556 ms - 57 + GET /public 200 3.238 ms - 57 + GET /public 200 2.538 ms - 57 + GET /public 200 2.890 ms - 57 + GET /public 200 2.666 ms - 57 + GET /public 200 2.729 ms - 57 + GET /public 200 2.580 ms - 57 + GET /public 200 2.919 ms - 57 + GET /public 200 2.630 ms - 57 + GET /public 200 2.857 ms - 57 + GET /public 200 2.716 ms - 57 + GET /public 200 1.693 ms - 57 + GET /public 200 2.715 ms - 57 + GET /public 200 2.729 ms - 57 + GET /public 200 2.655 ms - 57 + ``` + ## Destroy {% diff --git a/patterns/wireguard-with-cilium/eks.tf b/patterns/wireguard-with-cilium/eks.tf new file mode 100644 index 0000000000..5d448bcec3 --- /dev/null +++ b/patterns/wireguard-with-cilium/eks.tf @@ -0,0 +1,102 @@ +################################################################################ +# Cluster +################################################################################ + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.16" + + cluster_name = local.name + cluster_version = "1.27" + cluster_endpoint_public_access = true + + # EKS Addons + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = {} + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["m5.large"] + # Cilium Wireguard requires Linux Kernel 5.10 or above. + # For EKS 1.24 and above, the AMI the Kernel version is 5.10 + # For EKS 1.23 and below, you need to use Bottlerocket OS. For example: + # ami_type = "BOTTLEROCKET_x86_64" + # platform = "bottlerocket" + min_size = 1 + max_size = 3 + desired_size = 2 + } + } + # Extend node-to-node security group rules + node_security_group_additional_rules = { + # Cilium Wireguard Port https://github.com/cilium/cilium/blob/main/Documentation/security/network/encryption-wireguard.rst + ingress_cilium_wireguard = { + description = "Allow Cilium Wireguard node to node" + protocol = "udp" + from_port = 51871 + to_port = 51871 + type = "ingress" + self = true + } + } + + tags = local.tags +} + +################################################################################ +# Kubectl Output +################################################################################ + +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.7" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + helm_releases = { + cilium = { + name = "cilium" + chart = "cilium" + version = "1.14.1" + repository = "https://helm.cilium.io/" + description = "Cilium Add-on" + namespace = "kube-system" + create_namespace = false + + values = [ + <<-EOT + cni: + chainingMode: aws-cni + enableIPv4Masquerade: false + tunnel: disabled + endpointRoutes: + enabled: true + l7Proxy: false + encryption: + enabled: true + type: wireguard + EOT + ] + } + } + + tags = local.tags +} diff --git a/patterns/wireguard-with-cilium/main.tf b/patterns/wireguard-with-cilium/main.tf index 1b6f28e54c..e6c70ffea4 100644 --- a/patterns/wireguard-with-cilium/main.tf +++ b/patterns/wireguard-with-cilium/main.tf @@ -1,3 +1,29 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.47" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.9" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/wireguard-with-cilium/terraform.tfstate" + # } +} + provider "aws" { region = local.region } @@ -28,19 +54,9 @@ provider "helm" { } } -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} +################################################################################ +# Common data/locals +################################################################################ data "aws_availability_zones" "available" {} @@ -51,208 +67,12 @@ locals { vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) - tags = { Blueprint = local.name GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" } } -################################################################################ -# Cluster -################################################################################ - -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.16" - - cluster_name = local.name - cluster_version = "1.27" - cluster_endpoint_public_access = true - - # EKS Addons - cluster_addons = { - coredns = {} - kube-proxy = {} - vpc-cni = {} - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - - eks_managed_node_groups = { - initial = { - instance_types = ["m5.large"] - # Cilium Wireguard requires Linux Kernel 5.10 or aboved. - # For EKS 1.24 and above, the AMI the Kernerl version is 5.10 - # For EKS 1.23 and below, you need to use Bottlerocket OS. For example: - # ami_type = "BOTTLEROCKET_x86_64" - # platform = "bottlerocket" - min_size = 1 - max_size = 3 - desired_size = 2 - } - } - # Extend node-to-node security group rules - node_security_group_additional_rules = { - ingress_cilium_wireguard = { - description = "Allow Cilium Wireguard node to node" - protocol = "udp" - from_port = 51871 - to_port = 51871 # Cilium Wireguard Port https://github.com/cilium/cilium/blob/main/Documentation/security/network/encryption-wireguard.rst - type = "ingress" - self = true - } - } - - tags = local.tags -} - -################################################################################ -# Cilium Helm Chart for e2e encryption with Wireguard -################################################################################ - -resource "helm_release" "cilium" { - name = "cilium" - chart = "cilium" - version = "1.13.2" - repository = "https://helm.cilium.io/" - description = "Cilium Add-on" - namespace = "kube-system" - create_namespace = false - - values = [ - <<-EOT - cni: - chainingMode: aws-cni - enableIPv4Masquerade: false - tunnel: disabled - endpointRoutes: - enabled: true - l7Proxy: false - encryption: - enabled: true - type: wireguard - EOT - ] - - depends_on = [ - module.eks - ] -} - -#--------------------------------------------------------------- -# Sample App for Testing -#--------------------------------------------------------------- - -# For some reason the example pods can't be deployed right after helm install of cilium a delay needs to be introduced. This is being investigated -resource "time_sleep" "wait_wireguard" { - count = var.enable_example ? 1 : 0 - create_duration = "15s" - - depends_on = [helm_release.cilium] -} - -resource "kubectl_manifest" "server" { - count = var.enable_example ? 1 : 0 - - yaml_body = yamlencode({ - apiVersion = "v1" - kind = "Pod" - metadata = { - name = "server" - labels = { - blog = "wireguard" - name = "server" - } - } - spec = { - containers = [ - { - name = "server" - image = "nginx" - } - ] - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "kubernetes.io/hostname" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - blog = "wireguard" - } - } - } - ] - } - }) - - depends_on = [time_sleep.wait_wireguard] -} - -resource "kubectl_manifest" "service" { - count = var.enable_example ? 1 : 0 - - yaml_body = yamlencode({ - apiVersion = "v1" - kind = "Service" - metadata = { - name = "server" - } - spec = { - selector = { - name = "server" - } - ports = [ - { - port = 80 - } - ] - } - }) -} - -resource "kubectl_manifest" "client" { - count = var.enable_example ? 1 : 0 - - yaml_body = yamlencode({ - apiVersion = "v1" - kind = "Pod" - metadata = { - name = "client" - labels = { - blog = "wireguard" - name = "client" - } - } - spec = { - containers = [ - { - name = "client" - image = "busybox" - command = ["watch", "wget", "server"] - } - ] - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "kubernetes.io/hostname" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - blog = "wireguard" - } - } - } - ] - } - }) - - depends_on = [kubectl_manifest.server] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/patterns/wireguard-with-cilium/outputs.tf b/patterns/wireguard-with-cilium/outputs.tf deleted file mode 100644 index c624023e90..0000000000 --- a/patterns/wireguard-with-cilium/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" -} diff --git a/patterns/wireguard-with-cilium/sample.tf b/patterns/wireguard-with-cilium/sample.tf new file mode 100644 index 0000000000..626a208013 --- /dev/null +++ b/patterns/wireguard-with-cilium/sample.tf @@ -0,0 +1,83 @@ +################################################################################ +# Sample Application +################################################################################ + +resource "kubernetes_pod_v1" "server" { + metadata { + name = "server" + labels = { + blog = "wireguard" + name = "server" + } + } + + spec { + container { + image = "nginx" + name = "server" + } + + topology_spread_constraint { + max_skew = 1 + topology_key = "kubernetes.io/hostname" + when_unsatisfiable = "DoNotSchedule" + + label_selector { + match_expressions { + key = "blog" + operator = "In" + values = ["wireguard"] + } + } + } + } +} + +resource "kubernetes_service_v1" "server" { + metadata { + name = "server" + } + spec { + selector = { + name = kubernetes_pod_v1.server.metadata[0].labels.name + } + + session_affinity = "ClientIP" + + port { + port = 80 + } + } +} + +resource "kubernetes_pod_v1" "client" { + metadata { + name = "client" + labels = { + blog = "wireguard" + name = "client" + } + } + + spec { + container { + image = "busybox" + name = "client" + command = ["watch", "wget", "server"] + } + + topology_spread_constraint { + max_skew = 1 + topology_key = "kubernetes.io/hostname" + when_unsatisfiable = "DoNotSchedule" + + label_selector { + match_expressions { + key = "blog" + operator = "In" + values = ["wireguard"] + } + } + } + } +} diff --git a/patterns/wireguard-with-cilium/variables.tf b/patterns/wireguard-with-cilium/variables.tf deleted file mode 100644 index 433e9ec9d1..0000000000 --- a/patterns/wireguard-with-cilium/variables.tf +++ /dev/null @@ -1,6 +0,0 @@ -# tflint-ignore: terraform_unused_declarations -variable "enable_example" { - description = "Enable example to test this blueprint" - type = bool - default = true -} diff --git a/patterns/wireguard-with-cilium/versions.tf b/patterns/wireguard-with-cilium/versions.tf deleted file mode 100644 index d42480c1e6..0000000000 --- a/patterns/wireguard-with-cilium/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.47" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.9" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.20" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14" - } - time = { - source = "hashicorp/time" - version = ">= 0.9" - } - } - - # ## Used for end-to-end testing on project; update to suit your needs - # backend "s3" { - # bucket = "terraform-ssp-github-actions-state" - # region = "us-west-2" - # key = "e2e/wireguard-with-cilium/terraform.tfstate" - # } -}