From d4e3fc10fb12d709030756ba90790cdef06e5b3f Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Wed, 20 Nov 2024 19:19:17 -0300 Subject: [PATCH] feat (in-cluster): [ingress-controller] update nginx to v1.11.3 (#102) * upgrade old reference to 1.9.0 to 1.11.3 * snippet annotation no longer allowed in 1.11.3 by default * be explicit about enabling metrics since it is being scraped * ensure the image is the expected one * nginx no longer requires escalation priviledges * nginx requires being able to read the root filesystem * ensure nginx is not runnning as root * add seccomp profile type * inlcude the admission validation service port * Service account for admission now includes the token automount in true * update certgen image version * update sec context to be aligned with new nginx job * update certgen image version for patch container * update sec context to be aligned with new nginx job patch and resinstate fsgroup for create * update deployment docs * bug fix: remove sha256 from acr image imports * bug fix: indent nginx manifest file properly and remove sha256 image names * bug fix: allow-snippet-annotations set to true otherwise workload webui fails * bug fix: add location to rg from e2e validation --- .../ingress-nginx/deployment.yaml | 83 ++++++++++++------- docs/deploy/09-pre-cluster-stamp.md | 8 +- docs/deploy/13-validation.md | 2 +- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/cluster-manifests/ingress-nginx/deployment.yaml b/cluster-manifests/ingress-nginx/deployment.yaml index fa27498a..b348000a 100644 --- a/cluster-manifests/ingress-nginx/deployment.yaml +++ b/cluster-manifests/ingress-nginx/deployment.yaml @@ -1,4 +1,4 @@ -# Source: https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.9.0/deploy/static/provider/cloud/deploy.yaml +# Source: https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.3/deploy/static/provider/cloud/deploy.yaml --- apiVersion: v1 automountServiceAccountToken: true @@ -9,14 +9,14 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx namespace: ingress-nginx --- apiVersion: v1 data: #https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ - # This is our Azure Application Gateway range. The ONLY traffic that should be hitting + # This is the Azure Application Gateway subnet address space. The ONLY traffic that should be hitting # the ingress controller directly is AAG. whitelist-source-range: 10.240.5.0/24 ssl-redirect: "false" @@ -28,7 +28,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-controller namespace: ingress-nginx --- @@ -39,7 +39,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx rules: - apiGroups: @@ -120,7 +120,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io @@ -139,7 +139,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx namespace: ingress-nginx rules: @@ -229,7 +229,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx namespace: ingress-nginx roleRef: @@ -249,7 +249,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-controller-admission namespace: ingress-nginx spec: @@ -272,7 +272,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-controller namespace: ingress-nginx annotations: @@ -304,7 +304,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 pci-scope: out-of-scope name: ingress-nginx-controller namespace: ingress-nginx @@ -331,7 +331,7 @@ spec: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 pci-scope: out-of-scope spec: containers: @@ -347,6 +347,7 @@ spec: - --validating-webhook-certificate=/usr/local/certificates/cert - --validating-webhook-key=/usr/local/certificates/key - --v=3 + - --enable-metrics=true env: - name: POD_NAME valueFrom: @@ -358,7 +359,7 @@ spec: fieldPath: metadata.namespace - name: LD_PRELOAD value: /usr/local/lib/libmimalloc.so - image: YOUR_ACR.azurecr.io/live/ingress-nginx/controller:v1.9.0 + image: YOUR_ACR.azurecr.io/live/ingress-nginx/controller:v1.11.3 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -382,7 +383,7 @@ spec: protocol: TCP - containerPort: 8443 name: webhook - protocol: TCP + protocol: TCP - containerPort: 10254 name: metrics protocol: TCP @@ -395,7 +396,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 1 resources: requests: cpu: 100m @@ -404,13 +405,17 @@ spec: cpu: 500m memory: 300Mi securityContext: - allowPrivilegeEscalation: true + allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL runAsUser: 101 + readOnlyRootFilesystem: false + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - name: webhook-cert mountPath: /usr/local/certificates/ @@ -443,7 +448,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: nginx spec: controller: k8s.io/ingress-nginx @@ -456,7 +461,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-admission webhooks: - admissionReviewVersions: @@ -466,6 +471,7 @@ webhooks: name: ingress-nginx-controller-admission namespace: ingress-nginx path: /networking/v1/ingresses + port: 443 failurePolicy: Fail matchPolicy: Equivalent name: validate.nginx.ingress.kubernetes.io @@ -482,6 +488,7 @@ webhooks: sideEffects: None --- apiVersion: v1 +automountServiceAccountToken: true kind: ServiceAccount metadata: labels: @@ -489,7 +496,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-admission namespace: ingress-nginx --- @@ -501,7 +508,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-admission rules: - apiGroups: @@ -520,7 +527,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-admission roleRef: apiGroup: rbac.authorization.k8s.io @@ -539,7 +546,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-admission namespace: ingress-nginx rules: @@ -559,7 +566,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 name: ingress-nginx-admission namespace: ingress-nginx roleRef: @@ -579,7 +586,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 pci-scope: out-of-scope name: ingress-nginx-admission-create namespace: ingress-nginx @@ -592,7 +599,7 @@ spec: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 pci-scope: out-of-scope spec: containers: @@ -606,11 +613,17 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v20230407 + image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v1.4.4 imagePullPolicy: IfNotPresent name: create securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault resources: requests: cpu: 100m @@ -623,9 +636,9 @@ spec: pci-scope: out-of-scope restartPolicy: OnFailure securityContext: - fsGroup: 2000 runAsNonRoot: true - runAsUser: 2000 + runAsUser: 65532 + fsGroup: 2000 serviceAccountName: ingress-nginx-admission --- apiVersion: batch/v1 @@ -636,7 +649,7 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 pci-scope: out-of-scope namespace: ingress-nginx name: ingress-nginx-admission-patch @@ -649,7 +662,7 @@ spec: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.9.0 + app.kubernetes.io/version: 1.11.3 pci-scope: out-of-scope spec: containers: @@ -665,11 +678,17 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v20230407 + image: YOUR_ACR.azurecr.io/live/ingress-nginx/kube-webhook-certgen:v1.4.4 imagePullPolicy: IfNotPresent name: patch securityContext: allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault resources: requests: cpu: 100m @@ -683,6 +702,6 @@ spec: restartPolicy: OnFailure securityContext: runAsNonRoot: true - runAsUser: 2000 + runAsUser: 65532 fsGroup: 2000 serviceAccountName: ingress-nginx-admission diff --git a/docs/deploy/09-pre-cluster-stamp.md b/docs/deploy/09-pre-cluster-stamp.md index b4321c67..573571ea 100644 --- a/docs/deploy/09-pre-cluster-stamp.md +++ b/docs/deploy/09-pre-cluster-stamp.md @@ -82,8 +82,8 @@ Using a security agent that is container-aware and can operate from within the c az acr import --source docker.io/falcosecurity/falcoctl:0.6.2 -t quarantine/falcosecurity/falcoctl:0.6.2 -n $ACR_NAME_QUARANTINE && \ az acr import --source docker.io/library/busybox:1.36.1 -t quarantine/library/busybox:1.36.1 -n $ACR_NAME_QUARANTINE && \ az acr import --source ghcr.io/kubereboot/kured:1.14.0 -t quarantine/kubereboot/kured:1.14.0 -n $ACR_NAME_QUARANTINE && \ - az acr import --source registry.k8s.io/ingress-nginx/controller:v1.9.0 -t quarantine/ingress-nginx/controller:v1.9.0 -n $ACR_NAME_QUARANTINE && \ - az acr import --source registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407 -t quarantine/ingress-nginx/kube-webhook-certgen:v20230407 -n $ACR_NAME_QUARANTINE + az acr import --source registry.k8s.io/ingress-nginx/controller:v1.11.3 -t quarantine/ingress-nginx/controller:v1.11.3 -n $ACR_NAME_QUARANTINE && \ + az acr import --source registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.4.4 -t quarantine/ingress-nginx/kube-webhook-certgen:v1.4.4 -n $ACR_NAME_QUARANTINE ``` > The above imports account for 100% of the containers that you are actively bringing to the cluster, but not those that come with the AKS service itself nor any of its add-ons or extensions. Those images, outside of your direct control, are all sourced from Microsoft Container Registry's (MCR). While you do not have an affordance to inject yourself in the middle of their distribution to your cluster, you can still pull those images through your inspection process for your own audit and reporting purposes. *All container images that you directly bring to the cluster should pass through your quarantine process.* The *allowed images* Azure Policy associated with this cluster should be configured to match your specific needs. Be sure to update `allowedContainerImagesRegex` in [`cluster-stamp.json`](../../cluster-stamp.json) to define expected image sources to whatever specificity is manageable for you. Never allow a source that you do not intend to use. For example, if you do not bring Open Service Mesh into your cluster, you can remove the existing allowance for `mcr.microsoft.com` as a valid source of images, leaving just `/live/` repositories as the only valid source for non-system namespaces. @@ -122,8 +122,8 @@ Using a security agent that is container-aware and can operate from within the c az acr import --source quarantine/falcosecurity/falcoctl:0.6.2 -r $ACR_NAME_QUARANTINE -t live/falcosecurity/falcoctl:0.6.2 -n $ACR_NAME && \ az acr import --source quarantine/library/busybox:1.36.1 -r $ACR_NAME_QUARANTINE -t live/library/busybox:1.36.1 -n $ACR_NAME && \ az acr import --source quarantine/kubereboot/kured:1.14.0 -r $ACR_NAME_QUARANTINE -t live/kubereboot/kured:1.14.0 -n $ACR_NAME && \ - az acr import --source quarantine/ingress-nginx/controller:v1.9.0 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/controller:v1.9.0 -n $ACR_NAME && \ - az acr import --source quarantine/ingress-nginx/kube-webhook-certgen:v20230407 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/kube-webhook-certgen:v20230407 -n $ACR_NAME + az acr import --source quarantine/ingress-nginx/controller:v1.11.3 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/controller:v1.11.3 -n $ACR_NAME && \ + az acr import --source quarantine/ingress-nginx/kube-webhook-certgen:v1.4.4 -r $ACR_NAME_QUARANTINE -t live/ingress-nginx/kube-webhook-certgen:v1.4.4 -n $ACR_NAME ``` 1. Trigger quarantine violation. *Optional.* diff --git a/docs/deploy/13-validation.md b/docs/deploy/13-validation.md index 262c3aea..974986bd 100644 --- a/docs/deploy/13-validation.md +++ b/docs/deploy/13-validation.md @@ -11,7 +11,7 @@ This section will help you to validate the workload is exposed correctly and res 1. Get the public IP of Azure Application Gateway. ```bash - APPGW_PUBLIC_IP=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.appGwPublicIpAddress.value -o tsv) + APPGW_PUBLIC_IP=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-01 --query properties.outputs.appGwPublicIpAddress.value -o tsv) ``` 1. Create a DNS `A` Record 🛑