From e43a02c1e6b7b40ffb8e7f4f856581eaaec065b5 Mon Sep 17 00:00:00 2001 From: Adrian Berger Date: Mon, 30 Sep 2024 15:42:24 +0200 Subject: [PATCH 1/5] [vcluster]: Add audit feature to apiserver Signed-off-by: Adrian Berger --- charts/vcluster/Chart.yaml | 4 +-- charts/vcluster/README.md | 12 ++++++-- .../kubernetes/apiserver/deployment.yaml | 30 +++++++++++++++++++ .../kubernetes/apiserver/policy.yaml | 18 +++++++++++ charts/vcluster/values.yaml | 27 ++++++++++++++++- ct.yaml | 2 +- 6 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml diff --git a/charts/vcluster/Chart.yaml b/charts/vcluster/Chart.yaml index b3a39dd2..46f7f48f 100644 --- a/charts/vcluster/Chart.yaml +++ b/charts/vcluster/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: vcluster description: Virtual Kubernetes Cluster type: application -version: 0.5.7 +version: 0.6.0 appVersion: 0.1.0 keywords: - vcluster @@ -16,4 +16,4 @@ maintainers: dependencies: - name: common version: 2.14.1 - repository: https://charts.bitnami.com/bitnami + repository: http://charts.bitnami.com/bitnami diff --git a/charts/vcluster/README.md b/charts/vcluster/README.md index 42389133..9885b223 100644 --- a/charts/vcluster/README.md +++ b/charts/vcluster/README.md @@ -2,7 +2,7 @@ __This Chart is under active development! We try to improve documentation and values consistency over time__ -![Version: 0.5.7](https://img.shields.io/badge/Version-0.5.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) Virtual Kubernetes Cluster @@ -18,7 +18,7 @@ Virtual Kubernetes Cluster | Repository | Name | Version | |------------|------|---------| -| https://charts.bitnami.com/bitnami | common | 2.14.1 | +| http://charts.bitnami.com/bitnami | common | 2.14.1 | # Major Changes @@ -410,6 +410,14 @@ Deploys [Kubernetes API Server](https://kubernetes.io/docs/reference/command-lin | kubernetes.apiServer.affinity | object | `{}` | Affinity | | kubernetes.apiServer.annotations | object | `{}` | Annotations for Workload | | kubernetes.apiServer.args | object | `{}` | Extra arguments for the kube-apiserver | +| kubernetes.apiServer.audit.enabled | bool | `false` | Enable Audit Log | +| kubernetes.apiServer.audit.maxAge | int | `7` | Defines the maximum number of days to retain old audit log files | +| kubernetes.apiServer.audit.maxBackup | int | `2` | Defines the maximum number of audit log files to retain | +| kubernetes.apiServer.audit.maxSize | int | `100` | Defines the maximum size in megabytes of the audit log file before it gets rotated | +| kubernetes.apiServer.audit.policy | string | `"# Log all requests at the Metadata level.\napiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n - level: Metadata\n"` | Audit Policy | +| kubernetes.apiServer.audit.truncateEnabled | bool | `false` | Whether event and batch truncating is enabled | +| kubernetes.apiServer.audit.truncateMaxBatchSize | int | `10485760` | Maximum size in bytes of the batch sent to the underlying backend | +| kubernetes.apiServer.audit.truncateMaxEventSize | int | `102400` | Maximum size in bytes of the audit event sent to the underlying backend | | kubernetes.apiServer.autoscaling.enabled | bool | `false` | Enable Horizontal Pod Autoscaler | | kubernetes.apiServer.autoscaling.maxReplicas | int | `5` | Maximum available Replicas | | kubernetes.apiServer.autoscaling.minReplicas | int | `1` | Minimum available Replicas | diff --git a/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml b/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml index ad80d905..e91835e2 100644 --- a/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml +++ b/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml @@ -111,6 +111,18 @@ spec: - --advertise-address={{ . }} {{- end }} {{- end }} + {{- if $kubernetes.apiServer.audit.enabled }} + - --audit-policy-file=/etc/kubernetes/audit-policy.yaml + - --audit-log-path=/var/log/kubernetes/audit/audit.log + - --audit-log-maxage={{ $kubernetes.apiServer.audit.maxAge }} + - --audit-log-maxbackup={{ $kubernetes.apiServer.audit.maxBackup }} + - --audit-log-maxsize={{ $kubernetes.apiServer.audit.maxSize }} + {{- if $kubernetes.apiServer.audit.truncateEnabled }} + - --audit-log-truncate-enabled={{ $kubernetes.apiServer.audit.truncateEnabled }} + - --audit-log-truncate-max-batch-size={{ $kubernetes.apiServer.audit.truncateMaxBatchSize }} + - --audit-log-truncate-max-event-size={{ $kubernetes.apiServer.audit.truncateMaxEventSize }} + {{- end }} + {{- end }} {{- with $kubernetes.apiServer.args }} {{- include "pkg.utils.args" (dict "args" . "ctx" $) | nindent 8 }} {{- end }} @@ -171,6 +183,15 @@ spec: name: konnectivity-uds {{- end }} {{- end }} + {{- if $kubernetes.apiServer.audit.enabled }} + - mountPath: /etc/kubernetes/audit-policy.yaml + name: audit + readOnly: true + subPath: audit-policy.yaml + - mountPath: /var/log/kubernetes/audit/ + name: audit-log + readOnly: false + {{- end }} {{- with $kubernetes.apiServer.volumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -207,6 +228,15 @@ spec: {{- include "kubernetes.konnectivityServer.volumes" $ | nindent 6 }} {{- end }} {{- end }} + {{- if $kubernetes.apiServer.audit.enabled }} + - configMap: + name: "{{ $fullName }}-audit-policy" + name: audit + - name: audit-log + hostPath: + path: /var/log/kubernetes/audit/ + type: DirectoryOrCreate + {{- end }} {{- with $kubernetes.apiServer.volumes }} {{- toYaml . | nindent 6 }} {{- end }} diff --git a/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml b/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml new file mode 100644 index 00000000..8a1caebb --- /dev/null +++ b/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml @@ -0,0 +1,18 @@ +{{- if (include "kubernetes.enabled" $) -}} + {{- $kubernetes := $.Values.kubernetes -}} + {{- if and $kubernetes.apiServer.enabled $kubernetes.apiServer.audit.enabled -}} + {{- $fullName := include "kubernetes.fullname" . -}} + {{- $component_name := "apiserver" -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $fullName }}-audit-policy + labels: {{- include "kubernetes.labels" $ | nindent 4 }} + {{ include "pkg.common.labels.component" $ }}: {{ $component_name }} + namespace: {{ $.Release.Namespace }} +data: + audit-policy.yaml: | + {{- $kubernetes.apiServer.audit.policy | nindent 4 }} + {{- end -}} +{{- end -}} diff --git a/charts/vcluster/values.yaml b/charts/vcluster/values.yaml index 7fd0d91e..90e33731 100644 --- a/charts/vcluster/values.yaml +++ b/charts/vcluster/values.yaml @@ -998,7 +998,6 @@ osm: # -- Benchmark Memory Usage targetMemoryUtilizationPercentage: - # ---------------------------- # Kubernetes Component # ---------------------------- @@ -1453,6 +1452,32 @@ kubernetes: # -- Assign additional Annotations annotations: {} + # API Server Audit Configuration + audit: + # -- Enable Audit Log + enabled: false + + # -- Defines the maximum number of days to retain old audit log files + maxAge: 7 + # -- Defines the maximum number of audit log files to retain + maxBackup: 2 + # -- Defines the maximum size in megabytes of the audit log file before it gets rotated + maxSize: 100 + # -- Whether event and batch truncating is enabled + truncateEnabled: false + # -- Maximum size in bytes of the batch sent to the underlying backend + truncateMaxBatchSize: 10485760 + # -- Maximum size in bytes of the audit event sent to the underlying backend + truncateMaxEventSize: 102400 + + # -- Audit Policy + policy: | + # Log all requests at the Metadata level. + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + - level: Metadata + controllerManager: # -- Enable Kubernetes Controller-Manager enabled: true diff --git a/ct.yaml b/ct.yaml index 51efbc90..99a568c2 100644 --- a/ct.yaml +++ b/ct.yaml @@ -7,7 +7,7 @@ chart-dirs: chart-repos: - buttahtoast=https://buttahtoast.github.io/helm-charts/ - bedag=https://bedag.github.io/helm-charts/ - - bitnami=https://charts.bitnami.com/bitnami + - bitnami=http://charts.bitnami.com/bitnami validate-chart-schema: true validate-maintainers: false validate-yaml: true From d4c1ae1a80643ac3be25b3081b857ed05f27bb43 Mon Sep 17 00:00:00 2001 From: Adrian Berger Date: Mon, 7 Oct 2024 10:44:34 +0200 Subject: [PATCH 2/5] fix: Do not run chart-testing (install) in GitHub workflows Signed-off-by: Adrian Berger --- .github/workflows/lint-and-test.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index be1a9a4e..a7972396 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -53,20 +53,3 @@ jobs: - name: Run chart-testing (lint) run: ct lint --config ct.yaml --lint-conf lintconf.yaml if: steps.list-changed.outputs.changed == 'true' - - - name: Create kind cluster - uses: helm/kind-action@v1.5.0 - if: steps.list-changed.outputs.changed == 'true' - #with: - # kubectl_version: v1.22.0 - - # Install Dependencies - - name: Install Required dependencies - if: steps.list-changed.outputs.changed == 'true' - run: | - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml - kubectl create -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml - - - name: Run chart-testing (install) - run: ct install --config ct.yaml - if: steps.list-changed.outputs.changed == 'true' From 068442e2a5782b9997f7d2eec7f78a57d3bdd384 Mon Sep 17 00:00:00 2001 From: Adrian Berger Date: Mon, 7 Oct 2024 12:24:39 +0200 Subject: [PATCH 3/5] fix: Switch to https for dependency in vcluster again Signed-off-by: Adrian Berger --- charts/vcluster/Chart.yaml | 2 +- charts/vcluster/README.md | 2 +- ct.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/vcluster/Chart.yaml b/charts/vcluster/Chart.yaml index 46f7f48f..fe741fcf 100644 --- a/charts/vcluster/Chart.yaml +++ b/charts/vcluster/Chart.yaml @@ -16,4 +16,4 @@ maintainers: dependencies: - name: common version: 2.14.1 - repository: http://charts.bitnami.com/bitnami + repository: https://charts.bitnami.com/bitnami diff --git a/charts/vcluster/README.md b/charts/vcluster/README.md index 9885b223..3576e102 100644 --- a/charts/vcluster/README.md +++ b/charts/vcluster/README.md @@ -18,7 +18,7 @@ Virtual Kubernetes Cluster | Repository | Name | Version | |------------|------|---------| -| http://charts.bitnami.com/bitnami | common | 2.14.1 | +| https://charts.bitnami.com/bitnami | common | 2.14.1 | # Major Changes diff --git a/ct.yaml b/ct.yaml index 99a568c2..51efbc90 100644 --- a/ct.yaml +++ b/ct.yaml @@ -7,7 +7,7 @@ chart-dirs: chart-repos: - buttahtoast=https://buttahtoast.github.io/helm-charts/ - bedag=https://bedag.github.io/helm-charts/ - - bitnami=http://charts.bitnami.com/bitnami + - bitnami=https://charts.bitnami.com/bitnami validate-chart-schema: true validate-maintainers: false validate-yaml: true From 3ae8528a1bb72a60df2f606fec8fd2b210a0d3a1 Mon Sep 17 00:00:00 2001 From: Adrian Berger Date: Mon, 7 Oct 2024 13:43:22 +0200 Subject: [PATCH 4/5] fix: casting of integers in vclusters Signed-off-by: Adrian Berger --- charts/vcluster/README.md | 10 +++++----- .../components/kubernetes/apiserver/deployment.yaml | 10 +++++----- charts/vcluster/values.yaml | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/charts/vcluster/README.md b/charts/vcluster/README.md index 3576e102..2cb33be1 100644 --- a/charts/vcluster/README.md +++ b/charts/vcluster/README.md @@ -411,13 +411,13 @@ Deploys [Kubernetes API Server](https://kubernetes.io/docs/reference/command-lin | kubernetes.apiServer.annotations | object | `{}` | Annotations for Workload | | kubernetes.apiServer.args | object | `{}` | Extra arguments for the kube-apiserver | | kubernetes.apiServer.audit.enabled | bool | `false` | Enable Audit Log | -| kubernetes.apiServer.audit.maxAge | int | `7` | Defines the maximum number of days to retain old audit log files | -| kubernetes.apiServer.audit.maxBackup | int | `2` | Defines the maximum number of audit log files to retain | -| kubernetes.apiServer.audit.maxSize | int | `100` | Defines the maximum size in megabytes of the audit log file before it gets rotated | +| kubernetes.apiServer.audit.maxAge | string | `"7"` | Defines the maximum number of days to retain old audit log files | +| kubernetes.apiServer.audit.maxBackup | string | `"2"` | Defines the maximum number of audit log files to retain | +| kubernetes.apiServer.audit.maxSize | string | `"100"` | Defines the maximum size in megabytes of the audit log file before it gets rotated | | kubernetes.apiServer.audit.policy | string | `"# Log all requests at the Metadata level.\napiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n - level: Metadata\n"` | Audit Policy | | kubernetes.apiServer.audit.truncateEnabled | bool | `false` | Whether event and batch truncating is enabled | -| kubernetes.apiServer.audit.truncateMaxBatchSize | int | `10485760` | Maximum size in bytes of the batch sent to the underlying backend | -| kubernetes.apiServer.audit.truncateMaxEventSize | int | `102400` | Maximum size in bytes of the audit event sent to the underlying backend | +| kubernetes.apiServer.audit.truncateMaxBatchSize | string | `"10485760"` | Maximum size in bytes of the batch sent to the underlying backend | +| kubernetes.apiServer.audit.truncateMaxEventSize | string | `"102400"` | Maximum size in bytes of the audit event sent to the underlying backend | | kubernetes.apiServer.autoscaling.enabled | bool | `false` | Enable Horizontal Pod Autoscaler | | kubernetes.apiServer.autoscaling.maxReplicas | int | `5` | Maximum available Replicas | | kubernetes.apiServer.autoscaling.minReplicas | int | `1` | Minimum available Replicas | diff --git a/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml b/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml index e91835e2..59ddec7b 100644 --- a/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml +++ b/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml @@ -114,13 +114,13 @@ spec: {{- if $kubernetes.apiServer.audit.enabled }} - --audit-policy-file=/etc/kubernetes/audit-policy.yaml - --audit-log-path=/var/log/kubernetes/audit/audit.log - - --audit-log-maxage={{ $kubernetes.apiServer.audit.maxAge }} - - --audit-log-maxbackup={{ $kubernetes.apiServer.audit.maxBackup }} - - --audit-log-maxsize={{ $kubernetes.apiServer.audit.maxSize }} + - --audit-log-maxage={{ $kubernetes.apiServer.audit.maxAge | int }} + - --audit-log-maxbackup={{ $kubernetes.apiServer.audit.maxBackup | int }} + - --audit-log-maxsize={{ $kubernetes.apiServer.audit.maxSize | int }} {{- if $kubernetes.apiServer.audit.truncateEnabled }} - --audit-log-truncate-enabled={{ $kubernetes.apiServer.audit.truncateEnabled }} - - --audit-log-truncate-max-batch-size={{ $kubernetes.apiServer.audit.truncateMaxBatchSize }} - - --audit-log-truncate-max-event-size={{ $kubernetes.apiServer.audit.truncateMaxEventSize }} + - --audit-log-truncate-max-batch-size={{ $kubernetes.apiServer.audit.truncateMaxBatchSize | int }} + - --audit-log-truncate-max-event-size={{ $kubernetes.apiServer.audit.truncateMaxEventSize | int }} {{- end }} {{- end }} {{- with $kubernetes.apiServer.args }} diff --git a/charts/vcluster/values.yaml b/charts/vcluster/values.yaml index 90e33731..19b262fb 100644 --- a/charts/vcluster/values.yaml +++ b/charts/vcluster/values.yaml @@ -1458,17 +1458,17 @@ kubernetes: enabled: false # -- Defines the maximum number of days to retain old audit log files - maxAge: 7 + maxAge: "7" # -- Defines the maximum number of audit log files to retain - maxBackup: 2 + maxBackup: "2" # -- Defines the maximum size in megabytes of the audit log file before it gets rotated - maxSize: 100 + maxSize: "100" # -- Whether event and batch truncating is enabled truncateEnabled: false # -- Maximum size in bytes of the batch sent to the underlying backend - truncateMaxBatchSize: 10485760 + truncateMaxBatchSize: "10485760" # -- Maximum size in bytes of the audit event sent to the underlying backend - truncateMaxEventSize: 102400 + truncateMaxEventSize: "102400" # -- Audit Policy policy: | From e9bf2665e39aa90c5ef810f09b3cb961521fb08e Mon Sep 17 00:00:00 2001 From: Adrian Berger Date: Mon, 7 Oct 2024 13:45:15 +0200 Subject: [PATCH 5/5] fix: Remove subPath from configMap file mount in vcluster Signed-off-by: Adrian Berger --- .../components/kubernetes/apiserver/deployment.yaml | 5 ++--- .../templates/components/kubernetes/apiserver/policy.yaml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml b/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml index 59ddec7b..82613e80 100644 --- a/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml +++ b/charts/vcluster/templates/components/kubernetes/apiserver/deployment.yaml @@ -112,7 +112,7 @@ spec: {{- end }} {{- end }} {{- if $kubernetes.apiServer.audit.enabled }} - - --audit-policy-file=/etc/kubernetes/audit-policy.yaml + - --audit-policy-file=/etc/kubernetes/audit/policy.yaml - --audit-log-path=/var/log/kubernetes/audit/audit.log - --audit-log-maxage={{ $kubernetes.apiServer.audit.maxAge | int }} - --audit-log-maxbackup={{ $kubernetes.apiServer.audit.maxBackup | int }} @@ -184,10 +184,9 @@ spec: {{- end }} {{- end }} {{- if $kubernetes.apiServer.audit.enabled }} - - mountPath: /etc/kubernetes/audit-policy.yaml + - mountPath: /etc/kubernetes/audit/ name: audit readOnly: true - subPath: audit-policy.yaml - mountPath: /var/log/kubernetes/audit/ name: audit-log readOnly: false diff --git a/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml b/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml index 8a1caebb..4e8f36c8 100644 --- a/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml +++ b/charts/vcluster/templates/components/kubernetes/apiserver/policy.yaml @@ -12,7 +12,7 @@ metadata: {{ include "pkg.common.labels.component" $ }}: {{ $component_name }} namespace: {{ $.Release.Namespace }} data: - audit-policy.yaml: | + policy.yaml: | {{- $kubernetes.apiServer.audit.policy | nindent 4 }} {{- end -}} {{- end -}}