From 73430724c362eb1322b46fbb4ad9713749f90f5c Mon Sep 17 00:00:00 2001
From: Chris Werner Rau <cwr@teuto.net>
Date: Fri, 9 Aug 2024 11:05:32 +0200
Subject: [PATCH] feat(t8s-cluster/management-cluster): add memory safety net
 for kubelet

---
 .../management-cluster/clusterClass/_helpers.tpl  |  9 ++++++---
 .../_kubeadmConfigTemplateSpec.yaml               |  1 -
 .../kubeadmControlPlaneTemplate/_helpers.tpl      |  4 ++--
 .../clusterClass/patches/_kubelet.tpl             | 15 ++++++++++++++-
 .../_containerd_override.conf.tpl                 |  4 ++++
 .../systemdOverrides/_kubelet_override.conf.tpl   |  4 ++++
 .../clusterClass/systemdOverrides/_overrides.tpl  |  8 ++++++++
 .../clusterClass/systemdOverrides/_slices.tpl     |  3 +++
 8 files changed, 41 insertions(+), 7 deletions(-)
 create mode 100644 charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl
 create mode 100644 charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl
 create mode 100644 charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl
 create mode 100644 charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl

diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl
index b321ee0d4..049e00e83 100644
--- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl
@@ -26,7 +26,7 @@ openstack
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.postKubeadmCommands" -}}
-  {{- $commands := list -}}
+  {{- $commands := list "systemctl disable --now snapd.service snapd.socket" -}}
   {{- toYaml $commands }}
 {{- end -}}
 
@@ -109,9 +109,9 @@ server = {{ printf "https://%s" .registry | quote }}
 
 {{- define "t8s-cluster.clusterClass.configTemplate.files" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
-  {{- $files := list -}}
+  {{- $files := include "t8s-cluster.clusterClass.node.systemdOverrides" (dict) | fromYamlArray -}}
   {{- if not .excludePatches -}}
-    {{- $files = concat $files (include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | fromYamlArray) -}}
+    {{- $files = concat $files (include "t8s-cluster.patches.kubelet" (dict "context" .context) | fromYamlArray) -}}
   {{- end -}}
   {{- if .Values.containerRegistryMirror.mirrorEndpoint -}}
     {{- $files = concat $files (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | fromYamlArray) -}}
@@ -120,6 +120,9 @@ server = {{ printf "https://%s" .registry | quote }}
   {{- if .Values.global.injectedCertificateAuthorities }}
     {{- $files = append $files (dict "content" .Values.global.injectedCertificateAuthorities "path" "/usr/local/share/ca-certificates/injected-ca-certs.crt" ) -}}
   {{- end }}
+  {{- range $file := $files -}}
+    {{- $_ := set $file "content" (get $file "content" | trim) -}}
+  {{- end -}}
   {{- $files | toYaml -}}
 {{- end -}}
 
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml
index 1c5eef299..55562d2f9 100644
--- a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml
@@ -1,5 +1,4 @@
 {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" -}}
-  {{- $_ := mustMerge . (pick .context "Values") -}}
 joinConfiguration:
   nodeRegistration:
     kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | nindent 6 }}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl
index a4cd9de58..ac92f581f 100644
--- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl
@@ -28,10 +28,10 @@
     {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $file)) "path" $path) -}}
   {{- end -}}
   {{- $files = append $files (dict "content" (.Files.Get "files/kube-proxy.patch.sh") "path" "/etc/kube-proxy-patch.sh" "permissions" "0700") -}}
+  {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
+  {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- range $file := $files -}}
     {{- $_ := set $file "content" (get $file "content" | trim) -}}
   {{- end -}}
-  {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
-  {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- $files | toYaml -}}
 {{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl
index f2849f0cd..c672aa649 100644
--- a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl
@@ -17,6 +17,19 @@
   {{- $options = set $options "protectKernelDefaults" true -}}
   {{- $options = set $options "tlsCipherSuites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray) -}}
   {{- $options = set $options "seccompDefault" true -}}
+{{/*  {{- $options = set $options "enforceNodeAllocatable" (list "pods" "system-reserved" "kube-reserved") -}}*/}}
+  {{- $options = set $options "enforceNodeAllocatable" (list "pods") -}}
+{{/*  {{- $options = set $options "kubeReservedCgroup" (include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) | trimSuffix ".slice") -}}*/}}
+{{/*  {{- $options = set $options "systemReservedCgroup" "system" -}}*/}}
+  {{- $options = set $options "kubeReserved" (dict
+    "memory" "0.2Gi"
+    "ephemeral-storage" "1Gi"
+  ) -}}
+  {{- $options = set $options "systemReserved" (dict
+    "memory" "0.4Gi"
+    "ephemeral-storage" "1Gi"
+  ) -}}
+  {{- $options = set $options "evictionHard" (dict "memory.available" "64Mi" "nodefs.available" "2Gi") -}}
   {{- $options | toYaml -}}
 {{- end -}}
 
@@ -38,7 +51,7 @@
   {{- $patches | toYaml -}}
 {{- end -}}
 
-{{- define "t8s-cluster.patches.kubelet.patches" -}}
+{{- define "t8s-cluster.patches.kubelet" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $patches := include "t8s-cluster.patches.kubelet.default" (dict) | fromYamlArray -}}
   {{- with include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl
new file mode 100644
index 000000000..abcabe41d
--- /dev/null
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl
@@ -0,0 +1,4 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverride.containerd" -}}
+[Service]
+Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}}
+{{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl
new file mode 100644
index 000000000..6246c5452
--- /dev/null
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl
@@ -0,0 +1,4 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverride.kubelet" -}}
+[Service]
+Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}}
+{{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl
new file mode 100644
index 000000000..0c9dd24fb
--- /dev/null
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl
@@ -0,0 +1,8 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverrides" -}}
+  {{- $files := list -}}
+{{/* TODO: containerd is already running when this is set, therefore this is not working {{- range $service := list "containerd" "kubelet" -}}*/}}
+  {{- range $service := list "kubelet" -}}
+    {{- $files = append $files (dict "content" (include (printf "t8s-cluster.clusterClass.node.systemdOverride.%s" $service) (dict)) "path" (printf "/etc/systemd/system/%s.service.d/slice.conf" $service)) -}}
+  {{- end -}}
+  {{- $files | toYaml -}}
+{{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl
new file mode 100644
index 000000000..bbde7e5e3
--- /dev/null
+++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl
@@ -0,0 +1,3 @@
+{{- define "t8s-cluster.clusterClass.systemdOverride.slice.runtime" -}}
+runtime.slice
+{{- end -}}