feat(t8s-cluster/management-cluster): add memory safety net for kubelet

charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl

@@ -18,15 +18,16 @@ openstack
 
 {{- define "t8s-cluster.clusterClass.preKubeadmCommands" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
-  {{- $commands := list -}}
+  {{- $commands := list "systemctl stop kubelet.service" "systemctl disable --now snapd.service snapd.socket" -}}
   {{- if .Values.global.injectedCertificateAuthorities -}}
     {{- $commands = append $commands "update-ca-certificates" -}}
   {{- end -}}
   {{- toYaml $commands }}
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.postKubeadmCommands" -}}
-  {{- $commands := list -}}
+  {{/* This is to completely wipe and restart the containerd service in the correct slice. */}}
+  {{- $commands := list "systemctl reboot" -}}
   {{- toYaml $commands }}
 {{- end -}}
 
@@ -109,9 +110,9 @@ server = {{ printf "https://%s" .registry | quote }}
 
 {{- define "t8s-cluster.clusterClass.configTemplate.files" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
-  {{- $files := list -}}
+  {{- $files := include "t8s-cluster.clusterClass.node.systemdOverrides" (dict) | fromYamlArray -}}
   {{- if not .excludePatches -}}
-    {{- $files = concat $files (include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | fromYamlArray) -}}
+    {{- $files = concat $files (include "t8s-cluster.patches.kubelet" (dict "context" .context) | fromYamlArray) -}}
   {{- end -}}
   {{- if .Values.containerRegistryMirror.mirrorEndpoint -}}
     {{- $files = concat $files (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | fromYamlArray) -}}
@@ -120,6 +121,9 @@ server = {{ printf "https://%s" .registry | quote }}
   {{- if .Values.global.injectedCertificateAuthorities }}
     {{- $files = append $files (dict "content" .Values.global.injectedCertificateAuthorities "path" "/usr/local/share/ca-certificates/injected-ca-certs.crt" ) -}}
   {{- end }}
+  {{- range $file := $files -}}
+    {{- $_ := set $file "content" (get $file "content" | trim) -}}
+  {{- end -}}
   {{- $files | toYaml -}}
 {{- end -}}
 

charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml

@@ -1,5 +1,4 @@
 {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" -}}
-  {{- $_ := mustMerge . (pick .context "Values") -}}
 joinConfiguration:
   nodeRegistration:
     kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | nindent 6 }}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl

@@ -9,7 +9,7 @@
 {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.preKubeadmCommands" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $commands := list -}}
-  {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" -}}
+  {{- $commands = append $commands "/etc/kube-proxy-patch.sh" -}}
   {{- if .Values.global.injectedCertificateAuthorities -}}
     {{- $commands = append $commands "update-ca-certificates" -}}
   {{- end -}}
@@ -28,10 +28,10 @@
     {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $file)) "path" $path) -}}
   {{- end -}}
   {{- $files = append $files (dict "content" (.Files.Get "files/kube-proxy.patch.sh") "path" "/etc/kube-proxy-patch.sh" "permissions" "0700") -}}
+  {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
+  {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- range $file := $files -}}
     {{- $_ := set $file "content" (get $file "content" | trim) -}}
   {{- end -}}
-  {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
-  {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- $files | toYaml -}}
 {{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl

@@ -17,6 +17,20 @@
   {{- $options = set $options "protectKernelDefaults" true -}}
   {{- $options = set $options "tlsCipherSuites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray) -}}
   {{- $options = set $options "seccompDefault" true -}}
+  {{- $options = set $options "enforceNodeAllocatable" (list "pods" "system-reserved" "kube-reserved") -}}
+{{/*  {{- $options = set $options "enforceNodeAllocatable" (list "pods") -}}*/}}
+  {{- $options = set $options "kubeReservedCgroup" (include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) | trimSuffix ".slice") -}}
+  {{- $options = set $options "systemReservedCgroup" "system" -}}
+  {{/* the *Reserved need to include the memory.available, see https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#node-pressure-eviction-good-practices */}}
+  {{- $options = set $options "kubeReserved" (dict
+    "memory" "0.6Gi"
+    "ephemeral-storage" "1Gi"
+  ) -}}
+  {{- $options = set $options "systemReserved" (dict
+    "memory" "0.3Gi"
+    "ephemeral-storage" "1Gi"
+  ) -}}
+  {{- $options = set $options "evictionHard" (dict "memory.available" "128Mi" "nodefs.available" "2Gi") -}}
   {{- $options | toYaml -}}
 {{- end -}}
 
@@ -38,7 +52,7 @@
   {{- $patches | toYaml -}}
 {{- end -}}
 
-{{- define "t8s-cluster.patches.kubelet.patches" -}}
+{{- define "t8s-cluster.patches.kubelet" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $patches := include "t8s-cluster.patches.kubelet.default" (dict) | fromYamlArray -}}
   {{- with include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl

@@ -0,0 +1,4 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverride.containerd" -}}
+[Service]
+Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}}
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl

@@ -0,0 +1,4 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverride.kubelet" -}}
+[Service]
+Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}}
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl

@@ -0,0 +1,8 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverrides" -}}
+  {{- $files := list -}}
+{{/* TODO: containerd is already running when this is set, therefore this is not working {{- range $service := list "containerd" "kubelet" -}}*/}}
+  {{- range $service := list "containerd" "kubelet" -}}
+    {{- $files = append $files (dict "content" (include (printf "t8s-cluster.clusterClass.node.systemdOverride.%s" $service) (dict)) "path" (printf "/etc/systemd/system/%s.service.d/slice.conf" $service)) -}}
+  {{- end -}}
+  {{- $files | toYaml -}}
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl

@@ -0,0 +1,3 @@
+{{- define "t8s-cluster.clusterClass.systemdOverride.slice.runtime" -}}
+runtime.slice
+{{- end -}}