feat(t8s-cluster/management-cluster): add memory safety net for kubelet

teutonet · Sep 17, 2024 · 7343072 · 7343072
1 parent 515ce2d
commit 7343072
Show file tree

Hide file tree

Showing 8 changed files with 41 additions and 7 deletions.
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl
@@ -26,7 +26,7 @@ openstack
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.postKubeadmCommands" -}}
-  {{- $commands := list -}}
+  {{- $commands := list "systemctl disable --now snapd.service snapd.socket" -}}
   {{- toYaml $commands }}
 {{- end -}}
 
@@ -109,9 +109,9 @@ server = {{ printf "https://%s" .registry | quote }}
 
 {{- define "t8s-cluster.clusterClass.configTemplate.files" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
-  {{- $files := list -}}
+  {{- $files := include "t8s-cluster.clusterClass.node.systemdOverrides" (dict) | fromYamlArray -}}
   {{- if not .excludePatches -}}
-    {{- $files = concat $files (include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | fromYamlArray) -}}
+    {{- $files = concat $files (include "t8s-cluster.patches.kubelet" (dict "context" .context) | fromYamlArray) -}}
   {{- end -}}
   {{- if .Values.containerRegistryMirror.mirrorEndpoint -}}
     {{- $files = concat $files (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | fromYamlArray) -}}
@@ -120,6 +120,9 @@ server = {{ printf "https://%s" .registry | quote }}
   {{- if .Values.global.injectedCertificateAuthorities }}
     {{- $files = append $files (dict "content" .Values.global.injectedCertificateAuthorities "path" "/usr/local/share/ca-certificates/injected-ca-certs.crt" ) -}}
   {{- end }}
+  {{- range $file := $files -}}
+    {{- $_ := set $file "content" (get $file "content" | trim) -}}
+  {{- end -}}
   {{- $files | toYaml -}}
 {{- end -}}
 

diff --git a/...s/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/...s/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml
@@ -1,5 +1,4 @@
 {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" -}}
-  {{- $_ := mustMerge . (pick .context "Values") -}}
 joinConfiguration:
   nodeRegistration:
     kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | nindent 6 }}

diff --git a/...luster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl b/...luster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl
@@ -28,10 +28,10 @@
     {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $file)) "path" $path) -}}
   {{- end -}}
   {{- $files = append $files (dict "content" (.Files.Get "files/kube-proxy.patch.sh") "path" "/etc/kube-proxy-patch.sh" "permissions" "0700") -}}
+  {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
+  {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- range $file := $files -}}
     {{- $_ := set $file "content" (get $file "content" | trim) -}}
   {{- end -}}
-  {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
-  {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- $files | toYaml -}}
 {{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl
@@ -17,6 +17,19 @@
   {{- $options = set $options "protectKernelDefaults" true -}}
   {{- $options = set $options "tlsCipherSuites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray) -}}
   {{- $options = set $options "seccompDefault" true -}}
+{{/*  {{- $options = set $options "enforceNodeAllocatable" (list "pods" "system-reserved" "kube-reserved") -}}*/}}
+  {{- $options = set $options "enforceNodeAllocatable" (list "pods") -}}
+{{/*  {{- $options = set $options "kubeReservedCgroup" (include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) | trimSuffix ".slice") -}}*/}}
+{{/*  {{- $options = set $options "systemReservedCgroup" "system" -}}*/}}
+  {{- $options = set $options "kubeReserved" (dict
+    "memory" "0.2Gi"
+    "ephemeral-storage" "1Gi"
+  ) -}}
+  {{- $options = set $options "systemReserved" (dict
+    "memory" "0.4Gi"
+    "ephemeral-storage" "1Gi"
+  ) -}}
+  {{- $options = set $options "evictionHard" (dict "memory.available" "64Mi" "nodefs.available" "2Gi") -}}
   {{- $options | toYaml -}}
 {{- end -}}
 
@@ -38,7 +51,7 @@
   {{- $patches | toYaml -}}
 {{- end -}}
 
-{{- define "t8s-cluster.patches.kubelet.patches" -}}
+{{- define "t8s-cluster.patches.kubelet" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $patches := include "t8s-cluster.patches.kubelet.default" (dict) | fromYamlArray -}}
   {{- with include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml -}}

diff --git a/.../templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl b/.../templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl
@@ -0,0 +1,4 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverride.containerd" -}}
+[Service]
+Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}}
+{{- end -}}
diff --git a/...ter/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl b/...ter/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl
@@ -0,0 +1,4 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverride.kubelet" -}}
+[Service]
+Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}}
+{{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl
@@ -0,0 +1,8 @@
+{{- define "t8s-cluster.clusterClass.node.systemdOverrides" -}}
+  {{- $files := list -}}
+{{/* TODO: containerd is already running when this is set, therefore this is not working {{- range $service := list "containerd" "kubelet" -}}*/}}
+  {{- range $service := list "kubelet" -}}
+    {{- $files = append $files (dict "content" (include (printf "t8s-cluster.clusterClass.node.systemdOverride.%s" $service) (dict)) "path" (printf "/etc/systemd/system/%s.service.d/slice.conf" $service)) -}}
+  {{- end -}}
+  {{- $files | toYaml -}}
+{{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl
@@ -0,0 +1,3 @@
+{{- define "t8s-cluster.clusterClass.systemdOverride.slice.runtime" -}}
+runtime.slice
+{{- end -}}