diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl index b321ee0d4..c82cf1e79 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl @@ -18,7 +18,7 @@ openstack {{- define "t8s-cluster.clusterClass.preKubeadmCommands" -}} {{- $_ := mustMerge . (pick .context "Values") -}} - {{- $commands := list -}} + {{- $commands := list "systemctl stop kubelet.service" "systemctl disable --now snapd.service snapd.socket" -}} {{- if .Values.global.injectedCertificateAuthorities -}} {{- $commands = append $commands "update-ca-certificates" -}} {{- end -}} @@ -26,7 +26,8 @@ openstack {{- end -}} {{- define "t8s-cluster.clusterClass.postKubeadmCommands" -}} - {{- $commands := list -}} + {{/* This is to completely wipe and restart the containerd service in the correct slice. */}} + {{- $commands := list "systemctl reboot" -}} {{- toYaml $commands }} {{- end -}} @@ -109,9 +110,9 @@ server = {{ printf "https://%s" .registry | quote }} {{- define "t8s-cluster.clusterClass.configTemplate.files" -}} {{- $_ := mustMerge . (pick .context "Values") -}} - {{- $files := list -}} + {{- $files := include "t8s-cluster.clusterClass.node.systemdOverrides" (dict) | fromYamlArray -}} {{- if not .excludePatches -}} - {{- $files = concat $files (include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | fromYamlArray) -}} + {{- $files = concat $files (include "t8s-cluster.patches.kubelet" (dict "context" .context) | fromYamlArray) -}} {{- end -}} {{- if .Values.containerRegistryMirror.mirrorEndpoint -}} {{- $files = concat $files (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | fromYamlArray) -}} @@ -120,6 +121,9 @@ server = {{ printf "https://%s" .registry | quote }} {{- if .Values.global.injectedCertificateAuthorities }} {{- $files = append $files (dict "content" .Values.global.injectedCertificateAuthorities "path" "/usr/local/share/ca-certificates/injected-ca-certs.crt" ) -}} {{- end }} + {{- range $file := $files -}} + {{- $_ := set $file "content" (get $file "content" | trim) -}} + {{- end -}} {{- $files | toYaml -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml index 1c5eef299..55562d2f9 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml @@ -1,5 +1,4 @@ {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" -}} - {{- $_ := mustMerge . (pick .context "Values") -}} joinConfiguration: nodeRegistration: kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | nindent 6 }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl index a4cd9de58..6e5dfe4a4 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl @@ -9,7 +9,7 @@ {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.preKubeadmCommands" -}} {{- $_ := mustMerge . (pick .context "Values") -}} {{- $commands := list -}} - {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" -}} + {{- $commands = append $commands "/etc/kube-proxy-patch.sh" -}} {{- if .Values.global.injectedCertificateAuthorities -}} {{- $commands = append $commands "update-ca-certificates" -}} {{- end -}} @@ -28,10 +28,10 @@ {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $file)) "path" $path) -}} {{- end -}} {{- $files = append $files (dict "content" (.Files.Get "files/kube-proxy.patch.sh") "path" "/etc/kube-proxy-patch.sh" "permissions" "0700") -}} + {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}} + {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}} {{- range $file := $files -}} {{- $_ := set $file "content" (get $file "content" | trim) -}} {{- end -}} - {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}} - {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}} {{- $files | toYaml -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl index f2849f0cd..9bf20100a 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl @@ -17,6 +17,20 @@ {{- $options = set $options "protectKernelDefaults" true -}} {{- $options = set $options "tlsCipherSuites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray) -}} {{- $options = set $options "seccompDefault" true -}} + {{- $options = set $options "enforceNodeAllocatable" (list "pods" "system-reserved" "kube-reserved") -}} +{{/* {{- $options = set $options "enforceNodeAllocatable" (list "pods") -}}*/}} + {{- $options = set $options "kubeReservedCgroup" (include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) | trimSuffix ".slice") -}} + {{- $options = set $options "systemReservedCgroup" "system" -}} + {{/* the *Reserved need to include the memory.available, see https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#node-pressure-eviction-good-practices */}} + {{- $options = set $options "kubeReserved" (dict + "memory" "0.6Gi" + "ephemeral-storage" "1Gi" + ) -}} + {{- $options = set $options "systemReserved" (dict + "memory" "0.3Gi" + "ephemeral-storage" "1Gi" + ) -}} + {{- $options = set $options "evictionHard" (dict "memory.available" "128Mi" "nodefs.available" "2Gi") -}} {{- $options | toYaml -}} {{- end -}} @@ -38,7 +52,7 @@ {{- $patches | toYaml -}} {{- end -}} -{{- define "t8s-cluster.patches.kubelet.patches" -}} +{{- define "t8s-cluster.patches.kubelet" -}} {{- $_ := mustMerge . (pick .context "Values") -}} {{- $patches := include "t8s-cluster.patches.kubelet.default" (dict) | fromYamlArray -}} {{- with include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl new file mode 100644 index 000000000..abcabe41d --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_containerd_override.conf.tpl @@ -0,0 +1,4 @@ +{{- define "t8s-cluster.clusterClass.node.systemdOverride.containerd" -}} +[Service] +Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl new file mode 100644 index 000000000..6246c5452 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_kubelet_override.conf.tpl @@ -0,0 +1,4 @@ +{{- define "t8s-cluster.clusterClass.node.systemdOverride.kubelet" -}} +[Service] +Slice={{- include "t8s-cluster.clusterClass.systemdOverride.slice.runtime" (dict) -}} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl new file mode 100644 index 000000000..8ac179034 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_overrides.tpl @@ -0,0 +1,8 @@ +{{- define "t8s-cluster.clusterClass.node.systemdOverrides" -}} + {{- $files := list -}} +{{/* TODO: containerd is already running when this is set, therefore this is not working {{- range $service := list "containerd" "kubelet" -}}*/}} + {{- range $service := list "containerd" "kubelet" -}} + {{- $files = append $files (dict "content" (include (printf "t8s-cluster.clusterClass.node.systemdOverride.%s" $service) (dict)) "path" (printf "/etc/systemd/system/%s.service.d/slice.conf" $service)) -}} + {{- end -}} + {{- $files | toYaml -}} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl new file mode 100644 index 000000000..bbde7e5e3 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/systemdOverrides/_slices.tpl @@ -0,0 +1,3 @@ +{{- define "t8s-cluster.clusterClass.systemdOverride.slice.runtime" -}} +runtime.slice +{{- end -}}