diff --git a/charts/t8s-cluster/ci/hcp-values.yaml b/charts/t8s-cluster/ci/hcp-values.yaml new file mode 100644 index 000000000..fa58e7542 --- /dev/null +++ b/charts/t8s-cluster/ci/hcp-values.yaml @@ -0,0 +1,2 @@ +controlPlane: + hosted: true diff --git a/charts/t8s-cluster/templates/_etcd-defrag.yaml b/charts/t8s-cluster/templates/_etcd-defrag.yaml new file mode 100644 index 000000000..529315d1f --- /dev/null +++ b/charts/t8s-cluster/templates/_etcd-defrag.yaml @@ -0,0 +1,89 @@ +{{- define "t8s-cluster.etcd-defrag" -}} + {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: kube-etcd-defrag + namespace: {{ .hosted | ternary .Release.Namespace "kube-system" }} + labels: {{- include "common.labels.standard" . | nindent 4 }} +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 10 + successfulJobsHistoryLimit: 1 + schedule: '42 * * * *' + jobTemplate: + spec: + backoffLimit: 6 + template: + spec: + automountServiceAccountToken: false + containers: + - command: + - etcdctl + - defrag + - --cluster + - --cacert=/etc/kubernetes/pki/etcd/ca.crt + - --cert=/etc/kubernetes/pki/etcd/peer.crt + - --key=/etc/kubernetes/pki/etcd/peer.key + env: + - name: ETCDCTL_API + value: "3" + - name: ETCDCTL_ENDPOINTS + value: {{ .hosted | ternary (printf "kmc-%s-etcd:2379" .Release.Name) "localhost:2379" }} + image: {{ include "common.images.image" (dict "imageRoot" .Values.global.etcd.image "global" .Values.global) }} + imagePullPolicy: IfNotPresent + name: etcd-defrag + securityContext: + runAsGroup: 1000 + runAsUser: {{ .hosted | ternary 1000 0 }} + runAsNonRoot: {{ .hosted }} + privileged: false + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + readOnly: true + dnsPolicy: ClusterFirst + restartPolicy: OnFailure + {{- if not .hosted }} + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + {{- end }} + volumes: + - name: etcd-certs + {{- if .hosted }} + projected: + defaultMode: 420 + sources: + - secret: + items: + - key: tls.crt + path: ca.crt + name: {{ printf "%s-etcd" .Release.Name }} + - secret: + items: + - key: tls.crt + path: peer.crt + - key: tls.key + path: peer.key + name: {{ printf "%s-etcd-peer" .Release.Name }} + {{- else }} + hostPath: + path: /etc/kubernetes/pki/etcd + type: Directory + {{- end }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl index 6443c7c8b..b321ee0d4 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl @@ -13,13 +13,12 @@ openstack {{- define "t8s-cluster.clusterClass.tlsCipherSuites" -}} {{- $cipherSuites := list "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305" "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" "TLS_RSA_WITH_AES_256_GCM_SHA384" "TLS_RSA_WITH_AES_128_GCM_SHA256" -}} - {{- join "," $cipherSuites -}} + {{- $cipherSuites | toYaml -}} {{- end -}} {{- define "t8s-cluster.clusterClass.preKubeadmCommands" -}} {{- $_ := mustMerge . (pick .context "Values") -}} {{- $commands := list -}} - {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" }} {{- if .Values.global.injectedCertificateAuthorities -}} {{- $commands = append $commands "update-ca-certificates" -}} {{- end -}} @@ -35,3 +34,137 @@ openstack {{- $args := dict "cloud-provider" "external" -}} {{- toYaml $args -}} {{- end -}} + +{{- define "t8s-cluster.clusterClass.containerdConfig.plugins" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + {{- if .Values.containerRegistryMirror.mirrorEndpoint }} + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/registries.conf.d" + {{- end }} + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + # TODO: this is only needed because of https://github.com/containerd/containerd/issues/5837 + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + # TODO: this is only needed because of https://github.com/containerd/containerd/issues/5837 + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + {{- if .gpu }} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + privileged_without_host_devices = false + runtime_engine = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/local/nvidia/toolkit/nvidia-container-runtime" + {{- end -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs.content" -}} +server = {{ printf "https://%s" .registry | quote }} +{{ printf `[host."%s"]` .endpoint }} + capabilities = ["pull", "resolve"] +{{- end -}} + +{{- define "t8s-cluster.featureGates" -}} +{{/* {{- (dict "SeccompDefault" (list "kubelet")) | toYaml -}}*/}} + {{- (dict) | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.featureGates.forComponent" -}} + {{- $featureGates := dict -}} + {{- $component := .component -}} + {{- range $featureGate, $components := include "t8s-cluster.featureGates" (dict) | fromYaml -}} + {{- if $components | has $component -}} + {{- $featureGates = set $featureGates $featureGate true -}} + {{- end -}} + {{- end -}} + {{- $featureGates | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} + {{- $defaultMirroredRegistries := list + "gcr.io" + "ghcr.io" + "k8s.gcr.io" + "nvcr.io" + "quay.io" + "registry.gitlab.com" + "registry.k8s.io" + "registry.opensource.zalan.do" + "registry.teuto.io" + -}} + {{- $mirroredRegistries := concat $defaultMirroredRegistries (.Values.containerRegistryMirror.additionallyMirroredRegistries | default list) | sortAlpha | uniq -}} + {{- $files := list -}} + {{- range $registry := $mirroredRegistries }} + {{- $files = append $files (dict "content" (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs.content" (dict "registry" $registry "endpoint" $.Values.containerRegistryMirror.mirrorEndpoint)) "path" (printf `/etc/containerd/registries.conf.d/%s/hosts.toml` $registry)) -}} + {{- end }} + {{- $files = append $files (dict "content" (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs.content" (dict "registry" "registry-1.docker.io" "endpoint" $.Values.containerRegistryMirror.mirrorEndpoint)) "path" "/etc/containerd/registries.conf.d/docker.io/hosts.toml") -}} + {{- $files | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.configTemplate.files" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} + {{- $files := list -}} + {{- if not .excludePatches -}} + {{- $files = concat $files (include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | fromYamlArray) -}} + {{- end -}} + {{- if .Values.containerRegistryMirror.mirrorEndpoint -}} + {{- $files = concat $files (include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | fromYamlArray) -}} + {{- end -}} + {{- $files = append $files (dict "content" (include "t8s-cluster.clusterClass.containerdConfig.plugins" (dict "context" .context "gpu" .gpu)) "path" "/etc/containerd/conf.d/plugins.toml" ) -}} + {{- if .Values.global.injectedCertificateAuthorities }} + {{- $files = append $files (dict "content" .Values.global.injectedCertificateAuthorities "path" "/usr/local/share/ca-certificates/injected-ca-certs.crt" ) -}} + {{- end }} + {{- $files | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.args.base" -}} + {{- dict "profiling" "false" | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.args.shared" -}} + {{- $args := include "t8s-cluster.clusterClass.args.base" (dict) | fromYaml -}} + {{- $args = mustMerge (dict + "authorization-always-allow-paths" (list "/healthz" "/readyz" "/livez" "/metrics" | join ",") + "bind-address" "0.0.0.0" + ) $args -}} + {{- $args | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.args.scheduler" -}} + {{- include "t8s-cluster.clusterClass.args.shared" (dict) -}} +{{- end }} + +{{- define "t8s-cluster.clusterClass.args.sharedController" -}} + {{- $args := dict "cloud-provider" "external" -}} + {{- toYaml $args -}} +{{- end }} + +{{- define "t8s-cluster.clusterClass.args.controllerManager" -}} + {{- $args := include "t8s-cluster.clusterClass.args.shared" (dict) | fromYaml -}} + {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}} + {{- $args = set $args "terminated-pod-gc-threshold" "100" -}} + {{- $args | toYaml -}} +{{- end }} + +{{- define "t8s-cluster.clusterClass.apiServer.admissionPlugins" -}} + {{- $admissionPlugins := list "AlwaysPullImages" "NodeRestriction" -}} + {{- if not .excludePatches -}} + {{- $admissionPlugins = concat $admissionPlugins (list "EventRateLimit") -}} + {{- end -}} + {{- $admissionPlugins | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.args.apiServer" -}} + {{- $args := include "t8s-cluster.clusterClass.args.base" (dict "context" .context) | fromYaml -}} + {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}} + {{- $args = set $args "enable-admission-plugins" (include "t8s-cluster.clusterClass.apiServer.admissionPlugins" (dict "excludePatches" .excludePatches) | fromYamlArray | join ",") -}} + {{- $args = set $args "event-ttl" "4h" -}} + {{- $args = set $args "tls-cipher-suites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray | join ",") -}} + {{- $args | toYaml -}} +{{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml new file mode 100644 index 000000000..b113f0c50 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml @@ -0,0 +1,12 @@ +{{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate" -}} + {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: {{ .Values.controlPlane.hosted | ternary "K0sWorkerConfigTemplate" "KubeadmConfigTemplate" }} +metadata: + name: {{ printf "%s-%s-compute-plane" .Release.Name (.gpu | ternary "gpu" "standard") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "common.labels.standard" . | nindent 4 }} +spec: + template: + spec: {{- include (printf "t8s-cluster.clusterClass.bootstrapConfigTemplate.%s.spec" (.Values.controlPlane.hosted | ternary "k0smotron" "kubeadm")) (dict "gpu" .gpu "context" .context) | nindent 6 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_k0smotronConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_k0smotronConfigTemplateSpec.yaml new file mode 100644 index 000000000..f987219cf --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_k0smotronConfigTemplateSpec.yaml @@ -0,0 +1,23 @@ +{{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.k0smotron.spec" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} + {{- $kubeletExtraArgs := list -}} + {{- range $key, $value := include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | fromYaml -}} + {{- $kubeletExtraArgs = append $kubeletExtraArgs (printf "--%s=%s" $key $value) -}} + {{- end -}} +args: + - --cri-socket=remote:/run/containerd/containerd.sock + {{- if le (.Values.version.minor | int) 28 }} + - --enable-cloud-provider + {{- end }} + - {{ printf `--kubelet-extra-args="%s"` ($kubeletExtraArgs | join " ") }} +files: {{- include "t8s-cluster.clusterClass.configTemplate.files" (dict "context" .context "gpu" .gpu "excludePatches" true) | nindent 2 }} + {{/* this service isn't needed by k0s */}} + {{- $preStartCommands := list + "systemctl disable --now kubelet.service" + }} + {{- $preStartCommands = concat $preStartCommands + (include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" .context) | fromYamlArray) + (include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | fromYamlArray) + }} +preStartCommands: {{- $preStartCommands | default (list) | toYaml | nindent 2 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml new file mode 100644 index 000000000..1c5eef299 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml @@ -0,0 +1,11 @@ +{{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} +joinConfiguration: + nodeRegistration: + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | nindent 6 }} + patches: + directory: /etc/kubernetes/patches +files: {{- include "t8s-cluster.clusterClass.configTemplate.files" (dict "context" .context "gpu" .gpu) | nindent 2 }} +preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" .context) | nindent 2 }} +postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 2 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/gpuNodePoolBootstrapConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/gpuNodePoolBootstrapConfigTemplate.yaml new file mode 100644 index 000000000..4ec89c993 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/gpuNodePoolBootstrapConfigTemplate.yaml @@ -0,0 +1 @@ +{{- include "t8s-cluster.clusterClass.bootstrapConfigTemplate" (dict "gpu" true "context" $) -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/standardNodePoolBootstrapConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/standardNodePoolBootstrapConfigTemplate.yaml new file mode 100644 index 000000000..ad515cbbc --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/standardNodePoolBootstrapConfigTemplate.yaml @@ -0,0 +1 @@ +{{- include "t8s-cluster.clusterClass.bootstrapConfigTemplate" (dict "gpu" false "context" $) -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml index 2cf4fd8e3..0736b5d3e 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml @@ -10,6 +10,7 @@ metadata: labels: {{- include "common.labels.standard" $ | nindent 4 }} spec: controlPlane: + {{- if not .Values.controlPlane.hosted }} machineHealthCheck: maxUnhealthy: 1 nodeStartupTimeout: 10m @@ -26,10 +27,16 @@ spec: apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }} + {{- end }} ref: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + {{- if .Values.controlPlane.hosted }} + kind: K0smotronControlPlaneTemplate + name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" .) }} + {{- else }} kind: KubeadmControlPlaneTemplate name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} + {{- end }} infrastructure: ref: apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} @@ -72,6 +79,7 @@ spec: type: string default: compute-plane-placeholder patches: + {{- if not .Values.controlPlane.hosted }} - name: controlPlaneServerGroupID description: Sets the ServerGroupID for Control Plane machines. definitions: @@ -85,6 +93,7 @@ spec: kind: OpenStackMachineTemplate matchResources: controlPlane: true + {{- end }} - name: machineDeploymentServerGroupID description: Sets the ServerGroupID for MachineDeployment machines. definitions: @@ -127,6 +136,7 @@ spec: matchResources: machineDeploymentClass: names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }} + {{- if not .Values.controlPlane.hosted }} - jsonPatches: - <<: *imagePatch valueFrom: @@ -135,6 +145,8 @@ spec: <<: *imagePatchSelector matchResources: controlPlane: true + {{- end }} + {{- if not .Values.controlPlane.hosted }} - name: controlPlaneAvailabilityZones enabledIf: {{ `{{ if .controlPlaneAvailabilityZones }}true{{ end }}` | quote }} definitions: @@ -148,6 +160,7 @@ spec: path: /spec/template/spec/controlPlaneAvailabilityZones valueFrom: variable: controlPlaneAvailabilityZones + {{- end }} - name: dnsNameservers definitions: - selector: @@ -190,7 +203,7 @@ spec: bootstrap: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate + kind: {{ $.Values.controlPlane.hosted | ternary "K0sWorkerConfigTemplate" "KubeadmConfigTemplate" }} name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }} infrastructure: ref: diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl new file mode 100644 index 000000000..afc43df92 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" -}} + {{/* the full context is needed for .Files.Get */}} + {{- $inputs := (dict + "spec" (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" $) + ) -}} + {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml new file mode 100644 index 000000000..2c77521b5 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml @@ -0,0 +1,53 @@ +{{/* +KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades. +Here we are generating a hash suffix. +This function needs the whole `$` context to be able to use `.Files.Get` +*/}} +{{- define "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" -}} +replicas: {{ $.Values.controlPlane.singleNode | ternary 1 3 }} +service: + type: LoadBalancer + apiPort: 6443 + konnectivityPort: 8132 +controllerPlaneFlags: + - --disable-components=metrics-server +k0sConfig: + apiVersion: k0s.k0sproject.io/v1beta1 + kind: ClusterConfig + spec: + network: + provider: custom + api: + extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" . "excludePatches" true) | nindent 8 }} + # TODO: figure out how to make this work + # admission-control-config-file: &admissionControlConfigFilePath /etc/kubernetes/admission-control-config.yaml + # extraVolumes: + # - hostPath: *admissionControlConfigFilePath + # mountPath: *admissionControlConfigFilePath + # name: admission-control-config + # readOnly: true + # - hostPath: &eventRateLimitConfigFilePath {\{ $eventRateLimitConfigFilePath }} + # mountPath: *eventRateLimitConfigFilePath + # name: event-rate-limit-config + # readOnly: true + controllerManager: + extraArgs: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 8 }} + scheduler: + extraArgs: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 8 }} + telemetry: + enabled: false + workerProfiles: + - name: default + {{- $values := dict "cgroupDriver" "systemd" }} + {{ $values = mustMerge $values (include "t8s-cluster.kubelet.options" (dict) | fromYaml) }} + {{ $values = mustMerge $values (include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .) | fromYaml) }} + values: {{- $values | toYaml | nindent 10 }} + featureGates: {{- range $featureGate, $components := include "t8s-cluster.featureGates" (dict) | fromYaml }} + - name: {{ $featureGate }} + enabled: true + components: {{- $components | toYaml | nindent 10 }} + {{- end }} +etcd: + persistence: + size: 5Gi +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml new file mode 100644 index 000000000..d88339b81 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml @@ -0,0 +1,11 @@ +{{- if .Values.controlPlane.hosted -}} +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: K0smotronControlPlaneTemplate +metadata: + name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" .) }} + namespace: {{ $.Release.Namespace }} + labels: {{- include "common.labels.standard" $ | nindent 4 }} +spec: + template: + spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" . | nindent 6 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_helpers.tpl deleted file mode 100644 index e265e488b..000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_helpers.tpl +++ /dev/null @@ -1,55 +0,0 @@ -{{- define "t8s-cluster.clusterClass.containerdConfig.plugins" -}} - {{- $_ := mustMerge . (pick .context "Values") -}} -[plugins] - [plugins."io.containerd.grpc.v1.cri"] - {{- if .Values.containerRegistryMirror.mirrorEndpoint }} - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/registries.conf.d" - {{- end }} - [plugins."io.containerd.grpc.v1.cri".containerd] - default_runtime_name = "runc" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - # TODO: this is only needed because of https://github.com/containerd/containerd/issues/5837 - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - # TODO: this is only needed because of https://github.com/containerd/containerd/issues/5837 - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - {{- if .gpu }} - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] - privileged_without_host_devices = false - runtime_engine = "" - runtime_root = "" - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] - BinaryName = "/usr/local/nvidia/toolkit/nvidia-container-runtime" - {{- end -}} -{{- end -}} - -{{- define "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" -}} - {{- $_ := mustMerge . (pick .context "Values") -}} - {{- $defaultMirroredRegistries := list - "gcr.io" - "ghcr.io" - "k8s.gcr.io" - "nvcr.io" - "quay.io" - "registry.gitlab.com" - "registry.k8s.io" - "registry.opensource.zalan.do" - "registry.teuto.io" - -}} - {{- $mirroredRegistries := concat $defaultMirroredRegistries (.Values.containerRegistryMirror.additionallyMirroredRegistries | default list) | sortAlpha | uniq -}} - {{- range $registry := $mirroredRegistries }} -- content: |- - server = {{ printf "https://%s" $registry | quote }} - {{ printf `[host."%s"]` $.Values.containerRegistryMirror.mirrorEndpoint }} - capabilities = ["pull", "resolve"] - path: {{ printf `/etc/containerd/registries.conf.d/%s/hosts.toml` $registry }} - {{- end }} -- content: |- - server = "registry-1.docker.io" - {{ printf `[host."%s"]` $.Values.containerRegistryMirror.mirrorEndpoint }} - capabilities = ["pull", "resolve"] - path: /etc/containerd/registries.conf.d/docker.io/hosts.toml -{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml deleted file mode 100644 index aecff2248..000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- define "t8s-cluster.clusterClass.kubeadmConfigTemplate.spec" -}} - {{- $_ := mustMerge . (pick .context "Values") -}} -joinConfiguration: - nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }} - name: '{{ `{{ local_hostname }}` }}' - patches: - directory: /etc/kubernetes/patches -files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | nindent 2 }} - {{- if .Values.containerRegistryMirror.mirrorEndpoint }} - {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | nindent 2 }} - {{- end }} - - content: |- {{- include "t8s-cluster.clusterClass.containerdConfig.plugins" (dict "context" .context "gpu" .gpu) | nindent 6 }} - path: /etc/containerd/conf.d/plugins.toml - {{- if .Values.global.injectedCertificateAuthorities }} - - content: |- {{- .Values.global.injectedCertificateAuthorities | nindent 6 }} - path: /usr/local/share/ca-certificates/injected-ca-certs.crt - {{- end }} - {{ if .Values.global.injectedCertificateAuthorities }} -preKubeadmCommands: - - update-ca-certificates - {{- end }} -postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 2 }} -{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml deleted file mode 100644 index 03238cae4..000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- define "t8s-cluster.clusterClass.kubeadmConfigTemplate" -}} - {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: {{ printf "%s-%s-compute-plane" .Release.Name (.gpu | ternary "gpu" "standard") }} - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} -spec: - template: - spec: {{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate.spec" (dict "gpu" .gpu "context" .context) | nindent 6 }} -{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml deleted file mode 100644 index 101a690a1..000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml +++ /dev/null @@ -1 +0,0 @@ -{{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate" (dict "gpu" true "context" $) -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml deleted file mode 100644 index 2c91ab61f..000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml +++ /dev/null @@ -1 +0,0 @@ -{{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate" (dict "gpu" false "context" $) -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl index 272342781..a4cd9de58 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl @@ -5,3 +5,33 @@ ) -}} {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}} {{- end -}} + +{{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.preKubeadmCommands" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} + {{- $commands := list -}} + {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" -}} + {{- if .Values.global.injectedCertificateAuthorities -}} + {{- $commands = append $commands "update-ca-certificates" -}} + {{- end -}} + {{- toYaml $commands }} +{{- end -}} + +{{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.files" -}} + {{- $files := list -}} + {{- $files = concat $files (include "t8s-cluster.clusterClass.configTemplate.files" (dict "context" . "gpu" false) | fromYamlArray) -}} + {{- $configs := dict + "admission-control-config.yaml" (required "Missing" .admissionControlConfigFilePath) + "event-rate-limit-config.yaml" (required "Missing" .eventRateLimitConfigFilePath) + "kube-proxy.config.yaml" "/etc/kube-proxy-config.yaml" + -}} + {{- range $file, $path := $configs -}} + {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $file)) "path" $path) -}} + {{- end -}} + {{- $files = append $files (dict "content" (.Files.Get "files/kube-proxy.patch.sh") "path" "/etc/kube-proxy-patch.sh" "permissions" "0700") -}} + {{- range $file := $files -}} + {{- $_ := set $file "content" (get $file "content" | trim) -}} + {{- end -}} + {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}} + {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}} + {{- $files | toYaml -}} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml index 28f897e22..133d59f3e 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml @@ -4,76 +4,39 @@ Here we are generating a hash suffix. This function needs the whole `$` context to be able to use `.Files.Get` */}} {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" -}} + {{- $admissionControlConfigFilePath := "/etc/kubernetes/admission-control-config.yaml" -}} + {{- $eventRateLimitConfigFilePath := "/etc/kubernetes/event-rate-limit-config.yaml" -}} rolloutBefore: certificatesExpiryDays: 60 kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: - admission-control-config-file: &admissionControlConfigFilePath /etc/kubernetes/admission-control-config.yaml - cloud-provider: external - enable-admission-plugins: AlwaysPullImages,EventRateLimit,NodeRestriction - profiling: 'false' - tls-cipher-suites: {{ include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) }} - event-ttl: 4h + extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 8 }} + admission-control-config-file: &admissionControlConfigFilePath {{ $admissionControlConfigFilePath }} extraVolumes: - hostPath: *admissionControlConfigFilePath mountPath: *admissionControlConfigFilePath name: admission-control-config readOnly: true - - hostPath: &eventRateLimitConfigFilePath /etc/kubernetes/event-rate-limit-config.yaml + - hostPath: &eventRateLimitConfigFilePath {{ $eventRateLimitConfigFilePath }} mountPath: *eventRateLimitConfigFilePath name: event-rate-limit-config readOnly: true controllerManager: - extraArgs: - authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics - bind-address: 0.0.0.0 - cloud-provider: external - profiling: 'false' - terminated-pod-gc-threshold: '100' + extraArgs: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 8 }} etcd: local: extraArgs: listen-metrics-urls: http://0.0.0.0:2381 scheduler: - extraArgs: - authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics - bind-address: 0.0.0.0 - profiling: 'false' - files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" $) | nindent 4 }} - - content: |- {{- .Files.Get "files/admission-control-config.yaml" | nindent 8 }} - path: *admissionControlConfigFilePath - - content: |- {{- .Files.Get "files/event-rate-limit-config.yaml" | nindent 8 }} - path: *eventRateLimitConfigFilePath - - content: |- {{- .Files.Get "files/kube-proxy.patch.sh" | nindent 8 }} - path: /etc/kube-proxy-patch.sh - permissions: "0700" - - content: |- {{- .Files.Get "files/kube-proxy.config.yaml" | nindent 8 }} - path: /etc/kube-proxy-config.yaml - {{- if .Values.containerRegistryMirror.mirrorEndpoint }} - {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" $) | nindent 4 }} - {{- end }} - - content: |- {{- include "t8s-cluster.clusterClass.containerdConfig.plugins" (dict "context" $ "gpu" false) | nindent 8 }} - path: /etc/containerd/conf.d/plugins.toml - {{- if .Values.global.injectedCertificateAuthorities }} - - content: |- {{- .Values.global.injectedCertificateAuthorities | nindent 8 }} - path: /usr/local/share/ca-certificates/injected-ca-certs.crt - {{- end }} - {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) }} - {{- list (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) | toYaml | nindent 4 }} - initConfiguration: + extraArgs: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 8 }} + files: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.files" (merge . (dict "admissionControlConfigFilePath" $admissionControlConfigFilePath "eventRateLimitConfigFilePath" $eventRateLimitConfigFilePath)) | nindent 4 }} + initConfiguration: &configuration nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 8 }} - name: '{{ `{{ local_hostname }}` }}' + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .) | nindent 8 }} patches: directory: {{ include "t8s-cluster.patches.directory" (dict) }} - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 8 }} - name: '{{ `{{ local_hostname }}` }}' - patches: - directory: {{ include "t8s-cluster.patches.directory" (dict) }} - preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" $) | nindent 4 }} + joinConfiguration: *configuration + preKubeadmCommands: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.preKubeadmCommands" (dict "context" .) | nindent 4 }} postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 4 }} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml index 1022d7498..997a2b92b 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.controlPlane.hosted -}} apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: @@ -7,3 +8,4 @@ metadata: spec: template: spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 6 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml index 871bb180b..ad3bb9484 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml @@ -5,10 +5,11 @@ Here we are generating a hash suffix. {{- define "t8s-cluster.clusterClass.openStackClusterTemplate.spec" -}} {{- $_ := mustMerge . (pick .context "Values") -}} apiServerLoadBalancer: - enabled: true + enabled: {{ not .Values.controlPlane.hosted }} {{- with .Values.controlPlane.allowedCIDRs }} allowedCidrs: {{- toYaml . | nindent 4 }} {{- end }} +disableAPIServerFloatingIP: {{ .Values.controlPlane.hosted }} bastion: availabilityZone: {{ .Values.bastion.availabilityZone }} enabled: {{ .Values.bastion.enabled }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml index 098a8b7b4..dc8c0b087 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml @@ -1,9 +1,10 @@ {{- range $name := list "compute-plane" "control-plane" }} -{{- if false }} + {{- if or (ne $name "control-plane") (not $.Values.controlPlane.hosted) }} + {{- if false }} apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 -{{- else }} + {{- else }} apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} -{{- end }} + {{- end }} kind: OpenStackMachineTemplate metadata: name: {{ printf "%s-%s-%s" $.Release.Name $name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "name" $name "context" $)) }} @@ -13,4 +14,5 @@ spec: template: spec: {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.spec" (dict "name" $name "context" $) | nindent 6 }} --- -{{- end }} + {{- end }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl index e6f609c44..1f064281a 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl @@ -1,23 +1,39 @@ {{- define "t8s-cluster.patches.kubelet.imagePulls" -}} {{- $_ := mustMerge . (pick .context "Values") -}} - {{- include "t8s-cluster.patches.patchFile" (dict "values" (dict "serializeImagePulls" false "maxParallelImagePulls" .Values.global.kubeletExtraConfig.maxParallelImagePulls) "target" "kubeletconfiguration" "component" "imagePulls") -}} + {{- $values := dict -}} + {{- if and (or (gt (.Values.version.major | int) 1) (ge (.Values.version.minor | int) 27)) (gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1) -}} + {{- $values = mustMerge $values (dict "serializeImagePulls" false "maxParallelImagePulls" .Values.global.kubeletExtraConfig.maxParallelImagePulls) -}} + {{- end -}} + {{- $values | toYaml -}} +{{- end -}} + +{{- define "t8s-cluster.kubelet.featureGates"}} + {{- include "t8s-cluster.featureGates.forComponent" (dict "component" "kubelet") -}} +{{- end -}} + +{{- define "t8s-cluster.kubelet.options" -}} + {{- $options := dict -}} + {{- $options = set $options "eventRecordQPS" 0 -}} + {{- $options = set $options "protectKernelDefaults" true -}} + {{- $options = set $options "tlsCipherSuites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray) -}} + {{- $options = set $options "seccompDefault" true -}} + {{- $options | toYaml -}} {{- end -}} {{- define "t8s-cluster.patches.kubelet.default" -}} {{- $values := dict -}} - {{- $values = set $values "eventRecordQPS" 0 -}} - {{- $values = set $values "protectKernelDefaults" true -}} - {{- $values = set $values "featureGates" (dict) -}} - {{- $values = set $values "SeccompDefault" true -}} - {{- $values = set $values "tlsCipherSuites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | splitList ",") -}} + {{- range $option, $value := include "t8s-cluster.kubelet.options" (dict) | fromYaml -}} + {{- $values = set $values $option $value -}} + {{- end -}} + {{- $values = set $values "featureGates" (include "t8s-cluster.kubelet.featureGates" (dict) | fromYaml) -}} {{- include "t8s-cluster.patches.patchFile" (dict "values" $values "target" "kubeletconfiguration" "component" "default") -}} {{- end -}} {{- define "t8s-cluster.patches.kubelet.patches" -}} {{- $_ := mustMerge . (pick .context "Values") -}} {{- $patches := list (include "t8s-cluster.patches.kubelet.default" (dict) | fromYaml) -}} - {{- if and (eq (int .Values.version.major) 1) (ge (int .Values.version.minor) 27) (gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1) -}} - {{- $patches = append $patches (include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml) -}} + {{- with include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml -}} + {{- $patches = append $patches (include "t8s-cluster.patches.patchFile" (dict "values" . "target" "kubeletconfiguration" "component" "imagePulls") | fromYaml) -}} {{- end -}} {{- toYaml $patches -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_patches.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_patches.tpl index 734786b23..d83737c0a 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_patches.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_patches.tpl @@ -1,6 +1,5 @@ {{- define "t8s-cluster.patches.patchFile" -}} -content: |- {{- toYaml .values | nindent 2 }} -path: {{ printf "%s/%s-%s.yaml" (include "t8s-cluster.patches.directory" (dict)) .target .component }} + {{- dict "content" (toYaml .values) "path" (printf "%s/%s-%s.yaml" (include "t8s-cluster.patches.directory" (dict)) .target .component) | toYaml -}} {{- end -}} {{- define "t8s-cluster.patches.directory" -}} diff --git a/charts/t8s-cluster/templates/management-cluster/etcd-defrag.yaml b/charts/t8s-cluster/templates/management-cluster/etcd-defrag.yaml new file mode 100644 index 000000000..ae78f5077 --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/etcd-defrag.yaml @@ -0,0 +1,3 @@ +{{- if .Values.controlPlane.hosted -}} + {{- include "t8s-cluster.etcd-defrag" (dict "context" $ "hosted" true) | nindent 0 -}} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/repositories/helmRepositories.yaml b/charts/t8s-cluster/templates/management-cluster/repositories/helmRepositories.yaml index f504aa1bc..b1ffd99d2 100644 --- a/charts/t8s-cluster/templates/management-cluster/repositories/helmRepositories.yaml +++ b/charts/t8s-cluster/templates/management-cluster/repositories/helmRepositories.yaml @@ -4,7 +4,7 @@ {{- $create = eq (include "common.tplvalues.render" (dict "value" $config.condition "context" (deepCopy $))) "true" -}} {{- end -}} {{- if $create -}} -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: {{ printf "%s-%s" $.Release.Name $name | quote }} diff --git a/charts/t8s-cluster/templates/workload-cluster/_helmRelease.yaml b/charts/t8s-cluster/templates/workload-cluster/_helmRelease.yaml index 07092fd22..0ddcddc31 100644 --- a/charts/t8s-cluster/templates/workload-cluster/_helmRelease.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/_helmRelease.yaml @@ -1,6 +1,6 @@ {{- define "t8s-cluster.helm.resourceIntoCluster" -}} {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta2 +apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: {{ printf "%s-%s" .Release.Name .name }} diff --git a/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin.yaml b/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin.yaml index 5b7705f33..be81e12fb 100644 --- a/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin.yaml @@ -43,6 +43,11 @@ spec: name: cloud-config csi: plugin: + # TODO: keep this until https://github.com/k0sproject/k0s/issues/1842 is addressed + {{- if .Values.controlPlane.hosted }} + nodePlugin: + kubeletDir: /var/lib/k0s/kubelet + {{- end }} controllerPlugin: tolerations: - effect: NoSchedule diff --git a/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml b/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml index 9ef699624..11d6d4629 100644 --- a/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml @@ -21,9 +21,6 @@ spec: - name: {{ printf "%s-cni" .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} - kubeConfig: - secretRef: - name: {{ .Release.Name }}-kubeconfig install: remediation: retries: -1 @@ -34,6 +31,9 @@ spec: storageNamespace: kube-system targetNamespace: kube-system releaseName: ccm + kubeConfig: + secretRef: + name: {{ .Release.Name }}-kubeconfig valuesFrom: - &valuesFrom kind: Secret @@ -43,6 +43,17 @@ spec: valuesKey: cloud.conf - <<: *valuesFrom targetPath: cloudConfig.metadata.hash + {{- if .Values.controlPlane.hosted }} + postRenderers: + - kustomize: + patches: + - target: + kind: DaemonSet + patch: | + - op: replace + path: /kind + value: Deployment + {{- end }} values: cluster: name: {{ .Values.metadata.friendlyName | default .Release.Name }} @@ -51,8 +62,14 @@ spec: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule + {{- if not .Values.controlPlane.hosted }} - key: node-role.kubernetes.io/control-plane effect: NoSchedule nodeSelector: node-role.kubernetes.io/control-plane: "" + {{- end }} + {{- else }} + {{- if .Values.controlPlane.hosted }} + nodeSelector: null + {{- end }} {{- end }} diff --git a/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml b/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml index ce8ce0f35..dc852e575 100644 --- a/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml @@ -1,58 +1,3 @@ -{{- include "t8s-cluster.helm.resourceIntoCluster" (dict "name" "etcd-defrag" "resource" (include "t8s-cluster.etcd-defrag" (dict "context" $)) "context" $ "additionalLabels" (dict "app.kubernetes.io/component" "etcd")) | nindent 0 }} - -{{- define "t8s-cluster.etcd-defrag" -}} - {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} -apiVersion: batch/v1 -kind: CronJob -metadata: - name: kube-etcd-defrag - namespace: kube-system - labels: {{- include "common.labels.standard" . | nindent 4 }} -spec: - concurrencyPolicy: Forbid - failedJobsHistoryLimit: 10 - successfulJobsHistoryLimit: 1 - schedule: '42 * * * *' - jobTemplate: - spec: - backoffLimit: 6 - template: - spec: - containers: - - command: - - etcdctl - - defrag - - --cluster - - --cacert=/etc/kubernetes/pki/etcd/ca.crt - - --cert=/etc/kubernetes/pki/etcd/peer.crt - - --key=/etc/kubernetes/pki/etcd/peer.key - env: - - name: ETCDCTL_API - value: "3" - - name: ETCDCTL_ENDPOINTS - value: 'localhost:2379' - image: {{ include "common.images.image" (dict "imageRoot" .Values.global.etcd.image "global" .Values.global) }} - imagePullPolicy: IfNotPresent - name: etcd-defrag - volumeMounts: - - mountPath: /etc/kubernetes/pki/etcd - name: etcd-certs - readOnly: true - dnsPolicy: ClusterFirst - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/control-plane: "" - restartPolicy: OnFailure - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - volumes: - - hostPath: - path: /etc/kubernetes/pki/etcd - type: Directory - name: etcd-certs +{{- if not .Values.controlPlane.hosted -}} + {{- include "t8s-cluster.helm.resourceIntoCluster" (dict "name" "etcd-defrag" "resource" (include "t8s-cluster.etcd-defrag" (dict "context" $ "hosted" false)) "context" $ "additionalLabels" (dict "app.kubernetes.io/component" "etcd")) | nindent 0 -}} {{- end -}} diff --git a/charts/t8s-cluster/values.schema.json b/charts/t8s-cluster/values.schema.json index e173b8297..e8ea23409 100644 --- a/charts/t8s-cluster/values.schema.json +++ b/charts/t8s-cluster/values.schema.json @@ -103,6 +103,10 @@ "controlPlane": { "type": "object", "properties": { + "hosted": { + "type": "boolean", + "description": "Whether the control plane is hosted on the management cluster" + }, "flavor": { "type": "string" }, diff --git a/charts/t8s-cluster/values.yaml b/charts/t8s-cluster/values.yaml index ce8866089..3e5e4db33 100644 --- a/charts/t8s-cluster/values.yaml +++ b/charts/t8s-cluster/values.yaml @@ -38,6 +38,7 @@ metadata: friendlyName: "" controlPlane: + hosted: false flavor: standard.2.1905 singleNode: false allowedCIDRs: []