From 6a299ee5be1ebdf984743631fa2d8ab2d5ca556a Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Fri, 27 Sep 2024 16:41:09 +0200 Subject: [PATCH] refactor: figure out k8s version in helm chart --- chart/templates/_init-containers.tpl | 29 +++- chart/tests/statefulset_test.yaml | 58 ++++++++ config/default_extra_values.go | 206 +-------------------------- pkg/cli/create_helm.go | 2 +- pkg/cli/create_platform.go | 2 +- 5 files changed, 91 insertions(+), 206 deletions(-) diff --git a/chart/templates/_init-containers.tpl b/chart/templates/_init-containers.tpl index 3c74522a41..e500b5cfa0 100644 --- a/chart/templates/_init-containers.tpl +++ b/chart/templates/_init-containers.tpl @@ -8,15 +8,30 @@ {{- end -}} {{- end -}} +{{- define "vcluster.k8s.capabilities.version" -}} +{{/* We need to workaround here for unit tests because Capabilities.KubeVersion.Version is not supported, so we use .Chart.Version */}} +{{- if hasPrefix "test-" .Chart.Version -}} +{{- regexFind "^v[0-9]+\\.[0-9]+\\.[0-9]+" (trimPrefix "test-" .Chart.Version) -}} +{{- else -}} +{{- regexFind "^v[0-9]+\\.[0-9]+\\.[0-9]+" .Capabilities.KubeVersion.Version -}} +{{- end -}} +{{- end -}} + {{/* Bump $defaultTag value whenever k8s version is bumped */}} {{- define "vcluster.k8s.controllerManager.image.tag" -}} {{- $defaultTag := "v1.30.2" -}} {{- if and (not (empty .Values.controlPlane.distro.k8s.version)) (eq .Values.controlPlane.distro.k8s.controllerManager.image.tag $defaultTag) -}} -{{ .Values.controlPlane.distro.k8s.version}} +{{ .Values.controlPlane.distro.k8s.version }} +{{- else -}} +{{- if not (eq .Values.controlPlane.distro.k8s.controllerManager.image.tag $defaultTag) -}} +{{ .Values.controlPlane.distro.k8s.controllerManager.image.tag }} +{{- else if not (empty (include "vcluster.k8s.capabilities.version" .)) -}} +{{ include "vcluster.k8s.capabilities.version" . }} {{- else -}} {{ .Values.controlPlane.distro.k8s.controllerManager.image.tag }} {{- end -}} {{- end -}} +{{- end -}} {{/* Bump $defaultTag value whenever k8s version is bumped */}} {{- define "vcluster.k8s.apiServer.image.tag" -}} @@ -24,9 +39,15 @@ {{- if and (not (empty .Values.controlPlane.distro.k8s.version)) (eq .Values.controlPlane.distro.k8s.apiServer.image.tag $defaultTag) -}} {{ .Values.controlPlane.distro.k8s.version}} {{- else -}} +{{- if not (eq .Values.controlPlane.distro.k8s.apiServer.image.tag $defaultTag) -}} +{{ .Values.controlPlane.distro.k8s.apiServer.image.tag }} +{{- else if not (empty (include "vcluster.k8s.capabilities.version" .)) -}} +{{ include "vcluster.k8s.capabilities.version" . }} +{{- else -}} {{ .Values.controlPlane.distro.k8s.apiServer.image.tag }} {{- end -}} {{- end -}} +{{- end -}} {{/* Bump $defaultTag value whenever k8s version is bumped */}} @@ -35,7 +56,13 @@ {{- if and (not (empty .Values.controlPlane.distro.k8s.version)) (eq .Values.controlPlane.distro.k8s.scheduler.image.tag $defaultTag) -}} {{ .Values.controlPlane.distro.k8s.version}} {{- else -}} +{{- if not (eq .Values.controlPlane.distro.k8s.scheduler.image.tag $defaultTag) -}} {{ .Values.controlPlane.distro.k8s.scheduler.image.tag }} +{{- else if not (empty (include "vcluster.k8s.capabilities.version" .)) -}} +{{ include "vcluster.k8s.capabilities.version" . }} +{{- else -}} +{{ .Values.controlPlane.distro.k8s.scheduler.image.tag }} +{{- end -}} {{- end -}} {{- end -}} diff --git a/chart/tests/statefulset_test.yaml b/chart/tests/statefulset_test.yaml index f1428543c2..fd270cc6da 100644 --- a/chart/tests/statefulset_test.yaml +++ b/chart/tests/statefulset_test.yaml @@ -578,6 +578,8 @@ tests: claimName: my-custom-pvc - it: k8s version not set, default tag images used for apiServer and controllerManager + chart: + version: "test-" set: controlPlane: distro: @@ -591,6 +593,60 @@ tests: path: spec.template.spec.initContainers[2].image value: registry.k8s.io/kube-apiserver:v1.30.2 + - it: k8s capabilities set + chart: + version: "test-v1.29.3" + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.29.3 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-apiserver:v1.29.3 + + - it: k8s capabilities orbstack + chart: + version: "test-v1.29.3+orb1" + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.29.3 + - equal: + path: spec.template.spec.initContainers[2].image + value: registry.k8s.io/kube-apiserver:v1.29.3 + + - it: k8s capabilities invalid + chart: + version: "test-invalid" + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.30.2 + + - it: k8s capabilities incomplete + chart: + version: "test-v1.22" + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.30.2 + + - it: k8s capabilities incomplete 2 + chart: + version: "test-1.22.11" + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.30.2 + + - it: k8s capabilities incomplete 2 + chart: + version: "test-v1.22.33" + asserts: + - equal: + path: spec.template.spec.initContainers[1].image + value: registry.k8s.io/kube-controller-manager:v1.22.33 + - it: k8s version sets image tag for apiServer and controllerManager set: controlPlane: @@ -650,6 +706,8 @@ tests: value: registry.k8s.io/kube-apiserver:v99914 - it: k8s version not set, default tag images used for apiServer and controllerManager (virtual scheduler enabled) + chart: + version: "test-" set: controlPlane: distro: diff --git a/config/default_extra_values.go b/config/default_extra_values.go index 2c95f2d9aa..883ff30f5c 100644 --- a/config/default_extra_values.go +++ b/config/default_extra_values.go @@ -2,11 +2,7 @@ package config import ( "fmt" - "regexp" - "strconv" "strings" - - "github.com/blang/semver" ) const ( @@ -73,16 +69,6 @@ var K8SEtcdVersionMap = map[string]string{ "1.27": "registry.k8s.io/etcd:3.5.7-0", } -var vclusterVersionsWithK8sVersionSupport []semver.Version - -func init() { - vclusterVersionsWithK8sVersionSupport = []semver.Version{ - semver.MustParse("0.20.0-beta.16"), - semver.MustParse("0.20.0-alpha.99"), - semver.MustParse("0.21.0-alpha.4"), - } -} - // ExtraValuesOptions holds the chart options type ExtraValuesOptions struct { Distro string @@ -103,13 +89,13 @@ type KubernetesVersion struct { Minor string } -func GetExtraValues(options *ExtraValuesOptions, chartVersion string) (string, error) { +func GetExtraValues(options *ExtraValuesOptions) (string, error) { fromConfig, err := NewDefaultConfig() if err != nil { return "", err } - toConfig, err := getExtraValues(options, chartVersion) + toConfig, err := getExtraValues(options) if err != nil { return "", fmt.Errorf("get extra values: %w", err) } @@ -117,147 +103,17 @@ func GetExtraValues(options *ExtraValuesOptions, chartVersion string) (string, e return Diff(fromConfig, toConfig) } -func getExtraValues(options *ExtraValuesOptions, chartVersion string) (*Config, error) { +func getExtraValues(options *ExtraValuesOptions) (*Config, error) { vConfig, err := NewDefaultConfig() if err != nil { return nil, err } - // apply k3s values - err = applyK3SExtraValues(vConfig, options) - if err != nil { - return nil, err - } - - // apply k0s values - err = applyK0SExtraValues(vConfig, options) - if err != nil { - return nil, err - } - - // apply k8s values - err = applyK8SExtraValues(vConfig, options, chartVersion) - if err != nil { - return nil, err - } - // add common release values addCommonReleaseValues(vConfig, options) return vConfig, nil } -var replaceRegEx = regexp.MustCompile("[^0-9]+") - -func applyK3SExtraValues(vConfig *Config, options *ExtraValuesOptions) error { - // get k3s image - image, err := getImageByVersion(options.KubernetesVersion, K3SVersionMap) - if err != nil { - return err - } - - // build values - if image != "" { - vConfig.ControlPlane.Distro.K3S.Image = parseImage(image) - } - - return nil -} - -func applyK0SExtraValues(vConfig *Config, options *ExtraValuesOptions) error { - // get k0s image - image, err := getImageByVersion(options.KubernetesVersion, K0SVersionMap) - if err != nil { - return err - } - - // build values - if image != "" { - vConfig.ControlPlane.Distro.K0S.Image = parseImage(image) - } - - return nil -} - -func applyK8SExtraValues(vConfig *Config, options *ExtraValuesOptions, chartVersion string) error { - // get api server image - apiImage, err := getImageByVersion(options.KubernetesVersion, K8SAPIVersionMap) - if err != nil { - return err - } - - // get etcd image - etcdImage, err := getImageByVersion(options.KubernetesVersion, K8SEtcdVersionMap) - if err != nil { - return err - } - - // build values - k8sVersion, err := getK8sVersion(chartVersion, apiImage) - if err != nil { - return err - } - vConfig.ControlPlane.Distro.K8S.Version = k8sVersion - - if etcdImage != "" { - vConfig.ControlPlane.BackingStore.Etcd.Deploy.StatefulSet.Image = parseImage(etcdImage) - } - - return nil -} - -func getK8sVersion(chartVersion, apiImage string) (string, error) { - if apiImage == "" { - return "", nil - } - isSupported, err := isK8sVersionSupported(chartVersion) - if err != nil { - return "", err - } - - if isSupported { - return parseImage(apiImage).Tag, nil - } - // if we set a non-empty value for a vcluster chart version that does not support - // the k8s version field, vcluster will panic - return "", nil -} - -func isK8sVersionSupported(chartVersion string) (bool, error) { - if chartVersion == "" { - // probably dev chart - return true, nil - } - v, err := semver.Parse(strings.TrimPrefix(chartVersion, "v")) - if err != nil { - return false, err - } - for _, supportedStartVer := range vclusterVersionsWithK8sVersionSupport { - if v.Minor != supportedStartVer.Minor { - continue - } - if len(supportedStartVer.Pre) > 0 && len(v.Pre) > 0 && supportedStartVer.Pre[0] != v.Pre[0] { - // this is a special case- its possible in our releases for an alpha to be ahead of a beta; - // we cannot compare prebuilds unless they share the same first element. - return true, nil - } - if v.GTE(supportedStartVer) { - return true, nil - } - return false, nil - } - // in the case no relevant start version was found for minor we can assume k8s version is supported - return true, nil -} - -func parseImage(image string) Image { - registry, repository, tag := SplitImage(image) - return Image{ - Registry: registry, - Repository: repository, - Tag: tag, - } -} - func SplitImage(image string) (string, string, string) { imageSplitted := strings.Split(image, ":") if len(imageSplitted) == 1 { @@ -280,54 +136,6 @@ func SplitImage(image string) (string, string, string) { return registry, repository, imageSplitted[len(imageSplitted)-1] } -func getImageByVersion(kubernetesVersion KubernetesVersion, versionImageMap map[string]string) (string, error) { - // check if there is a minor and major version - if kubernetesVersion.Minor == "" || kubernetesVersion.Major == "" { - return "", nil - } - - // find highest and lowest supported version for this map - highestMinorVersion := 0 - lowestMinorVersion := 0 - for version := range versionImageMap { - kubeVersion, err := ParseKubernetesVersionInfo(version) - if err != nil { - return "", fmt.Errorf("parse kube version %s: %w", version, err) - } - - minorVersion, err := strconv.Atoi(kubeVersion.Minor) - if err != nil { - return "", fmt.Errorf("convert minor version %s: %w", kubeVersion.Minor, err) - } - - if lowestMinorVersion == 0 || minorVersion < lowestMinorVersion { - lowestMinorVersion = minorVersion - } - if highestMinorVersion == 0 || minorVersion > highestMinorVersion { - highestMinorVersion = minorVersion - } - } - - // figure out what image to use - serverVersionString := getKubernetesVersion(kubernetesVersion) - serverMinorInt, err := getKubernetesMinorVersion(kubernetesVersion) - if err != nil { - return "", err - } - - // try to get from map - image, ok := versionImageMap[serverVersionString] - if !ok { - if serverMinorInt > highestMinorVersion { - image = versionImageMap["1."+strconv.Itoa(highestMinorVersion)] - } else { - image = versionImageMap["1."+strconv.Itoa(lowestMinorVersion)] - } - } - - return image, nil -} - func addCommonReleaseValues(config *Config, options *ExtraValuesOptions) { if options.Expose { if config.ControlPlane.Service.Spec == nil { @@ -363,14 +171,6 @@ func addCommonReleaseValues(config *Config, options *ExtraValuesOptions) { } } -func getKubernetesVersion(serverVersion KubernetesVersion) string { - return replaceRegEx.ReplaceAllString(serverVersion.Major, "") + "." + replaceRegEx.ReplaceAllString(serverVersion.Minor, "") -} - -func getKubernetesMinorVersion(serverVersion KubernetesVersion) (int, error) { - return strconv.Atoi(replaceRegEx.ReplaceAllString(serverVersion.Minor, "")) -} - func ParseKubernetesVersionInfo(versionStr string) (*KubernetesVersion, error) { if versionStr[0] == 'v' { versionStr = versionStr[1:] diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index f695c0322e..a6d6f51e15 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -263,7 +263,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. if err != nil { return err } - chartValues, err := config.GetExtraValues(chartOptions, options.ChartVersion) + chartValues, err := config.GetExtraValues(chartOptions) if err != nil { return err } diff --git a/pkg/cli/create_platform.go b/pkg/cli/create_platform.go index cd19404554..d26abba323 100644 --- a/pkg/cli/create_platform.go +++ b/pkg/cli/create_platform.go @@ -594,7 +594,7 @@ func mergeValues(platformClient platform.Client, options *CreateOptions, log log if err != nil { return "", err } - chartValues, err := vclusterconfig.GetExtraValues(chartOptions, options.ChartVersion) + chartValues, err := vclusterconfig.GetExtraValues(chartOptions) if err != nil { return "", err }