From b2a799eb773da9679d98ac07335f3edfcaac62aa Mon Sep 17 00:00:00 2001 From: paurosello Date: Thu, 11 Apr 2024 10:07:02 +0200 Subject: [PATCH] Enable customizing node replacement values (#580) --- helm/cluster-aws/README.md | 4 +++- helm/cluster-aws/templates/_machine_pools.tpl | 3 ++- helm/cluster-aws/values.schema.json | 10 ++++++++++ helm/cluster-aws/values.yaml | 2 ++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/helm/cluster-aws/README.md b/helm/cluster-aws/README.md index 7e4d17f8..c5663f93 100644 --- a/helm/cluster-aws/README.md +++ b/helm/cluster-aws/README.md @@ -221,7 +221,9 @@ Node pools of the cluster. If not specified, this defaults to the value of `clus | `global.nodePools.PATTERN.instanceType` | **EC2 instance type**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.instanceTypeOverrides` | **Instance type overrides** - Ordered list of instance types to be used for the machine pool. The first instance type that is available in the region will be used. Read more in our docs https://docs.giantswarm.io/advanced/cluster-management/node-pools-capi/|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
**Default:** `[]`| | `global.nodePools.PATTERN.instanceTypeOverrides[*]` | **EC2 instance type**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| +| `global.nodePools.PATTERN.instanceWarmup` | **Time interval, in seconds, between node replacement.**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.maxSize` | **Maximum number of nodes**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| +| `global.nodePools.PATTERN.minHealthyPercentage` | **Minimum percentage of instances that must remain healthy during node replacement.**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.minSize` | **Minimum number of nodes**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.rootVolumeSizeGB` | **Root volume size (GB)**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.spotInstances` | **Spot instances** - Compared to on-demand instances, spot instances can help you save cost.|**Type:** `object`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| @@ -249,7 +251,7 @@ Properties within the `.global.podSecurityStandards` object | **Property** | **Description** | **More Details** | | :----------- | :-------------- | :--------------- | | `baseDomain` | **Base DNS domain**|**Type:** `string`
| -| `cluster` | **Cluster** - Helm values for the provider-independent cluster chart|**Type:** `object`
**Default:** `{"providerIntegration":{"apps":{"cilium":{"configTemplateName":"awsCiliumHelmValues"},"coredns":{"configTemplateName":"awsCorednsHelmValues"}},"clusterAnnotationsTemplateName":"awsConnectivityLabels","components":{"systemd":{"timesyncd":{"ntp":["169.254.169.123"]}}},"connectivity":{"proxy":{"noProxy":{"templateName":"awsNoProxyList","value":["elb.amazonaws.com","169.254.169.254"]}}},"controlPlane":{"kubeadmConfig":{"clusterConfiguration":{"apiServer":{"apiAudiences":{"templateName":"awsApiServerApiAudiences"},"featureGates":[{"enabled":true,"name":"CronJobTimeZone"}],"serviceAccountIssuer":{"templateName":"awsIrsaServiceAccountIssuer"}}},"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdc","format":"xfs","label":"etcd","wipeFilesystem":true},"name":"etcd"},{"mount":{"device":"/dev/xvdd","format":"xfs","label":"containerd","wipeFilesystem":true},"name":"containerd"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"kubelet","wipeFilesystem":true},"name":"kubelet"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/etcd","where":"/var/lib/etcd"},"unit":{"defaultDependencies":false,"description":"etcd volume"}},"enabled":true,"name":"var-lib-etcd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/kubelet","where":"/var/lib/kubelet"},"unit":{"defaultDependencies":false,"description":"kubelet volume"}},"enabled":true,"name":"var-lib-kubelet.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/containerd","where":"/var/lib/containerd"},"unit":{"defaultDependencies":false,"description":"containerd volume"}},"enabled":true,"name":"var-lib-containerd.mount"}]}}}}},"resources":{"infrastructureMachineTemplate":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachineTemplate","version":"v1beta1"},"infrastructureMachineTemplateSpecTemplateName":"controlplane-awsmachinetemplate-spec"}},"pauseProperties":{"global.connectivity.vpcMode":"private"},"provider":"aws","registry":{"templateName":"awsContainerImageRegistry"},"resourcesApi":{"bastionResourceEnabled":false,"ciliumHelmReleaseResourceEnabled":true,"cleanupHelmReleaseResourcesEnabled":true,"clusterResourceEnabled":true,"controlPlaneResourceEnabled":true,"coreDnsHelmReleaseResourceEnabled":true,"helmRepositoryResourcesEnabled":true,"infrastructureCluster":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSCluster","version":"v1beta1"},"infrastructureMachinePool":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachinePool","version":"v1beta1"},"machineHealthCheckResourceEnabled":true,"machinePoolResourcesEnabled":true,"networkPoliciesHelmReleaseResourceEnabled":true,"nodePoolKind":"MachinePool","verticalPodAutoscalerCrdHelmReleaseResourceEnabled":true},"workers":{"defaultNodePools":{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","maxSize":3,"minSize":3}},"kubeadmConfig":{"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}]}}}}`| +| `cluster` | **Cluster** - Helm values for the provider-independent cluster chart|**Type:** `object`
**Default:** `{"providerIntegration":{"apps":{"cilium":{"configTemplateName":"awsCiliumHelmValues"},"coredns":{"configTemplateName":"awsCorednsHelmValues"}},"clusterAnnotationsTemplateName":"awsConnectivityLabels","components":{"systemd":{"timesyncd":{"ntp":["169.254.169.123"]}}},"connectivity":{"proxy":{"noProxy":{"templateName":"awsNoProxyList","value":["elb.amazonaws.com","169.254.169.254"]}}},"controlPlane":{"kubeadmConfig":{"clusterConfiguration":{"apiServer":{"apiAudiences":{"templateName":"awsApiServerApiAudiences"},"featureGates":[{"enabled":true,"name":"CronJobTimeZone"}],"serviceAccountIssuer":{"templateName":"awsIrsaServiceAccountIssuer"}}},"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdc","format":"xfs","label":"etcd","wipeFilesystem":true},"name":"etcd"},{"mount":{"device":"/dev/xvdd","format":"xfs","label":"containerd","wipeFilesystem":true},"name":"containerd"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"kubelet","wipeFilesystem":true},"name":"kubelet"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/etcd","where":"/var/lib/etcd"},"unit":{"defaultDependencies":false,"description":"etcd volume"}},"enabled":true,"name":"var-lib-etcd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/kubelet","where":"/var/lib/kubelet"},"unit":{"defaultDependencies":false,"description":"kubelet volume"}},"enabled":true,"name":"var-lib-kubelet.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/containerd","where":"/var/lib/containerd"},"unit":{"defaultDependencies":false,"description":"containerd volume"}},"enabled":true,"name":"var-lib-containerd.mount"}]}}}}},"resources":{"infrastructureMachineTemplate":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachineTemplate","version":"v1beta1"},"infrastructureMachineTemplateSpecTemplateName":"controlplane-awsmachinetemplate-spec"}},"pauseProperties":{"global.connectivity.vpcMode":"private"},"provider":"aws","registry":{"templateName":"awsContainerImageRegistry"},"resourcesApi":{"bastionResourceEnabled":false,"ciliumHelmReleaseResourceEnabled":true,"cleanupHelmReleaseResourcesEnabled":true,"clusterResourceEnabled":true,"controlPlaneResourceEnabled":true,"coreDnsHelmReleaseResourceEnabled":true,"helmRepositoryResourcesEnabled":true,"infrastructureCluster":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSCluster","version":"v1beta1"},"infrastructureMachinePool":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachinePool","version":"v1beta1"},"machineHealthCheckResourceEnabled":true,"machinePoolResourcesEnabled":true,"networkPoliciesHelmReleaseResourceEnabled":true,"nodePoolKind":"MachinePool","verticalPodAutoscalerCrdHelmReleaseResourceEnabled":true},"workers":{"defaultNodePools":{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","instanceWarmup":300,"maxSize":3,"minHealthyPercentage":90,"minSize":3}},"kubeadmConfig":{"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}]}}}}`| | `cluster-shared` | **Library chart**|**Type:** `object`
| | `managementCluster` | **Management cluster** - Name of the Cluster API cluster managing this workload cluster.|**Type:** `string`
| | `provider` | **Cluster API provider name**|**Type:** `string`
| diff --git a/helm/cluster-aws/templates/_machine_pools.tpl b/helm/cluster-aws/templates/_machine_pools.tpl index ee6bc761..56b7a13a 100644 --- a/helm/cluster-aws/templates/_machine_pools.tpl +++ b/helm/cluster-aws/templates/_machine_pools.tpl @@ -65,7 +65,8 @@ spec: {{- end }} {{- end }} refreshPreferences: - instanceWarmup: 300 + instanceWarmup: {{ $value.instanceWarmup | default 300 }} + minHealthyPercentage: {{ $value.minHealthyPercentage | default 90 }} --- {{ end }} {{- end -}} diff --git a/helm/cluster-aws/values.schema.json b/helm/cluster-aws/values.schema.json index a8840aa0..ea7ee573 100644 --- a/helm/cluster-aws/values.schema.json +++ b/helm/cluster-aws/values.schema.json @@ -134,10 +134,18 @@ }, "default": [] }, + "instanceWarmup": { + "type": "integer", + "title": "Time interval, in seconds, between node replacement." + }, "maxSize": { "type": "integer", "title": "Maximum number of nodes" }, + "minHealthyPercentage": { + "type": "integer", + "title": "Minimum percentage of instances that must remain healthy during node replacement." + }, "minSize": { "type": "integer", "title": "Minimum number of nodes" @@ -399,7 +407,9 @@ "label=default" ], "instanceType": "r6i.xlarge", + "instanceWarmup": 300, "maxSize": 3, + "minHealthyPercentage": 90, "minSize": 3 } }, diff --git a/helm/cluster-aws/values.yaml b/helm/cluster-aws/values.yaml index 8940479f..f967bb9b 100644 --- a/helm/cluster-aws/values.yaml +++ b/helm/cluster-aws/values.yaml @@ -141,7 +141,9 @@ cluster: customNodeLabels: - label=default instanceType: r6i.xlarge + instanceWarmup: 300 maxSize: 3 + minHealthyPercentage: 90 minSize: 3 kubeadmConfig: files: