From 40f1af0f0330753fb9bdf58d7c091d8a14dde71e Mon Sep 17 00:00:00 2001 From: calvix Date: Thu, 9 May 2024 11:38:12 +0200 Subject: [PATCH] add-separate-disk-for-worker-nodes (#614) * add-separate-disk-for-worker-nodes --------- Co-authored-by: Andreas Sommer --- CHANGELOG.md | 8 + helm/cluster-aws/Chart.lock | 6 +- helm/cluster-aws/Chart.yaml | 2 +- helm/cluster-aws/README.md | 13 +- helm/cluster-aws/templates/_control_plane.tpl | 6 +- helm/cluster-aws/templates/_machine_pools.tpl | 11 +- helm/cluster-aws/values.schema.json | 147 ++++++++++++------ helm/cluster-aws/values.yaml | 83 ++++++---- 8 files changed, 184 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d91e26a1..e1e11a12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Worker nodes - Add `nonRootVolumes` fields to mount `/var/lib` and `/var/log` on separate disk volumes. + +### Changed + +- Control-plane nodes - combine kubelet disk `/var/lib/kubelet` and containerd disk `/var/lib/containerd` into single disk `/var/lib` to share the volume space and save cost. + ## [0.74.0] - 2024-05-08 ### Fixed diff --git a/helm/cluster-aws/Chart.lock b/helm/cluster-aws/Chart.lock index 4d89b037..1374302d 100644 --- a/helm/cluster-aws/Chart.lock +++ b/helm/cluster-aws/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: cluster repository: https://giantswarm.github.io/cluster-catalog - version: 0.22.0 + version: 0.23.0 - name: cluster-shared repository: https://giantswarm.github.io/cluster-catalog version: 0.7.0 -digest: sha256:8ad414f8fe2f6b4672f3d2408536728e227364eee8881bc9cb4a43ea58c6d6a4 -generated: "2024-05-07T17:23:28.811124056+02:00" +digest: sha256:aec820306e002c75e4bd4fd0e8d5db9999a93a8e6e45fa84beb89446b877c45a +generated: "2024-05-08T14:25:29.991771063+02:00" diff --git a/helm/cluster-aws/Chart.yaml b/helm/cluster-aws/Chart.yaml index 4794c701..6f186158 100644 --- a/helm/cluster-aws/Chart.yaml +++ b/helm/cluster-aws/Chart.yaml @@ -16,7 +16,7 @@ restrictions: - capa dependencies: - name: cluster - version: "0.22.0" + version: "0.23.0" repository: https://giantswarm.github.io/cluster-catalog - name: cluster-shared version: "0.7.0" diff --git a/helm/cluster-aws/README.md b/helm/cluster-aws/README.md index 6074707a..364da329 100644 --- a/helm/cluster-aws/README.md +++ b/helm/cluster-aws/README.md @@ -167,13 +167,12 @@ Properties within the `.global.controlPlane` object | `global.controlPlane.apiExtraCertSANs[*]` | **cert SAN**|**Type:** `string`
| | `global.controlPlane.apiMode` | **API mode** - Whether the Kubernetes API server load balancer should be reachable from the internet (public) or internal only (private).|**Type:** `string`
**Default:** `"public"`| | `global.controlPlane.apiServerPort` | **API server port** - The API server Load Balancer port. This option sets the Spec.ClusterNetwork.APIServerPort field on the Cluster CR. In CAPI this field isn't used currently. It is instead used in providers. In CAPA this sets only the public facing port of the Load Balancer. In CAPZ both the public facing and the destination port are set to this value. CAPV and CAPVCD do not use it.|**Type:** `integer`
**Default:** `443`| -| `global.controlPlane.containerdVolumeSizeGB` | **Containerd volume size (GB)**|**Type:** `integer`
**Default:** `100`| | `global.controlPlane.etcdVolumeSizeGB` | **Etcd volume size (GB)**|**Type:** `integer`
**Default:** `100`| | `global.controlPlane.instanceType` | **EC2 instance type**|**Type:** `string`
**Default:** `"r6i.xlarge"`| -| `global.controlPlane.kubeletVolumeSizeGB` | **Kubelet volume size (GB)**|**Type:** `integer`
**Default:** `100`| +| `global.controlPlane.libVolumeSizeGB` | **Lib volume size (GB)** - Size of the volume mounted at `/var/lib` on the control plane nodes. This disk is shared between kubelet folder `/var/lib/kubelet` and containerd folder `/var/lib/containerd`.|**Type:** `integer`
**Default:** `40`| | `global.controlPlane.loadBalancerIngressAllowCidrBlocks` | **Load balancer allow list** - IPv4 address ranges that are allowed to connect to the control plane load balancer, in CIDR notation. When setting this field, remember to add the Management cluster Nat Gateway IPs provided by Giant Swarm so that the cluster can still be managed. These Nat Gateway IPs can be found in the Management Cluster AWSCluster '.status.networkStatus.natGatewaysIPs' field.|**Type:** `array`
| | `global.controlPlane.loadBalancerIngressAllowCidrBlocks[*]` | **Address range**|**Type:** `string`
| -| `global.controlPlane.logVolumeSizeGB` | **Log volume size (GB)**|**Type:** `integer`
**Default:** `15`| +| `global.controlPlane.logVolumeSizeGB` | **Log volume size (GB)** - Size of the volume mounted at /var/log on the control plane nodes.|**Type:** `integer`
**Default:** `15`| | `global.controlPlane.machineHealthCheck` | **Machine health check**|**Type:** `object`
| | `global.controlPlane.machineHealthCheck.enabled` | **Enable**|**Type:** `boolean`
**Default:** `true`| | `global.controlPlane.machineHealthCheck.maxUnhealthy` | **Maximum unhealthy nodes**|**Type:** `string`
**Example:** `"40%"`
**Default:** `"40%"`| @@ -186,7 +185,7 @@ Properties within the `.global.controlPlane` object | `global.controlPlane.oidc.groupsClaim` | **Groups claim**|**Type:** `string`
| | `global.controlPlane.oidc.issuerUrl` | **Issuer URL** - Exact issuer URL that will be included in identity tokens.|**Type:** `string`
| | `global.controlPlane.oidc.usernameClaim` | **Username claim**|**Type:** `string`
| -| `global.controlPlane.rootVolumeSizeGB` | **Root volume size (GB)**|**Type:** `integer`
**Default:** `120`| +| `global.controlPlane.rootVolumeSizeGB` | **Root volume size (GB)**|**Type:** `integer`
**Default:** `8`| | `global.controlPlane.subnetTags` | **Subnet tags** - Tags to select AWS resources for the control plane by.|**Type:** `array`
| | `global.controlPlane.subnetTags[*]` | **Subnet tag**|**Type:** `object`
| | `global.controlPlane.subnetTags[*].*` | **Tag value**|**Type:** `string`
**Value pattern:** `^[ a-zA-Z0-9\._:/=+-@]+$`
| @@ -244,10 +243,12 @@ Node pools of the cluster. If not specified, this defaults to the value of `clus | `global.nodePools.PATTERN.instanceTypeOverrides` | **Instance type overrides** - Ordered list of instance types to be used for the machine pool. The first instance type that is available in the region will be used. Read more in our docs https://docs.giantswarm.io/advanced/cluster-management/node-pools-capi/|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
**Default:** `[]`| | `global.nodePools.PATTERN.instanceTypeOverrides[*]` | **EC2 instance type**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.instanceWarmup` | **Time interval, in seconds, between node replacement.**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| +| `global.nodePools.PATTERN.libVolumeSizeGB` | **Lib volume size (GB)** - Size of the volume mounted at `/var/lib` on the worker nodes. This disk is shared between kubelet folder `/var/lib/kubelet` and containerd folder `/var/lib/containerd`s.|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
**Default:** `120`| +| `global.nodePools.PATTERN.logVolumeSizeGB` | **Log volume size (GB)** - Size of the volume mounted at `/var/log` on the worker nodes.|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
**Default:** `30`| | `global.nodePools.PATTERN.maxSize` | **Maximum number of nodes**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.minHealthyPercentage` | **Minimum percentage of instances that must remain healthy during node replacement.**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.minSize` | **Minimum number of nodes**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| -| `global.nodePools.PATTERN.rootVolumeSizeGB` | **Root volume size (GB)**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| +| `global.nodePools.PATTERN.rootVolumeSizeGB` | **Root volume size (GB)**|**Type:** `integer`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
**Default:** `8`| | `global.nodePools.PATTERN.spotInstances` | **Spot instances** - Compared to on-demand instances, spot instances can help you save cost.|**Type:** `object`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| | `global.nodePools.PATTERN.spotInstances.enabled` | **Enable**|**Type:** `boolean`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
**Default:** `false`| | `global.nodePools.PATTERN.spotInstances.maxPrice` | **Maximum price to pay per instance per hour, in USD.**|**Type:** `number`
**Key pattern:**
`PATTERN`=`^[a-z0-9][-a-z0-9]{3,18}[a-z0-9]$`
| @@ -273,7 +274,7 @@ Properties within the `.global.podSecurityStandards` object | **Property** | **Description** | **More Details** | | :----------- | :-------------- | :--------------- | | `baseDomain` | **Base DNS domain**|**Type:** `string`
| -| `cluster` | **Cluster** - Helm values for the provider-independent cluster chart|**Type:** `object`
**Default:** `{"providerIntegration":{"apps":{"cilium":{"configTemplateName":"awsCiliumHelmValues"},"coredns":{"configTemplateName":"awsCorednsHelmValues"},"networkPolicies":{"configTemplateName":"awsNetworkPoliciesHelmValues"}},"clusterAnnotationsTemplateName":"awsConnectivityLabels","components":{"systemd":{"timesyncd":{"ntp":["169.254.169.123"]}}},"connectivity":{"proxy":{"noProxy":{"templateName":"awsNoProxyList","value":["elb.amazonaws.com","169.254.169.254"]}}},"controlPlane":{"kubeadmConfig":{"clusterConfiguration":{"apiServer":{"apiAudiences":{"templateName":"awsApiServerApiAudiences"},"featureGates":[{"enabled":true,"name":"CronJobTimeZone"}],"serviceAccountIssuer":{"templateName":"awsIrsaServiceAccountIssuer"}}},"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdc","format":"xfs","label":"etcd","wipeFilesystem":true},"name":"etcd"},{"mount":{"device":"/dev/xvdd","format":"xfs","label":"containerd","wipeFilesystem":true},"name":"containerd"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"kubelet","wipeFilesystem":true},"name":"kubelet"},{"mount":{"device":"/dev/xvdf","format":"xfs","label":"log","wipeFilesystem":true},"name":"kubelet"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/etcd","where":"/var/lib/etcd"},"unit":{"defaultDependencies":false,"description":"etcd volume"}},"enabled":true,"name":"var-lib-etcd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/kubelet","where":"/var/lib/kubelet"},"unit":{"defaultDependencies":false,"description":"kubelet volume"}},"enabled":true,"name":"var-lib-kubelet.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/containerd","where":"/var/lib/containerd"},"unit":{"defaultDependencies":false,"description":"containerd volume"}},"enabled":true,"name":"var-lib-containerd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}},"resources":{"infrastructureMachineTemplate":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachineTemplate","version":"v1beta1"},"infrastructureMachineTemplateSpecTemplateName":"controlplane-awsmachinetemplate-spec"}},"pauseProperties":{"global.connectivity.vpcMode":"private"},"provider":"aws","registry":{"templateName":"awsContainerImageRegistry"},"resourcesApi":{"bastionResourceEnabled":false,"ciliumHelmReleaseResourceEnabled":true,"cleanupHelmReleaseResourcesEnabled":true,"clusterResourceEnabled":true,"controlPlaneResourceEnabled":true,"coreDnsHelmReleaseResourceEnabled":true,"helmRepositoryResourcesEnabled":true,"infrastructureCluster":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSCluster","version":"v1beta1"},"infrastructureMachinePool":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachinePool","version":"v1beta1"},"machineHealthCheckResourceEnabled":true,"machinePoolResourcesEnabled":true,"networkPoliciesHelmReleaseResourceEnabled":true,"nodePoolKind":"MachinePool","verticalPodAutoscalerCrdHelmReleaseResourceEnabled":true},"workers":{"defaultNodePools":{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","instanceWarmup":600,"maxSize":3,"minHealthyPercentage":90,"minSize":3}},"kubeadmConfig":{"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}]}}}}`| +| `cluster` | **Cluster** - Helm values for the provider-independent cluster chart|**Type:** `object`
**Default:** `{"providerIntegration":{"apps":{"cilium":{"configTemplateName":"awsCiliumHelmValues"},"coredns":{"configTemplateName":"awsCorednsHelmValues"},"networkPolicies":{"configTemplateName":"awsNetworkPoliciesHelmValues"}},"clusterAnnotationsTemplateName":"awsConnectivityLabels","components":{"systemd":{"timesyncd":{"ntp":["169.254.169.123"]}}},"connectivity":{"proxy":{"noProxy":{"templateName":"awsNoProxyList","value":["elb.amazonaws.com","169.254.169.254"]}}},"controlPlane":{"kubeadmConfig":{"clusterConfiguration":{"apiServer":{"apiAudiences":{"templateName":"awsApiServerApiAudiences"},"featureGates":[{"enabled":true,"name":"CronJobTimeZone"}],"serviceAccountIssuer":{"templateName":"awsIrsaServiceAccountIssuer"}}},"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdc","format":"xfs","label":"etcd","wipeFilesystem":true},"name":"etcd"},{"mount":{"device":"/dev/xvdd","format":"xfs","label":"lib","wipeFilesystem":true},"name":"lib"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"log","wipeFilesystem":true},"name":"log"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/etcd","where":"/var/lib/etcd"},"unit":{"defaultDependencies":false,"description":"etcd volume"}},"enabled":true,"name":"var-lib-etcd.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/lib","where":"/var/lib"},"unit":{"defaultDependencies":false,"description":"var lib volume"}},"enabled":true,"name":"var-lib.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}},"resources":{"infrastructureMachineTemplate":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachineTemplate","version":"v1beta1"},"infrastructureMachineTemplateSpecTemplateName":"controlplane-awsmachinetemplate-spec"}},"pauseProperties":{"global.connectivity.vpcMode":"private"},"provider":"aws","registry":{"templateName":"awsContainerImageRegistry"},"resourcesApi":{"bastionResourceEnabled":false,"ciliumHelmReleaseResourceEnabled":true,"cleanupHelmReleaseResourcesEnabled":true,"clusterResourceEnabled":true,"controlPlaneResourceEnabled":true,"coreDnsHelmReleaseResourceEnabled":true,"helmRepositoryResourcesEnabled":true,"infrastructureCluster":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSCluster","version":"v1beta1"},"infrastructureMachinePool":{"group":"infrastructure.cluster.x-k8s.io","kind":"AWSMachinePool","version":"v1beta1"},"machineHealthCheckResourceEnabled":true,"machinePoolResourcesEnabled":true,"networkPoliciesHelmReleaseResourceEnabled":true,"nodePoolKind":"MachinePool","verticalPodAutoscalerCrdHelmReleaseResourceEnabled":true},"workers":{"defaultNodePools":{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","instanceWarmup":600,"maxSize":3,"minHealthyPercentage":90,"minSize":3}},"kubeadmConfig":{"files":[{"contentFrom":{"secret":{"key":"99-unmanaged-devices.network","name":"provider-specific-files","prependClusterNameAsPrefix":true}},"path":"/etc/systemd/network/99-unmanaged-devices.network","permissions":"0644"}],"ignition":{"containerLinuxConfig":{"additionalConfig":{"storage":{"filesystems":[{"mount":{"device":"/dev/xvdd","format":"xfs","label":"lib","wipeFilesystem":true},"name":"lib"},{"mount":{"device":"/dev/xvde","format":"xfs","label":"log","wipeFilesystem":true},"name":"log"}]},"systemd":{"units":[{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/lib","where":"/var/lib"},"unit":{"defaultDependencies":false,"description":"lib volume"}},"enabled":true,"name":"var-lib.mount"},{"contents":{"install":{"wantedBy":["local-fs-pre.target"]},"mount":{"type":"xfs","what":"/dev/disk/by-label/log","where":"/var/log"},"unit":{"defaultDependencies":false,"description":"log volume"}},"enabled":true,"name":"var-log.mount"}]}}}}}}}}`| | `cluster-shared` | **Library chart**|**Type:** `object`
| | `managementCluster` | **Management cluster** - Name of the Cluster API cluster managing this workload cluster.|**Type:** `string`
| | `provider` | **Cluster API provider name**|**Type:** `string`
| diff --git a/helm/cluster-aws/templates/_control_plane.tpl b/helm/cluster-aws/templates/_control_plane.tpl index 398c8e02..f403c0b0 100644 --- a/helm/cluster-aws/templates/_control_plane.tpl +++ b/helm/cluster-aws/templates/_control_plane.tpl @@ -14,13 +14,9 @@ nonRootVolumes: type: gp3 - deviceName: /dev/xvdd encrypted: true - size: {{ .Values.global.controlPlane.containerdVolumeSizeGB }} + size: {{ .Values.global.controlPlane.libVolumeSizeGB }} type: gp3 - deviceName: /dev/xvde - encrypted: true - size: {{ .Values.global.controlPlane.kubeletVolumeSizeGB }} - type: gp3 -- deviceName: /dev/xvdf encrypted: true size: {{ .Values.global.controlPlane.logVolumeSizeGB }} type: gp3 diff --git a/helm/cluster-aws/templates/_machine_pools.tpl b/helm/cluster-aws/templates/_machine_pools.tpl index 1ad3db60..fc2585e7 100644 --- a/helm/cluster-aws/templates/_machine_pools.tpl +++ b/helm/cluster-aws/templates/_machine_pools.tpl @@ -37,7 +37,16 @@ spec: iamInstanceProfile: nodes-{{ $name }}-{{ include "resource.default.name" $ }} instanceType: {{ $value.instanceType | default "r6i.xlarge" }} rootVolume: - size: {{ $value.rootVolumeSizeGB | default 300 }} + size: {{ $value.rootVolumeSizeGB | default 8 }} + type: gp3 + nonRootVolumes: + - deviceName: /dev/xvdd + encrypted: true + size: {{ $value.libVolumeSizeGB | default 120 }} + type: gp3 + - deviceName: /dev/xvde + encrypted: true + size: {{ $value.logVolumeSizeGB | default 30}} type: gp3 sshKeyName: "" {{- if $value.additionalSecurityGroups }} diff --git a/helm/cluster-aws/values.schema.json b/helm/cluster-aws/values.schema.json index ca5f43cc..f5425b60 100644 --- a/helm/cluster-aws/values.schema.json +++ b/helm/cluster-aws/values.schema.json @@ -138,6 +138,18 @@ "type": "integer", "title": "Time interval, in seconds, between node replacement." }, + "libVolumeSizeGB": { + "type": "integer", + "title": "Lib volume size (GB)", + "description": "Size of the volume mounted at `/var/lib` on the worker nodes. This disk is shared between kubelet folder `/var/lib/kubelet` and containerd folder `/var/lib/containerd`s.", + "default": 120 + }, + "logVolumeSizeGB": { + "type": "integer", + "title": "Log volume size (GB)", + "description": "Size of the volume mounted at `/var/log` on the worker nodes.", + "default": 30 + }, "maxSize": { "type": "integer", "title": "Maximum number of nodes" @@ -152,7 +164,8 @@ }, "rootVolumeSizeGB": { "type": "integer", - "title": "Root volume size (GB)" + "title": "Root volume size (GB)", + "default": 8 }, "spotInstances": { "type": "object", @@ -279,28 +292,19 @@ "mount": { "device": "/dev/xvdd", "format": "xfs", - "label": "containerd", + "label": "lib", "wipeFilesystem": true }, - "name": "containerd" + "name": "lib" }, { "mount": { "device": "/dev/xvde", "format": "xfs", - "label": "kubelet", - "wipeFilesystem": true - }, - "name": "kubelet" - }, - { - "mount": { - "device": "/dev/xvdf", - "format": "xfs", "label": "log", "wipeFilesystem": true }, - "name": "kubelet" + "name": "log" } ] }, @@ -335,36 +339,16 @@ }, "mount": { "type": "xfs", - "what": "/dev/disk/by-label/kubelet", - "where": "/var/lib/kubelet" + "what": "/dev/disk/by-label/lib", + "where": "/var/lib" }, "unit": { - "description": "kubelet volume", + "description": "var lib volume", "defaultDependencies": false } }, "enabled": true, - "name": "var-lib-kubelet.mount" - }, - { - "contents": { - "install": { - "wantedBy": [ - "local-fs-pre.target" - ] - }, - "mount": { - "type": "xfs", - "what": "/dev/disk/by-label/containerd", - "where": "/var/lib/containerd" - }, - "unit": { - "description": "containerd volume", - "defaultDependencies": false - } - }, - "enabled": true, - "name": "var-lib-containerd.mount" + "name": "var-lib.mount" }, { "contents": { @@ -458,7 +442,79 @@ "path": "/etc/systemd/network/99-unmanaged-devices.network", "permissions": "0644" } - ] + ], + "ignition": { + "containerLinuxConfig": { + "additionalConfig": { + "storage": { + "filesystems": [ + { + "mount": { + "device": "/dev/xvdd", + "format": "xfs", + "label": "lib", + "wipeFilesystem": true + }, + "name": "lib" + }, + { + "mount": { + "device": "/dev/xvde", + "format": "xfs", + "label": "log", + "wipeFilesystem": true + }, + "name": "log" + } + ] + }, + "systemd": { + "units": [ + { + "contents": { + "install": { + "wantedBy": [ + "local-fs-pre.target" + ] + }, + "mount": { + "type": "xfs", + "what": "/dev/disk/by-label/lib", + "where": "/var/lib" + }, + "unit": { + "description": "lib volume", + "defaultDependencies": false + } + }, + "enabled": true, + "name": "var-lib.mount" + }, + { + "contents": { + "install": { + "wantedBy": [ + "local-fs-pre.target" + ] + }, + "mount": { + "type": "xfs", + "what": "/dev/disk/by-label/log", + "where": "/var/log" + }, + "unit": { + "description": "log volume", + "defaultDependencies": false + } + }, + "enabled": true, + "name": "var-log.mount" + } + ] + } + } + } + } } } } @@ -1068,11 +1124,6 @@ "maximum": 65535, "minimum": 0 }, - "containerdVolumeSizeGB": { - "type": "integer", - "title": "Containerd volume size (GB)", - "default": 100 - }, "etcdVolumeSizeGB": { "type": "integer", "title": "Etcd volume size (GB)", @@ -1083,10 +1134,11 @@ "title": "EC2 instance type", "default": "r6i.xlarge" }, - "kubeletVolumeSizeGB": { + "libVolumeSizeGB": { "type": "integer", - "title": "Kubelet volume size (GB)", - "default": 100 + "title": "Lib volume size (GB)", + "description": "Size of the volume mounted at `/var/lib` on the control plane nodes. This disk is shared between kubelet folder `/var/lib/kubelet` and containerd folder `/var/lib/containerd`.", + "default": 40 }, "loadBalancerIngressAllowCidrBlocks": { "type": "array", @@ -1100,6 +1152,7 @@ "logVolumeSizeGB": { "type": "integer", "title": "Log volume size (GB)", + "description": "Size of the volume mounted at /var/log on the control plane nodes.", "default": 15 }, "machineHealthCheck": { @@ -1180,7 +1233,7 @@ "rootVolumeSizeGB": { "type": "integer", "title": "Root volume size (GB)", - "default": 120 + "default": 8 }, "subnetTags": { "type": "array", diff --git a/helm/cluster-aws/values.yaml b/helm/cluster-aws/values.yaml index e8fd6187..fbac740d 100644 --- a/helm/cluster-aws/values.yaml +++ b/helm/cluster-aws/values.yaml @@ -55,21 +55,15 @@ cluster: - mount: device: /dev/xvdd format: xfs - label: containerd + label: lib wipeFilesystem: true - name: containerd + name: lib - mount: device: /dev/xvde format: xfs - label: kubelet - wipeFilesystem: true - name: kubelet - - mount: - device: /dev/xvdf - format: xfs label: log wipeFilesystem: true - name: kubelet + name: log systemd: units: - contents: @@ -91,26 +85,13 @@ cluster: - local-fs-pre.target mount: type: xfs - what: /dev/disk/by-label/kubelet - where: /var/lib/kubelet + what: /dev/disk/by-label/lib + where: /var/lib unit: defaultDependencies: false - description: kubelet volume + description: var lib volume enabled: true - name: var-lib-kubelet.mount - - contents: - install: - wantedBy: - - local-fs-pre.target - mount: - type: xfs - what: /dev/disk/by-label/containerd - where: /var/lib/containerd - unit: - defaultDependencies: false - description: containerd volume - enabled: true - name: var-lib-containerd.mount + name: var-lib.mount - contents: install: wantedBy: @@ -175,6 +156,51 @@ cluster: prependClusterNameAsPrefix: true path: /etc/systemd/network/99-unmanaged-devices.network permissions: "0644" + ignition: + containerLinuxConfig: + additionalConfig: + storage: + filesystems: + - mount: + device: /dev/xvdd + format: xfs + label: lib + wipeFilesystem: true + name: lib + - mount: + device: /dev/xvde + format: xfs + label: log + wipeFilesystem: true + name: log + systemd: + units: + - contents: + install: + wantedBy: + - local-fs-pre.target + mount: + type: xfs + what: /dev/disk/by-label/lib + where: /var/lib + unit: + defaultDependencies: false + description: lib volume + enabled: true + name: var-lib.mount + - contents: + install: + wantedBy: + - local-fs-pre.target + mount: + type: xfs + what: /dev/disk/by-label/log + where: /var/log + unit: + defaultDependencies: false + description: log volume + enabled: true + name: var-log.mount global: apps: awsCloudControllerManager: {} @@ -238,10 +264,9 @@ global: controlPlane: apiMode: public apiServerPort: 443 - containerdVolumeSizeGB: 100 etcdVolumeSizeGB: 100 instanceType: r6i.xlarge - kubeletVolumeSizeGB: 100 + libVolumeSizeGB: 40 logVolumeSizeGB: 15 machineHealthCheck: enabled: true @@ -250,7 +275,7 @@ global: unhealthyNotReadyTimeout: 10m0s unhealthyUnknownTimeout: 10m0s oidc: {} - rootVolumeSizeGB: 120 + rootVolumeSizeGB: 8 metadata: preventDeletion: false servicePriority: highest