diff --git a/helm/cluster-aws/README.md b/helm/cluster-aws/README.md index bb54ddc8..60d257e7 100644 --- a/helm/cluster-aws/README.md +++ b/helm/cluster-aws/README.md @@ -47,6 +47,7 @@ Properties within the `.connectivity` top-level object | `connectivity.dns` | **DNS**|**Type:** `object`
| | `connectivity.dns.resolverRulesOwnerAccount` | **Resolver rules owner** - ID of the AWS account that created the resolver rules to be associated with the workload cluster VPC.|**Type:** `string`
| | `connectivity.network` | **Network**|**Type:** `object`
| +| `connectivity.network.internetGatewayId` | **Internet Gateway ID** - ID of the Internet gateway for the VPC.|**Type:** `string`
| | `connectivity.network.pods` | **Pods**|**Type:** `object`
| | `connectivity.network.pods.cidrBlocks` | **Pod subnets**|**Type:** `array`
**Default:** `["100.64.0.0/12"]`| | `connectivity.network.pods.cidrBlocks[*]` | **Pod subnet** - IPv4 address range for pods, in CIDR notation.|**Type:** `string`
**Example:** `"10.244.0.0/16"`
| @@ -54,6 +55,7 @@ Properties within the `.connectivity` top-level object | `connectivity.network.services.cidrBlocks` | **K8s Service subnets**|**Type:** `array`
**Default:** `["172.31.0.0/16"]`| | `connectivity.network.services.cidrBlocks[*]` | **Service subnet** - IPv4 address range for kubernetes services, in CIDR notation.|**Type:** `string`
**Example:** `"172.31.0.0/16"`
| | `connectivity.network.vpcCidr` | **VPC subnet** - IPv4 address range to assign to this cluster's VPC, in CIDR notation.|**Type:** `string`
**Default:** `"10.0.0.0/16"`| +| `connectivity.network.vpcId` | **VPC id** - ID of the VPC, where the cluster will be deployed. The VPC must exist and it case this is set, VPC wont be created by controllers.|**Type:** `string`
| | `connectivity.proxy` | **Proxy** - Whether/how outgoing traffic is routed through proxy servers.|**Type:** `object`
| | `connectivity.proxy.enabled` | **Enable**|**Type:** `boolean`
| | `connectivity.proxy.httpProxy` | **HTTP proxy** - To be passed to the HTTP_PROXY environment variable in all hosts.|**Type:** `string`
| @@ -68,7 +70,10 @@ Properties within the `.connectivity` top-level object | `connectivity.subnets[*].cidrBlocks[*].cidr` | **Address range** - IPv4 address range, in CIDR notation.|**Type:** `string`
| | `connectivity.subnets[*].cidrBlocks[*].tags` | **Tags** - AWS resource tags to assign to this subnet.|**Type:** `object`
| | `connectivity.subnets[*].cidrBlocks[*].tags.*` | **Tag value**|**Type:** `string`
**Value pattern:** `^[ a-zA-Z0-9\._:/=+-@]+$`
| +| `connectivity.subnets[*].id` | **ID of the subnet** - ID of an existing subnet. When set, this subnet will be used instead of creating a new one.|**Type:** `string`
| | `connectivity.subnets[*].isPublic` | **Public**|**Type:** `boolean`
| +| `connectivity.subnets[*].natGatewayId` | **ID of the NAT Gateway** - ID of the NAT Gateway used for this existing subnet.|**Type:** `string`
| +| `connectivity.subnets[*].routeTableId` | **ID of route table** - ID of the route table, assigned to the existing subnet. Must be provided when defining subnet via ID.|**Type:** `string`
| | `connectivity.subnets[*].tags` | **Tags** - AWS resource tags to assign to this CIDR block.|**Type:** `object`
| | `connectivity.subnets[*].tags.*` | **Tag value**|**Type:** `string`
**Value pattern:** `^[ a-zA-Z0-9\._:/=+-@]+$`
| | `connectivity.topology` | **Topology** - Networking architecture between management cluster and workload cluster.|**Type:** `object`
| @@ -83,6 +88,13 @@ Properties within the `.controlPlane` top-level object | **Property** | **Description** | **More Details** | | :----------- | :-------------- | :--------------- | +| `controlPlane.additionalSecurityGroups` | **Control Plane additional security groups** - Additional security groups that will be added to the control plane nodes.|**Type:** `array`
| +| `controlPlane.additionalSecurityGroups[*]` | **Security group**|**Type:** `object`
| +| `controlPlane.additionalSecurityGroups[*].id` | **Id of the security group** - ID of the security group that will be added to the control plane nodes. The security group must exist.|**Type:** `string`
| +| `controlPlane.apiExtraArgs` | **API extra arguments** - Extra arguments passed to the kubernetes API server.|**Type:** `object`
| +| `controlPlane.apiExtraArgs.PATTERN` | **argument**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^.+:.+$`
| +| `controlPlane.apiExtraCertSANs` | **API extra cert SANs** - Extra certs SANs passed to the kubeadmcontrolplane CR.|**Type:** `array`
| +| `controlPlane.apiExtraCertSANs[*]` | **cert SAN**|**Type:** `string`
| | `controlPlane.apiMode` | **API mode** - Whether the Kubernetes API server load balancer should be reachable from the internet (public) or internal only (private).|**Type:** `string`
**Default:** `"public"`| | `controlPlane.containerdVolumeSizeGB` | **Containerd volume size (GB)**|**Type:** `integer`
**Default:** `100`| | `controlPlane.etcdVolumeSizeGB` | **Etcd volume size (GB)**|**Type:** `integer`
**Default:** `100`| @@ -113,8 +125,27 @@ For Giant Swarm internal use only, not stable, or not supported by UIs. | :----------- | :-------------- | :--------------- | | `internal.hashSalt` | **Hash salt** - If specified, this token is used as a salt to the hash suffix of some resource names. Can be used to force-recreate some resources.|**Type:** `string`
| | `internal.kubernetesVersion` | **Kubernetes version**|**Type:** `string`
**Example:** `"1.24.7"`
**Default:** `"1.24.14"`| +| `internal.migration` | **Migration values** - Section used for migration of cluster from vintage to CAPI|**Type:** `object`
| +| `internal.migration.apiBindPort` | **Kubernetes API bind port** - Kubernetes API bind port used for kube api pod|**Type:** `integer`
**Default:** `6443`| +| `internal.migration.controlPlaneExtraFiles` | **Control Plane extra files** - Additional fiels that will be provisioned to control-plane nodes, reference is from secret in the same namespace.|**Type:** `array`
| +| `internal.migration.controlPlaneExtraFiles[*]` | **file**|**Type:** `object`
| +| `internal.migration.controlPlaneExtraFiles[*].contentFrom` | **content from**|**Type:** `object`
| +| `internal.migration.controlPlaneExtraFiles[*].contentFrom.secret` | **secret**|**Type:** `object`
| +| `internal.migration.controlPlaneExtraFiles[*].contentFrom.secret.key` | **secret key for file content**|**Type:** `string`
| +| `internal.migration.controlPlaneExtraFiles[*].contentFrom.secret.name` | **secret name for file content**|**Type:** `string`
| +| `internal.migration.controlPlaneExtraFiles[*].path` | **file path**|**Type:** `string`
| +| `internal.migration.controlPlaneExtraFiles[*].permissions` | **file permissions in form 0644**|**Type:** `string`
**Default:** `"0644"`| +| `internal.migration.controlPlanePostKubeadmCommands` | **Control Plane Post Kubeadm Commands** - Additional Post-Kubeadm Commands executed on the control plane node.|**Type:** `array`
| +| `internal.migration.controlPlanePostKubeadmCommands[*]` | **command**|**Type:** `string`
| +| `internal.migration.controlPlanePreKubeadmCommands` | **Control Plane Pre Kubeadm Commands** - Additional Pre-Kubeadm Commands executed on the control plane node.|**Type:** `array`
| +| `internal.migration.controlPlanePreKubeadmCommands[*]` | **command**|**Type:** `string`
| +| `internal.migration.etcdExtraArgs` | **Etcd extra arguments**|**Type:** `object`
| +| `internal.migration.etcdExtraArgs.PATTERN` | **argument**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^.+:.+$`
| | `internal.nodePools` | **Default node pool**|**Type:** `object`
**Default:** `{"def00":{"customNodeLabels":["label=default"],"instanceType":"r6i.xlarge","maxSize":3,"minSize":3}}`| | `internal.nodePools.PATTERN` | **Node pool**|**Type:** `object`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| +| `internal.nodePools.PATTERN.additionalSecurityGroups` | **Machine pool additional security groups** - Additional security groups that will be added to the machine pool nodes.|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| +| `internal.nodePools.PATTERN.additionalSecurityGroups[*]` | **security group**|**Type:** `object`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| +| `internal.nodePools.PATTERN.additionalSecurityGroups[*].id` | **Id of the security group** - ID of the security group that will be added to the machine pool nodes. The security group must exist.|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| | `internal.nodePools.PATTERN.availabilityZones` | **Availability zones**|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| | `internal.nodePools.PATTERN.availabilityZones[*]` | **Availability zone**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| | `internal.nodePools.PATTERN.customNodeLabels` | **Custom node labels**|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| @@ -165,6 +196,9 @@ Node pools of the cluster. If not specified, this defaults to the value of `inte | **Property** | **Description** | **More Details** | | :----------- | :-------------- | :--------------- | | `nodePools.PATTERN` | **Node pool**|**Type:** `object`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| +| `nodePools.PATTERN.additionalSecurityGroups` | **Machine pool additional security groups** - Additional security groups that will be added to the machine pool nodes.|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| +| `nodePools.PATTERN.additionalSecurityGroups[*]` | **security group**|**Type:** `object`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| +| `nodePools.PATTERN.additionalSecurityGroups[*].id` | **Id of the security group** - ID of the security group that will be added to the machine pool nodes. The security group must exist.|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| | `nodePools.PATTERN.availabilityZones` | **Availability zones**|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| | `nodePools.PATTERN.availabilityZones[*]` | **Availability zone**|**Type:** `string`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| | `nodePools.PATTERN.customNodeLabels` | **Custom node labels**|**Type:** `array`
**Key pattern:**
`PATTERN`=`^[a-z0-9]{5,10}$`
| diff --git a/helm/cluster-aws/templates/_aws_cluster.tpl b/helm/cluster-aws/templates/_aws_cluster.tpl index be6acd61..0984d312 100644 --- a/helm/cluster-aws/templates/_aws_cluster.tpl +++ b/helm/cluster-aws/templates/_aws_cluster.tpl @@ -41,9 +41,24 @@ spec: toPort: -1 vpc: availabilityZoneUsageLimit: {{ .Values.connectivity.availabilityZoneUsageLimit }} + {{- if .Values.connectivity.network.vpcId }} + id: {{ .Values.connectivity.network.vpcId }} + {{- else }} cidrBlock: {{ .Values.connectivity.network.vpcCidr }} + {{- end }} + {{- if .Values.connectivity.network.internetGatewayId }} + internetGatewayId: {{ .Values.connectivity.network.internetGatewayId }} + {{- end }} subnets: {{- range $j, $subnet := .Values.connectivity.subnets }} + {{- if $subnet.id }} + - id: {{ $subnet.id }} + isPublic: {{ $subnet.isPublic }} + routeTableId: {{ $subnet.routeTableId }} + {{- if $subnet.natGatewayId }} + natGatewayId: {{ $subnet.natGatewayId }} + {{- end }} + {{- else }} {{- range $i, $cidr := $subnet.cidrBlocks }} - cidrBlock: "{{ $cidr.cidr }}" {{- if eq (len $cidr.availabilityZone) 1 }} @@ -59,6 +74,7 @@ spec: {{- end }} {{- end }} {{- end }} + {{- end }} sshKeyName: ssh-key s3Bucket: controlPlaneIAMInstanceProfile: control-plane-{{ include "resource.default.name" $ }} diff --git a/helm/cluster-aws/templates/_control_plane.tpl b/helm/cluster-aws/templates/_control_plane.tpl index 4e7a131a..e2676d81 100644 --- a/helm/cluster-aws/templates/_control_plane.tpl +++ b/helm/cluster-aws/templates/_control_plane.tpl @@ -30,6 +30,10 @@ template: size: {{ .Values.controlPlane.rootVolumeSizeGB }} type: gp3 iamInstanceProfile: control-plane-{{ include "resource.default.name" $ }} + {{- if .Values.controlPlane.additionalSecurityGroups }} + additionalSecurityGroups: + {{- toYaml .Values.controlPlane.additionalSecurityGroups | nindent 4 }} + {{- end }} sshKeyName: "" subnet: filters: @@ -93,6 +97,9 @@ spec: certSANs: - "api.{{ include "resource.default.name" $ }}.{{ required "The baseDomain value is required" .Values.baseDomain }}" - 127.0.0.1 + {{- if .Values.controlPlane.apiExtraCertSANs -}} + {{- toYaml .Values.controlPlane.apiExtraCertSANs | nindent 10 }} + {{- end }} extraArgs: cloud-provider: external service-account-issuer: "https://irsa.{{ include "resource.default.name" $ }}.{{ required "The baseDomain value is required" .Values.baseDomain }}" @@ -122,6 +129,9 @@ spec: service-account-lookup: "true" tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 service-cluster-ip-range: {{ .Values.connectivity.network.services.cidrBlocks | first }} + {{- if .Values.controlPlane.apiExtraArgs -}} + {{- toYaml .Values.controlPlane.apiExtraArgs | nindent 10 }} + {{- end }} extraVolumes: - name: auditlog hostPath: /var/log/apiserver @@ -157,6 +167,9 @@ spec: extraArgs: listen-metrics-urls: "http://0.0.0.0:2381" quota-backend-bytes: "8589934592" + {{- if .Values.internal.migration.etcdExtraArgs -}} + {{- toYaml .Values.internal.migration.etcdExtraArgs | nindent 12 }} + {{- end }} networking: serviceSubnet: {{ join "," .Values.connectivity.network.services.cidrBlocks }} files: @@ -167,13 +180,16 @@ spec: {{- if .Values.connectivity.proxy.enabled }}{{- include "proxyFiles" . | nindent 4 }}{{- end }} {{- include "kubernetesFiles" . | nindent 4 }} {{- include "registryFiles" . | nindent 4 }} + {{- if .Values.internal.migration.controlPlaneExtraFiles }} + {{- toYaml .Values.internal.migration.controlPlaneExtraFiles | nindent 4}} + {{- end }} initConfiguration: skipPhases: - addon/kube-proxy - addon/coredns localAPIEndpoint: advertiseAddress: "" - bindPort: 0 + bindPort: {{ .Values.internal.migration.apiBindPort }} nodeRegistration: kubeletExtraArgs: cloud-provider: external @@ -194,6 +210,9 @@ spec: {{- end }} joinConfiguration: discovery: {} + controlPlane: + localAPIEndpoint: + bindPort: {{ .Values.internal.migration.apiBindPort }} nodeRegistration: kubeletExtraArgs: cloud-provider: external @@ -210,12 +229,18 @@ spec: {{- end }} {{- end }} preKubeadmCommands: - {{- include "flatcarKubeadmPreCommands" . | nindent 4 }} {{- include "sshPreKubeadmCommands" . | nindent 4 }} + {{- if .Values.internal.migration.controlPlanePreKubeadmCommands -}} + {{- toYaml .Values.internal.migration.controlPlanePreKubeadmCommands | nindent 4 }} + {{- end }} + {{- include "flatcarKubeadmPreCommands" . | nindent 4 }} {{- if .Values.connectivity.proxy.enabled }}{{- include "proxyCommand" $ | nindent 4 }}{{- end }} postKubeadmCommands: {{- include "kubeletConfigPostKubeadmCommands" . | nindent 4 }} {{- include "controlPlanePostKubeadmCommands" . | nindent 4 }} + {{- if .Values.internal.migration.controlPlanePostKubeadmCommands -}} + {{- toYaml .Values.internal.migration.controlPlanePostKubeadmCommands | nindent 4 }} + {{- end }} users: {{- include "sshUsers" . | nindent 4 }} replicas: 3 diff --git a/helm/cluster-aws/templates/_machine_pools.tpl b/helm/cluster-aws/templates/_machine_pools.tpl index 4fa84b6a..974edbd0 100644 --- a/helm/cluster-aws/templates/_machine_pools.tpl +++ b/helm/cluster-aws/templates/_machine_pools.tpl @@ -66,6 +66,10 @@ spec: size: {{ $value.rootVolumeSizeGB | default 300 }} type: gp3 sshKeyName: "" + {{- if $value.additionalSecurityGroups }} + additionalSecurityGroups: + {{- toYaml $value.additionalSecurityGroups | nindent 4 }} + {{- end }} {{- if and $value.spotInstances $value.spotInstances.enabled }} spotMarketOptions: maxPrice: {{ $value.spotInstances.maxPrice | quote }} diff --git a/helm/cluster-aws/values.schema.json b/helm/cluster-aws/values.schema.json index 40f806f0..e95a7bb8 100644 --- a/helm/cluster-aws/values.schema.json +++ b/helm/cluster-aws/values.schema.json @@ -13,6 +13,22 @@ "type": "object", "title": "Node pool", "properties": { + "additionalSecurityGroups": { + "type": "array", + "title": "Machine pool additional security groups", + "description": "Additional security groups that will be added to the machine pool nodes.", + "items": { + "type": "object", + "title": "security group", + "properties": { + "id": { + "type": "string", + "title": "Id of the security group", + "description": "ID of the security group that will be added to the machine pool nodes. The security group must exist." + } + } + } + }, "availabilityZones": { "type": "array", "title": "Availability zones", @@ -244,6 +260,11 @@ "type": "object", "title": "Network", "properties": { + "internetGatewayId": { + "type": "string", + "title": "Internet Gateway ID", + "description": "ID of the Internet gateway for the VPC." + }, "pods": { "type": "object", "title": "Pods", @@ -295,6 +316,11 @@ "title": "VPC subnet", "description": "IPv4 address range to assign to this cluster's VPC, in CIDR notation.", "default": "10.0.0.0/16" + }, + "vpcId": { + "type": "string", + "title": "VPC id", + "description": "ID of the VPC, where the cluster will be deployed. The VPC must exist and it case this is set, VPC wont be created by controllers." } } }, @@ -367,10 +393,25 @@ } } }, + "id": { + "type": "string", + "title": "ID of the subnet", + "description": "ID of an existing subnet. When set, this subnet will be used instead of creating a new one." + }, "isPublic": { "type": "boolean", "title": "Public" }, + "natGatewayId": { + "type": "string", + "title": "ID of the NAT Gateway", + "description": "ID of the NAT Gateway used for this existing subnet." + }, + "routeTableId": { + "type": "string", + "title": "ID of route table", + "description": "ID of the route table, assigned to the existing subnet. Must be provided when defining subnet via ID." + }, "tags": { "type": "object", "title": "Tags", @@ -472,6 +513,48 @@ "type": "object", "title": "Control plane", "properties": { + "additionalSecurityGroups": { + "type": "array", + "title": "Control Plane additional security groups", + "description": "Additional security groups that will be added to the control plane nodes.", + "items": { + "type": "object", + "title": "Security group", + "properties": { + "id": { + "type": "string", + "title": "Id of the security group", + "description": "ID of the security group that will be added to the control plane nodes. The security group must exist." + } + } + } + }, + "apiExtraArgs": { + "type": "object", + "title": "API extra arguments", + "description": "Extra arguments passed to the kubernetes API server.", + "patternProperties": { + "^.+:.+$": { + "type": "string", + "title": "argument" + } + }, + "example": [ + "audit-log-maxsize: 100" + ] + }, + "apiExtraCertSANs": { + "type": "array", + "title": "API extra cert SANs", + "description": "Extra certs SANs passed to the kubeadmcontrolplane CR.", + "items": { + "type": "string", + "title": "cert SAN", + "example": [ + "test.mydomain.com" + ] + } + }, "apiMode": { "type": "string", "title": "API mode", @@ -614,6 +697,87 @@ ], "default": "1.24.14" }, + "migration": { + "type": "object", + "title": "Migration values", + "description": "Section used for migration of cluster from vintage to CAPI", + "properties": { + "apiBindPort": { + "type": "integer", + "title": "Kubernetes API bind port", + "description": "Kubernetes API bind port used for kube api pod", + "default": 6443 + }, + "controlPlaneExtraFiles": { + "type": "array", + "title": "Control Plane extra files", + "description": "Additional fiels that will be provisioned to control-plane nodes, reference is from secret in the same namespace.", + "items": { + "type": "object", + "title": "file", + "properties": { + "contentFrom": { + "type": "object", + "title": "content from", + "properties": { + "secret": { + "type": "object", + "title": "secret", + "properties": { + "key": { + "type": "string", + "title": "secret key for file content" + }, + "name": { + "type": "string", + "title": "secret name for file content" + } + } + } + } + }, + "path": { + "type": "string", + "title": "file path" + }, + "permissions": { + "type": "string", + "title": "file permissions in form 0644", + "default": "0644" + } + } + } + }, + "controlPlanePostKubeadmCommands": { + "type": "array", + "title": "Control Plane Post Kubeadm Commands", + "description": "Additional Post-Kubeadm Commands executed on the control plane node.", + "items": { + "type": "string", + "title": "command" + } + }, + "controlPlanePreKubeadmCommands": { + "type": "array", + "title": "Control Plane Pre Kubeadm Commands", + "description": "Additional Pre-Kubeadm Commands executed on the control plane node.", + "items": { + "type": "string", + "title": "command" + } + }, + "etcdExtraArgs": { + "type": "object", + "title": "Etcd extra arguments", + "patternProperties": { + "^.+:.+$": { + "type": "string", + "title": "argument" + } + } + } + } + }, "nodePools": { "type": "object", "title": "Default node pool", diff --git a/helm/cluster-aws/values.yaml b/helm/cluster-aws/values.yaml index ab8033cf..b36d663d 100644 --- a/helm/cluster-aws/values.yaml +++ b/helm/cluster-aws/values.yaml @@ -58,6 +58,8 @@ controlPlane: rootVolumeSizeGB: 120 internal: kubernetesVersion: 1.24.14 + migration: + apiBindPort: 6443 nodePools: def00: customNodeLabels: