From 1555c35b1b315ff31c062a789c5f756a6122d057 Mon Sep 17 00:00:00 2001 From: Martina Iglesias Date: Fri, 4 Sep 2020 12:53:10 +0200 Subject: [PATCH] Add documentation about the bootstrapping in AL2 (#2611) --- docs/current-001-node_bootstrap.md | 231 +++++++++++++++++++++++++---- 1 file changed, 199 insertions(+), 32 deletions(-) diff --git a/docs/current-001-node_bootstrap.md b/docs/current-001-node_bootstrap.md index 4496c8813a..1728e1c93e 100644 --- a/docs/current-001-node_bootstrap.md +++ b/docs/current-001-node_bootstrap.md @@ -61,48 +61,99 @@ Summary: More specifically, as part of CloudFormation template, `eksctl` will define full content of the following files: -- `/etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf` - systemd drop-in for kubelet -- `/etc/eksctl/kubelet.yaml` - fully-formed kubelet configuration file (NOTE: currently it doesn't support all the options that flags support) -- `/etc/eksctl/kubeconfig.yaml` - client credentials for kubelet -- `/etc/eksctl/ca.crt` -- `/etc/eksctl/metadata.env` - known metadata for all nodes in a nodegroup, used by the bootstrap process and for kubelet flags that are not otherwise settable via `/etc/eksctl/kubelet.yaml` - - `AWS_DEFAULT_REGION` - - `AWS_EKS_CLUSTER_NAME` - - `AWS_EKS_ENDPOINT` - - `AWS_EKS_ECR_ACCOUNT` -- `/etc/eksctl/kubelet.env` - kubelet-specific metadata, used for flags that are not otherwise settable via `/etc/eksctl/kubelet.yaml` - - `NODE_LABELS` - - `NODE_TAINTS` - - `MAX_PODS` (optional when user chose to override it) -- `/etc/eksctl/max_pods.map` - used by the bootstrap script (NOTE: based on [`eni-max-pods.txt`](https://raw.github.com/awslabs/amazon-eks-ami/master/files/eni-max-pods.txt), but format differs to ease parsing) +- [`/etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf`](#systemd-unit-for-amazon-linux) - systemd drop-in for kubelet +- [`/etc/eksctl/kubelet.yaml`](#kubelet-configuration) - fully-formed kubelet configuration file (NOTE: currently it doesn't support all the options that flags support) +- [`/etc/eksctl/kubeconfig.yaml`](#kubeconfig) - client credentials for kubelet +- [`/etc/eksctl/ca.crt`](#ca-file) +- [`/etc/eksctl/metadata.env`](#metadata-environment-variables) - known metadata for all nodes in a nodegroup, used by the bootstrap process and for kubelet flags that are not otherwise settable via `/etc/eksctl/kubelet.yaml` +- [`/etc/eksctl/kubelet.env`](#kubelet-environment-variables) - kubelet-specific metadata, used for flags that are not otherwise settable via `/etc/eksctl/kubelet.yaml` +- [`/etc/eksctl/kubelet.local.env`](#kubelet-local-environment-variables) - kubelet-specific metadata that is local to the specific node +- [`/etc/eksctl/max_pods.map`](#max-pods-file) - used by the bootstrap script (NOTE: based on [`eni-max-pods.txt`](https://raw.github.com/awslabs/amazon-eks-ami/master/files/eni-max-pods.txt), but format differs to ease parsing) ## Source Code +The bootstrapping of un-managed nodegroups is done through CloudFormation with the files that eksctl sends as `userData` +(the user data is a property of the launch template used to spawn the nodes in EC2). + +The main files are the [bootstrap script](#bootstrap-script-for-amazon-linux-2), the [systemd unit](#systemd-unit-for-amazon-linux), the +[kubelet.yaml](#kubelet-configuration) and the [kubeconfig.yalm](#kubeconfig). + + ### [Bootstrap Script for Amazon Linux 2](https://github.com/weaveworks/eksctl/blob/master/pkg/nodebootstrap/assets/bootstrap.al2.sh) +This is sent as one of the `runScript`s in the `userData` and it is used to bootstrap the node. It can also be +[overwritten](/usage/schema/#nodeGroups-overrideBootstrapCommand) by the user or appended with +[extra shell commands](/usage/schema/#nodeGroups-preBootstrapCommands). + Dependencies: - systemd - cat - bash - curl +
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+function get_max_pods() {
+  while read instance_type pods; do
+    if  [[ "${instance_type}" == "${1}" ]] && [[ "${pods}" =~ ^[0-9]+$ ]] ; then
+      echo ${pods}
+      return
+    fi
+  done < /etc/eksctl/max_pods.map
+}
+
+# Use IMDSv2 to get metadata
+TOKEN="$(curl --silent -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 600" http://169.254.169.254/latest/api/token)"
+function get_metadata() {
+  curl --silent -H "X-aws-ec2-metadata-token: $TOKEN" "http://169.254.169.254/latest/meta-data/$1"
+}
+
+NODE_IP="$(get_metadata local-ipv4)"
+INSTANCE_ID="$(get_metadata instance-id)"
+INSTANCE_TYPE="$(get_metadata instance-type)"
+AWS_SERVICES_DOMAIN="$(get_metadata services/domain)"
+
+
+source /etc/eksctl/kubelet.env # this can override MAX_PODS
+
+cat > /etc/eksctl/kubelet.local.env  <<EOF
+NODE_IP=${NODE_IP}
+INSTANCE_ID=${INSTANCE_ID}
+INSTANCE_TYPE=${INSTANCE_TYPE}
+AWS_SERVICES_DOMAIN=${AWS_SERVICES_DOMAIN}
+MAX_PODS=${MAX_PODS:-$(get_max_pods "${INSTANCE_TYPE}")}
+EOF
+
+systemctl daemon-reload
+systemctl enable kubelet
+systemctl start kubelet
+
+
+ + ### [Bootstrap Script for Ubuntu](https://github.com/weaveworks/eksctl/blob/master/pkg/nodebootstrap/assets/bootstrap.ubuntu.sh) Same as above, but Ubuntu-specific commands are used instead of `systemctl` and the drop-in unit. -### [Systemd Drop-in Unit for Amazon Linux](https://github.com/weaveworks/eksctl/blob/70041a226bb8ef5c51a229d587235551a2410eda/pkg/nodebootstrap/assets/10-eksclt.al2.conf) +### [Systemd Unit for Amazon Linux](https://github.com/weaveworks/eksctl/blob/70041a226bb8ef5c51a229d587235551a2410eda/pkg/nodebootstrap/assets/10-eksclt.al2.conf) -```ini +This is the systemd unit file used to start the kubelet service. It makes use of other configuration files sent by +eksctl and it is stored in `etc/systemd/system/kubelet.service.d/10-eksctl.al2.conf`. + +
 # eksctl-specific systemd drop-in unit for kubelet, for Amazon Linux 2 (AL2)
 
 [Service]
-# Local metadata parameters: REGION, AWS_DEFAULT_REGION
-EnvironmentFile=/etc/eksctl/metadata.env
-# Global and static parameters: CLUSTER_DNS, NODE_LABELS, NODE_TAINTS
-EnvironmentFile=/etc/eksctl/kubelet.env
-# Local non-static parameters: NODE_IP, INSTANCE_ID
-EnvironmentFile=/etc/eksctl/kubelet.local.env
+EnvironmentFile=/etc/eksctl/metadata.env
+EnvironmentFile=/etc/eksctl/kubelet.env
+EnvironmentFile=/etc/eksctl/kubelet.local.env
 
+ExecStart=
 ExecStart=/usr/bin/kubelet \
   --node-ip=${NODE_IP} \
   --node-labels=${NODE_LABELS},alpha.eksctl.io/instance-id=${INSTANCE_ID} \
@@ -113,20 +164,27 @@ ExecStart=/usr/bin/kubelet \
   --network-plugin=cni \
   --cni-bin-dir=/opt/cni/bin \
   --cni-conf-dir=/etc/cni/net.d \
-  --pod-infra-container-image=${AWS_EKS_ECR_ACCOUNT}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/eks/pause-amd64:3.1 \
-  --kubeconfig=/etc/eksctl/kubeconfig.yaml \
-  --config=/etc/eksctl/kubelet.yaml
-```
+  --pod-infra-container-image=${AWS_EKS_ECR_ACCOUNT}.dkr.ecr.${AWS_DEFAULT_REGION}.${AWS_SERVICES_DOMAIN}/eks/pause:3.1-eksbuild.1 \
+  --kubeconfig=/etc/eksctl/kubeconfig.yaml \
+  --config=/etc/eksctl/kubelet.yaml
+
+
+ + +### [Kubelet configuration](https://github.com/weaveworks/eksctl/blob/70041a226bb8ef5c51a229d587235551a2410eda/pkg/nodebootstrap/assets/kubelet.yaml) -### [`kubelet.yaml`](https://github.com/weaveworks/eksctl/blob/70041a226bb8ef5c51a229d587235551a2410eda/pkg/nodebootstrap/assets/kubelet.yaml) +The configuration for the kubelet is stored in `/etc/eksctl/kubelet.yaml`. -```YAML +
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1
 
+clusterDNS:
+- 10.100.0.10
+
 address: 0.0.0.0
 clusterDomain: cluster.local
-
+serverTLSBootstrap: true
 authentication:
   anonymous:
     enabled: false
@@ -134,7 +192,7 @@ authentication:
     cacheTTL: 2m0s
     enabled: true
   x509:
-    clientCAFile: /etc/eksctl/ca.crt
+    clientCAFile: /etc/eksctl/ca.crt
 
 authorization:
   mode: Webhook
@@ -142,13 +200,122 @@ authorization:
     cacheAuthorizedTTL: 5m0s
     cacheUnauthorizedTTL: 30s
 
-serverTLSBootstrap: true
-
 cgroupDriver: cgroupfs
+kubeReserved:
+  cpu: 60m
+  ephemeral-storage: 1Gi
+  memory: 343Mi
 
 featureGates:
   RotateKubeletServerCertificate: true
+
+ +### Kubeconfig + +The kubeconfig is stored in `/etc/eksctl/kubeconfig.yaml`. + +
+kind: Config
+apiVersion: v1
+
+clusters:
+- cluster:
+    certificate-authority: /etc/eksctl/ca.crt
+    server: https://0A3....gr7.us-west-2.eks.amazonaws.com
+  name: my-cluster-1.us-west-2.eksctl.io
+contexts:
+- context:
+    cluster: my-cluster-1.us-west-2.eksctl.io
+    user: kubelet@my-cluster-1.us-west-2.eksctl.io
+  name: kubelet@my-cluster-1.us-west-2.eksctl.io
+current-context: kubelet@my-cluster-1.us-west-2.eksctl.io
+
+preferences: {}
+
+users:
+- name: kubelet@my-cluster-1.us-west-2.eksctl.io
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1alpha1
+      args:
+      - eks
+      - get-token
+      - --cluster-name
+      - my-cluster-1
+      - --region
+      - us-west-2
+      command: aws
+      env:
+      - name: AWS_STS_REGIONAL_ENDPOINTS
+        value: regional
+
+ +### Kubelet local environment variables + +Stored in `/etc/eksctl/kubelet.local.env`, this file contains dynamic data and is generated by eksctl when generating +the CloudFormation template. + ``` +NODE_IP=192.168.72.51 +INSTANCE_ID=i-0b704274a75a321a7 +INSTANCE_TYPE=m6g.medium +AWS_SERVICES_DOMAIN=amazonaws.com +MAX_PODS=8 +``` + +### Metadata environment variables + +`/etc/eksctl/metadata.env` + +``` +AWS_DEFAULT_REGION=us-west-2 +AWS_EKS_CLUSTER_NAME=my-cluster-1 +AWS_EKS_ENDPOINT=https://0A123.....gr7.us-west-2.eks.amazonaws.com +AWS_EKS_ECR_ACCOUNT=602401143452 + +``` + +### Kubelet environment variables + +`/etc/eksctl/kubelet.env` + +``` +NODE_LABELS=alpha.eksctl.io/cluster-name=my-cluster-1,alpha.eksctl.io/nodegroup-name=ng-1 +NODE_TAINTS= +``` + +### CA File + +`etc/eksctl/ca.crt` + + +``` +-----BEGIN CERTIFICATE----- +1231231231231231231231231231231231231231231231231231231231231233 +ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCA +........ +-----END CERTIFICATE----- +``` + + +### Max pods file + +`/etc/eksctl/max_pods.map` + +``` +t2.micro 4 +x1e.32xlarge 234 +c1.xlarge 58 +g4dn.2xlarge 29 +r5.24xlarge 737 +r5a.12xlarge 234 +r5ad.xlarge 58 +t3.2xlarge 58 +c1.medium 12 +m5d.16xlarge 737 +... +``` + ## Comparison to EKS Amazon Linux 2 script