Merge pull request #7 from dsohk/rancher-2.6

Updated lab materials to be Rancher 2.6 and RKE2 based
dsohk · Sep 28, 2021 · 9ddac1f · 9ddac1f
2 parents e4813fb + 63e599f
commit 9ddac1f
Show file tree

Hide file tree

Showing 12 changed files with 260 additions and 78 deletions.
diff --git a/setup-rke-cluster1.sh b/setup-rke-cluster1.sh
@@ -27,5 +27,5 @@ echo "Registering cluster1 as All-in-one RKE..."
 SSH_VM=$(<ssh-mylab-cluster1.sh)
 CMD="$RANCHER_REGCMD --node-name cluster1 --address $PUB_IP --internal-address $PRIV_IP --etcd --controlplane --worker"
 echo $CMD
-eval "$SSH_VM $CMD"
+eval "$SSH_VM \"$CMD\""
 
diff --git a/setup-rke-cluster2.sh b/setup-rke-cluster2.sh
@@ -27,5 +27,5 @@ echo "Registering cluster2 as All-in-one RKE..."
 SSH_VM=$(<ssh-mylab-cluster2.sh)
 CMD="$RANCHER_REGCMD --node-name cluster2 --address $PUB_IP --internal-address $PRIV_IP --etcd --controlplane --worker"
 echo $CMD
-eval "$SSH_VM $CMD"
+eval "$SSH_VM \"$CMD\""
 
diff --git a/setup-rke-devsecops.sh b/setup-rke-devsecops.sh
@@ -27,7 +27,7 @@ if [ -f ssh-mylab-devsecops-m1.sh ]; then
   SSH_VM=$(<ssh-mylab-devsecops-m1.sh)
   CMD="$RANCHER_REGCMD --node-name devsecops-m1 --address $PUB_IP --internal-address $PRIV_IP --etcd --controlplane"
   echo $CMD
-  eval "$SSH_VM $CMD"
+  eval "$SSH_VM \"$CMD\""
   sleep 10
 fi
 
@@ -45,10 +45,11 @@ do
     SSH_VM=$(<ssh-mylab-devsecops-w$n.sh)
     CMD="$RANCHER_REGCMD --node-name devsecops-w$n --address $PUB_IP --internal-address $PRIV_IP --worker"
     echo $CMD
-    eval "$SSH_VM $CMD"
+    eval "$SSH_VM \"$CMD\""
   fi
 done 
 
+
 echo
 echo "The devsecops cluster is now being provisioned by Rancher. It may take a few minutes to complete."
 echo "Once it's ready, please install Longhorn on it and download KUBECONFIG file into your Harbor VM. Thank you!"

diff --git a/setup/_awsls_functions.sh b/setup/_awsls_functions.sh
@@ -5,16 +5,8 @@
 function create-vm() {
 
   # Randomly choose availability zone in the selected AWS region ...
-  if [ "ap-south-1" == $AWS_REGION ]; then
-    export AWS_AVAIL_AZ=("a" "b")
-  elif [ "ap-northeast-1" == $AWS_REGION ]; then
-    export AWS_AVAIL_AZ=("a" "c" "d") 
-  elif [ "ap-northeast-2" == $AWS_REGION ]; then
-    export AWS_AVAIL_AZ=("a" "c")
-  else
-    export AWS_AVAIL_AZ=("a" "b" "c") 
-  fi
-  AWS_SELECTED_AZ=${AWS_AVAIL_AZ[$RANDOM % ${#AWS_AVAIL_AZ[@]} ]}
+  IFS=', ' read -r -a AVAIL_AZ <<< "$AWS_AVAIL_AZ"
+  AWS_SELECTED_AZ=${AVAIL_AZ[$RANDOM % ${#AVAIL_AZ[@]} ]}
   AWS_AZ=${AWS_REGION}${AWS_SELECTED_AZ}
 
   aws lightsail create-instances \

diff --git a/setup/_awsls_locations.txt b/setup/_awsls_locations.txt
@@ -0,0 +1,19 @@
+# https://lightsail.aws.amazon.com/ls/docs/overview/article/understanding-regions-and-availability-zones-in-amazon-lightsail
+Continent | Region                | Region Code    | AZ          | Bunddle_Suffix
+----------+-----------------------+----------------+-------------+---------------
+US        | US East - N. Virginia | us-east-1      | a,b,c,d,e,f | 2_0
+US        | US East - Ohio        | us-east-2      | a,b,c       | 2_0
+US        | US West - Oregon      | us-west-2      | a,b,c       | 2_0
+US        | Canada (Central)      | ca-central-1   | a,b         | 2_0
+EU        | Frankfurt             | eu-central-1   | a,b,c       | 2_0
+EU        | Ireland               | eu-west-1      | a,b,c       | 2_0
+EU        | London                | eu-west-2      | a,b,c       | 2_0
+EU        | Paris                 | eu-west-3      | a,b,c       | 2_0
+EU        | Stockholm             | eu-north-1     | a,b,c       | 2_0
+AP        | Mumbai                | ap-south-1     | a,b         | 2_1
+AP        | Seoul                 | ap-northeast-2 | a,c         | 2_0
+AP        | Singapore             | ap-southeast-1 | a,b,c       | 2_0
+AP        | Sydney                | ap-southeast-2 | a,b,c       | 2_2
+AP        | Tokyo                 | ap-northeast-1 | a,c,d       | 2_0
+
+
diff --git a/setup/harbor/04-configure-containerd-registry.sh b/setup/harbor/04-configure-containerd-registry.sh
@@ -0,0 +1,30 @@
+#! /bin/bash
+
+source $HOME/myharbor.sh
+
+echo "Configure containerd to access harbor instance with self-signed cert ..."
+sudo mkdir -p /etc/rancher/rke2
+
+echo "Download Harbor CA cert into /etc/rancher/rke2/demo-harbor folder ..."
+sudo mkdir -p /etc/rancher/rke2/demo-harbor
+openssl s_client -showcerts -connect $HARBOR_URL < /dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > ca.crt
+sudo mv ca.crt /etc/rancher/rke2/demo-harbor
+
+export REGISTRY_YAML=/etc/rancher/rke2/registries.yaml
+sudo echo "configs:" > $REGISTRY_YAML
+sudo echo "  \"${HARBOR_URL}\":" >> $REGISTRY_YAML
+sudo echo "    auth:" >> $REGISTRY_YAML
+sudo echo "      username: ${HARBOR_USR}" >> $REGISTRY_YAML
+sudo echo "      password: ${HARBOR_PWD}" >> $REGISTRY_YAML
+sudo echo "    tls:" >> $REGISTRY_YAML
+sudo echo "      ca_file: /etc/rancher/rke2/demo-harbor/ca.crt" >> $REGISTRY_YAML
+sudo echo "      insecure_skip_verify: true" >> $REGISTRY_YAML
+
+if sudo systemctl list-units --type=service | grep rke2-server; then
+  sudo systemctl restart rke2-server
+fi
+
+if sudo systemctl list-units --type=service | grep rke2-agent; then
+  sudo systemctl restart rke2-agent
+fi
+
diff --git a/setup/jenkins/01-distribute-harbor-ca-to-other-vms.sh b/setup/jenkins/01-distribute-harbor-ca-to-other-vms.sh
@@ -4,7 +4,7 @@ for vm in rancher devsecops-m1 devsecops-w1 devsecops-w2 devsecops-w3 devsecops-
   echo
   echo "Distribute the self-signed harbor certs to $vm ..."
   scp $HOME/myharbor.sh $vm:~
-  scp $HOME/04-configure-docker-client.sh $vm:~/configure-docker-client.sh
-  ssh $vm ./configure-docker-client.sh
+  scp $HOME/04-configure-containerd-registry.sh $vm:~/configure-containerd-node.sh
+  ssh $vm "sudo ./configure-containerd-node.sh"
 done
 
diff --git a/setup/jenkins/99-one-step-install-jenkins.sh b/setup/jenkins/99-one-step-install-jenkins.sh
@@ -4,8 +4,8 @@ for vm in rancher devsecops-m1 devsecops-w1 devsecops-w2 devsecops-w3 devsecops-
   echo
   echo "Distribute the self-signed harbor certs to $vm ..."
   scp $HOME/myharbor.sh $vm:~
-  scp $HOME/04-configure-docker-client.sh $vm:~/configure-docker-client.sh
-  ssh $vm ./configure-docker-client.sh
+  scp $HOME/04-configure-containerd-registry.sh $vm:~/configure-containerd-node.sh
+  ssh $vm "sudo ./configure-containerd-node.sh"
 done
 
 #! /bin/bash -e

diff --git a/setup/jenkins/jenkins-values-template.yaml b/setup/jenkins/jenkins-values-template.yaml
@@ -31,7 +31,7 @@ controller:
   #         cpu: 10m
   #         memory: 32Mi
   prometheus:
-    enabled: true
+    enabled: false
     # Additional labels to add to the ServiceMonitor object
     serviceMonitorAdditionalLabels: {}
     # Set a custom namespace where to deploy ServiceMonitor resource

diff --git a/setup/rancher/02-install-rancher-server.sh b/setup/rancher/02-install-rancher-server.sh
@@ -1,23 +1,76 @@
 #! /bin/bash -e
 
 # install rancher server
-echo "Install Rancher Server ..."
+echo "Install Rancher Server using helm chart on RKE2 ..."
 
-sudo mkdir -p /opt/rancher
+source $HOME/mylab_rancher_version.sh
 
-sudo docker run -d --restart=unless-stopped \
-  -p 80:80 -p 443:443 \
-  --privileged \
-  -v /opt/rancher:/var/lib/rancher \
-  rancher/rancher:v2.5.9 \
+echo "Install RKE2 v1.21 ..."
+sudo bash -c 'curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL="v1.21" sh -'
+sudo mkdir -p /etc/rancher/rke2
+sudo bash -c 'echo "write-kubeconfig-mode: \"0644\"" > /etc/rancher/rke2/config.yaml'
+sudo systemctl enable rke2-server.service
+sudo systemctl start rke2-server.service
+
+mkdir -p $HOME/.kube
+ln -s /etc/rancher/rke2/rke2.yaml $HOME/.kube/config
+export KUBECONFIG=$HOME/.kube/config
+
+# Wait until the RKE2 is ready
+echo "Initializing RKE2 cluster ..."
+while [ `kubectl get deploy -n kube-system | grep 1/1 | wc -l` -ne 3 ]
+do
+  sleep 5
+  kubectl get po -n kube-system
+done
+echo "Your RKE2 cluster is ready!"
+kubectl get node
+
+echo "Install Cert Manager v1.5.1 ..."
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml
+helm repo add jetstack https://charts.jetstack.io
+helm install \
+  cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --version v1.5.1 \
+  --create-namespace
+kubectl -n cert-manager rollout status deploy/cert-manager
+
+# Wait until cert-manager deployment complete
+echo "Wait until cert-manager deployment finish ..."
+while [ `kubectl get deploy -n cert-manager | grep 1/1 | wc -l` -ne 3 ]
+do
+  sleep 5
+  kubectl get po -n cert-manager
+done
+
+# Install Rancher with helm chart
+echo "Install Rancher ${RANCHER_VERSION} ..."
+RANCHER_IP=`curl -qs http://checkip.amazonaws.com`
+RANCHER_FQDN=rancher.$RANCHER_IP.sslip.io
+helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
+helm install rancher rancher-latest/rancher \
+  --namespace cattle-system \
+  --set hostname=$RANCHER_FQDN \
+  --set replicas=1 \
+  --version ${RANCHER_VERSION} --devel \
+  --create-namespace
+
+echo "Wait until cattle-system deployment finish ..."
+while [ `kubectl get deploy -n cattle-system | grep 1/1 | wc -l` -ne 1 ]
+do
+  sleep 5
+  kubectl get po -n cattle-system
+done
+
+RANCHER_BOOTSTRAP_PWD=`kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{ "\n" }}'`
 
-export RANCHER_IP=`curl -qs http://checkip.amazonaws.com`
 
 echo
 echo "---------------------------------------------------------"
-echo "Please wait for 5-10 mins to initializing Rancher server."
+echo "Your Rancher Server is ready."
 echo
-echo "Your Rancher Server URL: https://${RANCHER_IP}" > rancher-url.txt
+echo "Your Rancher Server URL: https://${RANCHER_FQDN}" > rancher-url.txt
+echo "Bootstrap Password: ${RANCHER_BOOTSTRAP_PWD}" >> rancher-url.txt
 cat rancher-url.txt
-echo
-
+echo "---------------------------------------------------------"
diff --git a/setup/rancher/99-one-step-install-rancher.sh b/setup/rancher/99-one-step-install-rancher.sh
@@ -17,23 +17,76 @@ sudo mv /home/ec2-user/.arkade/bin/kubectl /usr/local/bin/
 #! /bin/bash -e
 
 # install rancher server
-echo "Install Rancher Server ..."
+echo "Install Rancher Server using helm chart on RKE2 ..."
 
-sudo mkdir -p /opt/rancher
+source $HOME/mylab_rancher_version.sh
 
-sudo docker run -d --restart=unless-stopped \
-  -p 80:80 -p 443:443 \
-  --privileged \
-  -v /opt/rancher:/var/lib/rancher \
-  rancher/rancher:v2.5.9 \
+echo "Install RKE2 v1.21 ..."
+sudo bash -c 'curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL="v1.21" sh -'
+sudo mkdir -p /etc/rancher/rke2
+sudo bash -c 'echo "write-kubeconfig-mode: \"0644\"" > /etc/rancher/rke2/config.yaml'
+sudo systemctl enable rke2-server.service
+sudo systemctl start rke2-server.service
+
+mkdir -p $HOME/.kube
+ln -s /etc/rancher/rke2/rke2.yaml $HOME/.kube/config
+export KUBECONFIG=$HOME/.kube/config
+
+# Wait until the RKE2 is ready
+echo "Initializing RKE2 cluster ..."
+while [ `kubectl get deploy -n kube-system | grep 1/1 | wc -l` -ne 3 ]
+do
+  sleep 5
+  kubectl get po -n kube-system
+done
+echo "Your RKE2 cluster is ready!"
+kubectl get node
+
+echo "Install Cert Manager v1.5.1 ..."
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml
+helm repo add jetstack https://charts.jetstack.io
+helm install \
+  cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --version v1.5.1 \
+  --create-namespace
+kubectl -n cert-manager rollout status deploy/cert-manager
+
+# Wait until cert-manager deployment complete
+echo "Wait until cert-manager deployment finish ..."
+while [ `kubectl get deploy -n cert-manager | grep 1/1 | wc -l` -ne 3 ]
+do
+  sleep 5
+  kubectl get po -n cert-manager
+done
+
+# Install Rancher with helm chart
+echo "Install Rancher ${RANCHER_VERSION} ..."
+RANCHER_IP=`curl -qs http://checkip.amazonaws.com`
+RANCHER_FQDN=rancher.$RANCHER_IP.sslip.io
+helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
+helm install rancher rancher-latest/rancher \
+  --namespace cattle-system \
+  --set hostname=$RANCHER_FQDN \
+  --set replicas=1 \
+  --version 2.6.0 \
+  --create-namespace
+
+echo "Wait until cattle-system deployment finish ..."
+while [ `kubectl get deploy -n cattle-system | grep 1/1 | wc -l` -ne 1 ]
+do
+  sleep 5
+  kubectl get po -n cattle-system
+done
+
+RANCHER_BOOTSTRAP_PWD=`kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{ "\n" }}'`
 
-export RANCHER_IP=`curl -qs http://checkip.amazonaws.com`
 
 echo
 echo "---------------------------------------------------------"
-echo "Please wait for 5-10 mins to initializing Rancher server."
+echo "Your Rancher Server is ready."
 echo
-echo "Your Rancher Server URL: https://${RANCHER_IP}" > rancher-url.txt
+echo "Your Rancher Server URL: https://${RANCHER_FQDN}" > rancher-url.txt
+echo "Bootstrap Password: ${RANCHER_BOOTSTRAP_PWD}" >> rancher-url.txt
 cat rancher-url.txt
-echo
-
+echo "---------------------------------------------------------"