Include k3s in CI tests (#195)

* scripts: Update deployment scripts to support k8s manager Signed-off-by: Din Music <[email protected]> * github/workflows: Test k3s on each PR Signed-off-by: Din Music <[email protected]> * embed/ansible: Set Ansible 9.3.0 pip requirement Signed-off-by: Din Music <[email protected]> * k3s: Fix fetching of the final kubeconfig Signed-off-by: Din Music <[email protected]> * scripts: Use dnsutils instead of busybox for testing DNS Signed-off-by: Din Music <[email protected]> * scripts: Simplify waiting for nodes and pods readiness Signed-off-by: Din Music <[email protected]> * scripts: Retry pods readiness If pod readiness is checked before any pod is deployed, the command will result in an error. Therefore, retry a few times to avoid unnecessary test failures. Signed-off-by: Din Music <[email protected]> --------- Signed-off-by: Din Music <[email protected]>
MusicDin · Apr 17, 2024 · e307351 · e307351
1 parent a609077
commit e307351
Show file tree

Hide file tree

Showing 8 changed files with 84 additions and 131 deletions.
diff --git a/.github/workflows/tests-deploy-cluster.yml b/.github/workflows/tests-deploy-cluster.yml
diff --git a/.github/workflows/tests-deploy-node.yml → .github/workflows/tests-deployment.yml b/.github/workflows/tests-deploy-node.yml → .github/workflows/tests-deployment.yml
@@ -1,5 +1,5 @@
-name: Test Deployment (Single Node)
-run-name: "${{ github.ref_name }}: Test Deployment (Single Node)"
+name: Test Deployment
+run-name: "${{ github.ref_name }}: Test Deployment"
 
 on:
   workflow_dispatch:
@@ -16,6 +16,44 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Test cluster deployment.
+  test-cluster:
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    name: Cluster
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        manager:
+          - kubespray
+          - k3s
+        k8s:
+          - v1.27.10
+          - v1.28.6
+        distro:
+          - ubuntu22
+        networkPlugin:
+          - calico # K3s will use flannel
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: ./.github/actions/runner-setup
+
+      - name: Deploy cluster
+        run: |
+          ./scripts/deploy-cluster.sh k8s \
+            ${{ matrix.distro }} \
+            ${{ matrix.networkPlugin }} \
+            ${{ matrix.k8s }} \
+            ${{ matrix.manager }}
+
+      - name: Test
+        run: |
+          ./scripts/test-cluster.sh
+
   # Test multiple k8s versions using the default distro and network plugin.
   test-single-node-quick:
     if: github.event_name == 'push' || github.event_name == 'pull_request'
@@ -24,14 +62,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        manager:
+          - k3s
+          - kubespray
         k8sVersion:
-          - v1.26.13
           - v1.27.10
           - v1.28.6
         distro:
           - ubuntu22
         networkPlugin:
-          - calico
+          - calico # K3s will use flannel
 
     steps:
       - name: Checkout
@@ -45,7 +85,8 @@ jobs:
           ./scripts/deploy-node.sh k8s \
             ${{ matrix.distro }} \
             ${{ matrix.networkPlugin }} \
-            ${{ matrix.k8sVersion }}
+            ${{ matrix.k8sVersion }} \
+            ${{ matrix.manager }}
 
       - name: Test
         run: |
@@ -55,15 +96,18 @@ jobs:
         run: |
           ./scripts/destroy-cluster.sh k8s
 
-  # Test most combinations of Kubernetes versions, distros,
-  # and network plugins. Run this only on push.
+  # Test most combinations of managers, versions, distros,
+  # and network plugins. Run this only on schedule.
   test-single-node-all:
-    if: github.event_name != 'push' && github.event_name != 'pull_request'
-    name: Node
+    if: github.event_name == 'schedule'
+    name: Node Matrix
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
+        manager:
+          - k3s
+          - kubespray
         k8sVersion:
           - v1.26.13
           - v1.27.10
@@ -91,7 +135,8 @@ jobs:
           ./scripts/deploy-node.sh k8s \
             ${{ matrix.distro }} \
             ${{ matrix.networkPlugin }} \
-            ${{ matrix.k8sVersion }}
+            ${{ matrix.k8sVersion }} \
+            ${{ matrix.manager }}
 
       - name: Test
         run: |

diff --git a/embed/ansible/kubitect/finalize.yaml b/embed/ansible/kubitect/finalize.yaml
@@ -10,17 +10,6 @@
         dest: "{{ config_dir }}/admin.conf"
         flat: true
 
-- name: Fetch kubeconfig from first master node
-  hosts: server[0]
-  gather_facts: false
-  any_errors_fatal: true
-  tasks:
-    - name: Fetch Kubeconfig
-      fetch:
-        src: "/home/{{ ansible_user }}/.kube/config"
-        dest: "{{ config_dir }}/admin.conf"
-        flat: true
-
 - name: Finalize cluster deployment
   hosts: localhost
   gather_facts: false

diff --git a/embed/ansible/kubitect/requirements.txt b/embed/ansible/kubitect/requirements.txt
@@ -1,3 +1,3 @@
-ansible-core==2.16.2
+ansible==9.3.0
 jinja2==3.1.2
 netaddr==0.9.0
diff --git a/pkg/cluster/managers/k3s_playbooks.go b/pkg/cluster/managers/k3s_playbooks.go
@@ -22,7 +22,8 @@ func (e *k3s) K3sCreate(inventory string) error {
 		"api_endpoint":      string(e.InfraConfig.Nodes.LoadBalancer.VIP),
 		"api_port":          "6443",
 		"user_kubectl":      "true", // Set to false to kubectl via root user.
-		"cluster_context":   e.ClusterName,
+		"cluster_context":   "default",
+		"kubeconfig":        filepath.Join(e.ConfigDir, "admin.conf"),
 		"extra_server_args": "",
 		"extra_agent_args":  "",
 	}

diff --git a/scripts/deploy-cluster.sh b/scripts/deploy-cluster.sh
@@ -2,15 +2,16 @@
 set -eu
 
 # Check input arguments.
-if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ]; then
-    echo "Usage: ${0} <cluster_name> <distro> <network_plugin> <k8s_version>"
+if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ] || [ "${5:-}" = "" ]; then
+    echo "Usage: ${0} <cluster_name> <distro> <network_plugin> <k8s_version> <k8s_manager>"
     exit 1
 fi
 
 CLUSTER="${1}"
 DISTRO="${2}"
 NETWORK_PLUGIN="${3}"
 K8S_VERSION="${4}"
+K8S_MANAGER="${5}"
 
 echo "==> DEPLOY: Cluster ${DISTRO}/${NETWORK_PLUGIN}/${K8S_VERSION}"
 
@@ -63,6 +64,7 @@ cluster:
           ip: 192.168.113.21
 
 kubernetes:
+  manager: ${K8S_MANAGER}
   version: ${K8S_VERSION}
   networkPlugin: ${NETWORK_PLUGIN}
 EOF

diff --git a/scripts/deploy-node.sh b/scripts/deploy-node.sh
@@ -2,15 +2,16 @@
 set -eu
 
 # Check input arguments.
-if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ]; then
-    echo "Usage: ${0} <cluster_name> <distro> <network_plugin> <k8s_version>"
+if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ] || [ "${5:-}" = "" ]; then
+    echo "Usage: ${0} <cluster_name> <distro> <network_plugin> <k8s_version> <k8s_manager>"
     exit 1
 fi
 
 CLUSTER="${1}"
 DISTRO="${2}"
 NETWORK_PLUGIN="${3}"
 K8S_VERSION="${4}"
+K8S_MANAGER="${5}"
 
 echo "==> DEPLOY: Cluster (Single Node) ${DISTRO}/${NETWORK_PLUGIN}/${K8S_VERSION}"
 
@@ -45,6 +46,7 @@ cluster:
           ip: 192.168.113.10
 
 kubernetes:
+  manager: ${K8S_MANAGER}
   version: ${K8S_VERSION}
   networkPlugin: ${NETWORK_PLUGIN}
 EOF

diff --git a/scripts/test-cluster.sh b/scripts/test-cluster.sh
@@ -1,8 +1,6 @@
 #!/bin/sh
 set -eu
 
-TIMEOUT=600 # seconds
-
 defer() {
     if [ "${FAIL}" = "1" ]; then
         echo "==> DEBUG: Cluster events"
@@ -19,71 +17,33 @@ defer() {
 FAIL=1
 trap defer EXIT HUP INT TERM
 
-echo "==> TEST: Cluster readiness"
-
-startTime=$(date +%s)
-nodes=$(kubectl get nodes | awk 'NR>1 {print $1}')
-
-for node in $nodes; do
-    while :; do
-        isReady=$(kubectl get node "${node}" \
-            -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
-        )
-
-        if [ "${isReady}" = "True" ]; then
-            echo "===> PASS: Node ${node} is ready."
-            break
-        fi
-
-        currentTime=$(date +%s)
-        elapsedTime=$((currentTime - timeStart))
-
-        if [ "${elapsedTime}" -gt "${TIMEOUT}" ]; then
-            echo "FAIL: Node ${node} is NOT READY after ${TIMEOUT} seconds!"
-            kubectl get nodes
-            break
-        fi
-
-        sleep 10
-    done
-done
-
-echo "==> TEST: Running pods"
-
-startTime=$(date +%s)
-
-while :; do
-    failedPods=$(kubectl get pods \
-        --all-namespaces \
-        --field-selector="status.phase!=Succeeded,status.phase!=Running" \
-        --output custom-columns="NAMESPACE:metadata.namespace,POD:metadata.name,STATUS:status.phase"
-    )
+echo "==> TEST: All nodes ready"
+kubectl wait --for=condition=ready node --all --timeout=120s
 
-    if [ "$(echo "${failedPods}" | awk 'NR>1')" = "" ]; then
-        echo "===> PASS: All pods are running."
-        break
-    fi
-
-    currentTime=$(date +%s)
-    elapsedTime=$((currentTime - startTime))
-
-    if [ "${elapsedTime}" -gt "${TIMEOUT}" ]; then
-        echo "==> FAIL: Pods not running after ${TIMEOUT} seconds!"
-        echo "${failedPods}"
+echo "==> TEST: All pods ready"
+set +e
+# Wait for all pods to be ready. Retry a few times, as it may happen that
+# cluster has no pods when check is ran, which will result in an error.
+for i in $(seq 5); do
+    podsReadiness=$(kubectl wait --for=condition=ready pods --all -A --timeout=30s)
+    if [ "$?" -eq 0 ]; then
         break
+    else
+        echo "(attempt $i/5) Pods are still not ready. Retrying in 10 seconds..."
+        sleep 10
     fi
-
-    sleep 10
 done
+set -e
+echo "${podsReadiness}"
 
 echo "==> TEST: DNS"
-kubectl run dns-test --image=busybox:1.28.4 --restart=Never -- sleep 180
-kubectl wait --for=condition=Ready pod/dns-test --timeout=60s
+kubectl apply -f https://k8s.io/examples/admin/dns/dnsutils.yaml
+kubectl wait --for=condition=Ready pod/dnsutils --timeout=60s
 
-kubectl exec dns-test -- nslookup kubernetes.default
+kubectl exec dnsutils -- nslookup kubernetes.default
 echo "===> PASS: Local lookup (kubernetes.default)."
 
-kubectl exec dns-test -- nslookup kubitect.io
+kubectl exec dnsutils -- nslookup kubitect.io
 echo "===> PASS: External lookup (kubitect.io)."
 
 # All tests have passed.