From eba0414db3dd52fe67a017db61adf04362457423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Din=20Mu=C5=A1i=C4=87?= Date: Sat, 6 Apr 2024 20:41:41 +0200 Subject: [PATCH] Test cluster deployments (#184) * github/actions: Add runner setup action Runner setup action prepares GH runner for deployment of Kubitect clusters and also builds the Kubitect binary from source. Signed-off-by: Din Music * scripts: Add scripts for deploying and testing clusters Signed-off-by: Din Music * github/workflows: Add integration test workflow Signed-off-by: Din Music * pkg/cluster/executors: Remove temporary Kubespray patch Signed-off-by: Din Music --------- Signed-off-by: Din Music --- .github/actions/runner-setup/action.yml | 40 +++++++++ .github/workflows/tests-deploy-cluster.yml | 46 ++++++++++ .github/workflows/tests-deploy-node.yml | 94 ++++++++++++++++++++ pkg/cluster/executors/kubespray/kubespray.go | 9 -- scripts/deploy-cluster.sh | 82 +++++++++++++++++ scripts/deploy-node.sh | 64 +++++++++++++ scripts/test-cluster.sh | 90 +++++++++++++++++++ 7 files changed, 416 insertions(+), 9 deletions(-) create mode 100644 .github/actions/runner-setup/action.yml create mode 100644 .github/workflows/tests-deploy-cluster.yml create mode 100644 .github/workflows/tests-deploy-node.yml create mode 100755 scripts/deploy-cluster.sh create mode 100755 scripts/deploy-node.sh create mode 100755 scripts/test-cluster.sh diff --git a/.github/actions/runner-setup/action.yml b/.github/actions/runner-setup/action.yml new file mode 100644 index 00000000..7c13fd86 --- /dev/null +++ b/.github/actions/runner-setup/action.yml @@ -0,0 +1,40 @@ +name: Setup Environment +description: Composite action that sets up the environment for deploying Kubitect clusters. + +runs: + using: composite + steps: + - name: Install dependencies + shell: bash + run: | + sudo apt update + sudo apt-get install -y --no-install-recommends \ + curl \ + libvirt-clients \ + libvirt-daemon-system \ + mkisofs \ + virtualenv \ + python3-pip \ + qemu-utils \ + qemu-system \ + spice-client-gtk + + - name: Configure libvirt + shell: bash + run: | + sudo usermod -aG libvirt $(whoami) + sudo chmod o+rw /var/run/libvirt/libvirt-sock + echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf > /dev/null + sudo systemctl restart libvirtd + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build Kubitect + shell: bash + run: | + go build -ldflags "-s -w" -trimpath . + sudo mv kubitect /usr/local/bin/kubitect + kubitect --version diff --git a/.github/workflows/tests-deploy-cluster.yml b/.github/workflows/tests-deploy-cluster.yml new file mode 100644 index 00000000..3ab26a67 --- /dev/null +++ b/.github/workflows/tests-deploy-cluster.yml @@ -0,0 +1,46 @@ +name: Test Deployment (Cluster) +run-name: "${{ github.ref_name }}: Test Deployment (Cluster)" + +on: + pull_request: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-cluster: + name: Cluster + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k8sVersion: + - v1.26.13 + - v1.27.10 + - v1.28.6 + distro: + - ubuntu22 + networkPlugin: + - calico + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup environment + uses: ./.github/actions/runner-setup + + - name: Deploy cluster + run: | + ./scripts/deploy-cluster.sh k8s \ + ${{ matrix.distro }} \ + ${{ matrix.networkPlugin }} \ + ${{ matrix.k8sVersion }} + + - name: Test + run: | + ./scripts/test-cluster.sh diff --git a/.github/workflows/tests-deploy-node.yml b/.github/workflows/tests-deploy-node.yml new file mode 100644 index 00000000..7d7e7cf6 --- /dev/null +++ b/.github/workflows/tests-deploy-node.yml @@ -0,0 +1,94 @@ +name: Test Deployment (Single Node) +run-name: "${{ github.ref_name }}: Test Deployment (Single Node)" + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + schedule: + # Run every Saturday at midnight. + - cron: '0 0 * * 6' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Test multiple k8s versions using the default distro and network plugin. + test-single-node-quick: + if: github.event_name == 'push' || github.event_name == 'pull_request' + name: Node + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k8sVersion: + - v1.26.13 + - v1.27.10 + - v1.28.6 + distro: + - ubuntu22 + networkPlugin: + - calico + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup environment + uses: ./.github/actions/runner-setup + + - name: Deploy single node + run: | + ./scripts/deploy-node.sh k8s \ + ${{ matrix.distro }} \ + ${{ matrix.networkPlugin }} \ + ${{ matrix.k8sVersion }} + + - name: Test + run: | + ./scripts/test-cluster.sh + + # Test most combinations of Kubernetes versions, distros, + # and network plugins. Run this only on push. + test-single-node-all: + if: github.event_name != 'push' && github.event_name != 'pull_request' + name: Node + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k8sVersion: + - v1.26.13 + - v1.27.10 + - v1.28.6 + distro: + - ubuntu22 + - debian12 + - centos9 + - rocky9 + networkPlugin: + - calico + - cilium + - flannel + - kube-router + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup environment + uses: ./.github/actions/runner-setup + + - name: Deploy single node + run: | + ./scripts/deploy-node.sh k8s \ + ${{ matrix.distro }} \ + ${{ matrix.networkPlugin }} \ + ${{ matrix.k8sVersion }} + + - name: Test + run: | + ./scripts/test-cluster.sh diff --git a/pkg/cluster/executors/kubespray/kubespray.go b/pkg/cluster/executors/kubespray/kubespray.go index 3f83cb81..7e288224 100644 --- a/pkg/cluster/executors/kubespray/kubespray.go +++ b/pkg/cluster/executors/kubespray/kubespray.go @@ -14,7 +14,6 @@ import ( "github.com/MusicDin/kubitect/pkg/tools/git" "github.com/MusicDin/kubitect/pkg/tools/virtualenv" "github.com/MusicDin/kubitect/pkg/ui" - "github.com/MusicDin/kubitect/pkg/utils/file" "gopkg.in/yaml.v3" ) @@ -83,14 +82,6 @@ func (e *kubespray) Init() error { return err } - // Patch: There is an issue with unsafe conditionals in - // Kubespray with ansible-core version > 2.14.11. - reqPath := path.Join(dst, "requirements.txt") - reqPatch := []byte("ansible-core==2.14.11") - if err := file.Append(reqPath, reqPatch); err != nil { - return err - } - if err := e.VirtualEnv.Init(); err != nil { return fmt.Errorf("kubespray exec: initialize virtual environment: %v", err) } diff --git a/scripts/deploy-cluster.sh b/scripts/deploy-cluster.sh new file mode 100755 index 00000000..59abbd5c --- /dev/null +++ b/scripts/deploy-cluster.sh @@ -0,0 +1,82 @@ +#!/bin/sh +set -eu + +# Check input arguments. +if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ]; then + echo "Usage: ${0} " + exit 1 +fi + +CLUSTER="${1}" +DISTRO="${2}" +NETWORK_PLUGIN="${3}" +K8S_VERSION="${4}" + +echo "==> DEPLOY: Cluster ${DISTRO}/${NETWORK_PLUGIN}/${K8S_VERSION}" + +# Create config. +cat <<-EOF > config.yaml +hosts: + - name: localhost + connection: + type: local + +cluster: + name: ${CLUSTER} + network: + mode: nat + cidr: 192.168.113.0/24 + nodeTemplate: + user: k8s + updateOnBoot: true + cpuMode: host-passthrough + ssh: + addToKnownHosts: true + os: + distro: ${DISTRO} + nodes: + loadBalancer: + default: + cpu: 1 + ram: 1 + mainDiskSize: 8 + instances: + - id: 1 + ip: 192.168.113.100 + master: + default: + cpu: 2 + ram: 2 + mainDiskSize: 16 + instances: + - id: 1 + ip: 192.168.113.10 + worker: + default: + cpu: 2 + ram: 2 + mainDiskSize: 16 + instances: + - id: 1 + ip: 192.168.113.20 + - id: 2 + ip: 192.168.113.21 + +kubernetes: + version: ${K8S_VERSION} + networkPlugin: ${NETWORK_PLUGIN} +EOF + +echo "Config:" +echo "---" +cat config.yaml +echo "---" + +# Apply config and export kubeconfig. +mkdir -p "${HOME}/.kube" +kubitect apply --config config.yaml +kubitect export kubeconfig --cluster "${CLUSTER}" > "${HOME}/.kube/config" + +echo "==> DEBUG: Cluster info" +kubectl cluster-info +kubectl get nodes diff --git a/scripts/deploy-node.sh b/scripts/deploy-node.sh new file mode 100755 index 00000000..0d6bd8b1 --- /dev/null +++ b/scripts/deploy-node.sh @@ -0,0 +1,64 @@ +#!/bin/sh +set -eu + +# Check input arguments. +if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ]; then + echo "Usage: ${0} " + exit 1 +fi + +CLUSTER="${1}" +DISTRO="${2}" +NETWORK_PLUGIN="${3}" +K8S_VERSION="${4}" + +echo "==> DEPLOY: Cluster (Single Node) ${DISTRO}/${NETWORK_PLUGIN}/${K8S_VERSION}" + +# Create config. +cat <<-EOF > config.yaml +hosts: + - name: localhost + connection: + type: local + +cluster: + name: ${CLUSTER} + network: + mode: nat + cidr: 192.168.113.0/24 + nodeTemplate: + user: k8s + updateOnBoot: true + cpuMode: host-passthrough + ssh: + addToKnownHosts: true + os: + distro: ${DISTRO} + nodes: + master: + default: + cpu: 2 + ram: 4 + mainDiskSize: 32 + instances: + - id: 1 + ip: 192.168.113.10 + +kubernetes: + version: ${K8S_VERSION} + networkPlugin: ${NETWORK_PLUGIN} +EOF + +echo "Config:" +echo "---" +cat config.yaml +echo "---" + +# Apply config and export kubeconfig. +mkdir -p "${HOME}/.kube" +kubitect apply --config config.yaml +kubitect export kubeconfig --cluster "${CLUSTER}" > "${HOME}/.kube/config" + +echo "==> DEBUG: Cluster info" +kubectl cluster-info +kubectl get nodes diff --git a/scripts/test-cluster.sh b/scripts/test-cluster.sh new file mode 100755 index 00000000..e2981c1e --- /dev/null +++ b/scripts/test-cluster.sh @@ -0,0 +1,90 @@ +#!/bin/sh +set -eu + +TIMEOUT=600 # seconds + +defer() { + if [ "${FAIL}" = "1" ]; then + echo "==> DEBUG: Cluster events" + kubectl get events --all-namespaces + + echo "==> FAIL" + exit 1 + fi + + echo "==> PASS" + exit 0 +} + +FAIL=1 +trap defer EXIT HUP INT TERM + +echo "==> TEST: Cluster readiness" + +startTime=$(date +%s) +nodes=$(kubectl get nodes | awk 'NR>1 {print $1}') + +for node in $nodes; do + while :; do + isReady=$(kubectl get node "${node}" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + ) + + if [ "${isReady}" = "True" ]; then + echo "===> PASS: Node ${node} is ready." + break + fi + + currentTime=$(date +%s) + elapsedTime=$((currentTime - timeStart)) + + if [ "${elapsedTime}" -gt "${TIMEOUT}" ]; then + echo "FAIL: Node ${node} is NOT READY after ${TIMEOUT} seconds!" + kubectl get nodes + break + fi + + sleep 10 + done +done + +echo "==> TEST: Running pods" + +startTime=$(date +%s) + +while :; do + failedPods=$(kubectl get pods \ + --all-namespaces \ + --field-selector="status.phase!=Succeeded,status.phase!=Running" \ + --output custom-columns="NAMESPACE:metadata.namespace,POD:metadata.name,STATUS:status.phase" + ) + + if [ "$(echo "${failedPods}" | awk 'NR>1')" = "" ]; then + echo "===> PASS: All pods are running." + break + fi + + currentTime=$(date +%s) + elapsedTime=$((currentTime - startTime)) + + if [ "${elapsedTime}" -gt "${TIMEOUT}" ]; then + echo "==> FAIL: Pods not running after ${TIMEOUT} seconds!" + echo "${failedPods}" + break + fi + + sleep 10 +done + +echo "==> TEST: DNS" +kubectl run dns-test --image=busybox:1.28.4 --restart=Never -- sleep 180 +kubectl wait --for=condition=Ready pod/dns-test --timeout=60s + +kubectl exec dns-test -- nslookup kubernetes.default +echo "===> PASS: Local lookup (kubernetes.default)." + +kubectl exec dns-test -- nslookup kubitect.io +echo "===> PASS: External lookup (kubitect.io)." + +# All tests have passed. +FAIL=0