Test cluster deployments (#184)

* github/actions: Add runner setup action Runner setup action prepares GH runner for deployment of Kubitect clusters and also builds the Kubitect binary from source. Signed-off-by: Din Music <[email protected]> * scripts: Add scripts for deploying and testing clusters Signed-off-by: Din Music <[email protected]> * github/workflows: Add integration test workflow Signed-off-by: Din Music <[email protected]> * pkg/cluster/executors: Remove temporary Kubespray patch Signed-off-by: Din Music <[email protected]> --------- Signed-off-by: Din Music <[email protected]>
MusicDin · Apr 6, 2024 · eba0414 · eba0414
1 parent fc6a944
commit eba0414
Show file tree

Hide file tree

Showing 7 changed files with 416 additions and 9 deletions.
diff --git a/.github/actions/runner-setup/action.yml b/.github/actions/runner-setup/action.yml
@@ -0,0 +1,40 @@
+name: Setup Environment
+description: Composite action that sets up the environment for deploying Kubitect clusters.
+
+runs:
+  using: composite
+  steps:
+    - name: Install dependencies
+      shell: bash
+      run: |
+        sudo apt update
+        sudo apt-get install -y --no-install-recommends \
+          curl \
+          libvirt-clients \
+          libvirt-daemon-system \
+          mkisofs \
+          virtualenv \
+          python3-pip \
+          qemu-utils \
+          qemu-system \
+          spice-client-gtk
+
+    - name: Configure libvirt
+      shell: bash
+      run: |
+        sudo usermod -aG libvirt $(whoami)
+        sudo chmod o+rw /var/run/libvirt/libvirt-sock
+        echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf > /dev/null
+        sudo systemctl restart libvirtd
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Build Kubitect
+      shell: bash
+      run: |
+        go build -ldflags "-s -w" -trimpath .
+        sudo mv kubitect /usr/local/bin/kubitect
+        kubitect --version
diff --git a/.github/workflows/tests-deploy-cluster.yml b/.github/workflows/tests-deploy-cluster.yml
@@ -0,0 +1,46 @@
+name: Test Deployment (Cluster)
+run-name: "${{ github.ref_name }}: Test Deployment (Cluster)"
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-cluster:
+    name: Cluster
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k8sVersion:
+          - v1.26.13
+          - v1.27.10
+          - v1.28.6
+        distro:
+          - ubuntu22
+        networkPlugin:
+          - calico
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: ./.github/actions/runner-setup
+
+      - name: Deploy cluster
+        run: |
+          ./scripts/deploy-cluster.sh k8s \
+            ${{ matrix.distro }} \
+            ${{ matrix.networkPlugin }} \
+            ${{ matrix.k8sVersion }}
+
+      - name: Test
+        run: |
+          ./scripts/test-cluster.sh
diff --git a/.github/workflows/tests-deploy-node.yml b/.github/workflows/tests-deploy-node.yml
@@ -0,0 +1,94 @@
+name: Test Deployment (Single Node)
+run-name: "${{ github.ref_name }}: Test Deployment (Single Node)"
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+  schedule:
+    # Run every Saturday at midnight.
+    - cron: '0 0 * * 6'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Test multiple k8s versions using the default distro and network plugin.
+  test-single-node-quick:
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    name: Node
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k8sVersion:
+          - v1.26.13
+          - v1.27.10
+          - v1.28.6
+        distro:
+          - ubuntu22
+        networkPlugin:
+          - calico
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: ./.github/actions/runner-setup
+
+      - name: Deploy single node
+        run: |
+          ./scripts/deploy-node.sh k8s \
+            ${{ matrix.distro }} \
+            ${{ matrix.networkPlugin }} \
+            ${{ matrix.k8sVersion }}
+
+      - name: Test
+        run: |
+          ./scripts/test-cluster.sh
+
+  # Test most combinations of Kubernetes versions, distros,
+  # and network plugins. Run this only on push.
+  test-single-node-all:
+    if: github.event_name != 'push' && github.event_name != 'pull_request'
+    name: Node
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k8sVersion:
+          - v1.26.13
+          - v1.27.10
+          - v1.28.6
+        distro:
+          - ubuntu22
+          - debian12
+          - centos9
+          - rocky9
+        networkPlugin:
+          - calico
+          - cilium
+          - flannel
+          - kube-router
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: ./.github/actions/runner-setup
+
+      - name: Deploy single node
+        run: |
+          ./scripts/deploy-node.sh k8s \
+            ${{ matrix.distro }} \
+            ${{ matrix.networkPlugin }} \
+            ${{ matrix.k8sVersion }}
+
+      - name: Test
+        run: |
+          ./scripts/test-cluster.sh
diff --git a/pkg/cluster/executors/kubespray/kubespray.go b/pkg/cluster/executors/kubespray/kubespray.go
@@ -14,7 +14,6 @@ import (
 	"github.com/MusicDin/kubitect/pkg/tools/git"
 	"github.com/MusicDin/kubitect/pkg/tools/virtualenv"
 	"github.com/MusicDin/kubitect/pkg/ui"
-	"github.com/MusicDin/kubitect/pkg/utils/file"
 	"gopkg.in/yaml.v3"
 )
 
@@ -83,14 +82,6 @@ func (e *kubespray) Init() error {
 		return err
 	}
 
-	// Patch: There is an issue with unsafe conditionals in
-	// Kubespray with ansible-core version > 2.14.11.
-	reqPath := path.Join(dst, "requirements.txt")
-	reqPatch := []byte("ansible-core==2.14.11")
-	if err := file.Append(reqPath, reqPatch); err != nil {
-		return err
-	}
-
 	if err := e.VirtualEnv.Init(); err != nil {
 		return fmt.Errorf("kubespray exec: initialize virtual environment: %v", err)
 	}

diff --git a/scripts/deploy-cluster.sh b/scripts/deploy-cluster.sh
@@ -0,0 +1,82 @@
+#!/bin/sh
+set -eu
+
+# Check input arguments.
+if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ]; then
+    echo "Usage: ${0} <cluster_name> <distro> <network_plugin> <k8s_version>"
+    exit 1
+fi
+
+CLUSTER="${1}"
+DISTRO="${2}"
+NETWORK_PLUGIN="${3}"
+K8S_VERSION="${4}"
+
+echo "==> DEPLOY: Cluster ${DISTRO}/${NETWORK_PLUGIN}/${K8S_VERSION}"
+
+# Create config.
+cat <<-EOF > config.yaml
+hosts:
+  - name: localhost
+    connection:
+      type: local
+
+cluster:
+  name: ${CLUSTER}
+  network:
+    mode: nat
+    cidr: 192.168.113.0/24
+  nodeTemplate:
+    user: k8s
+    updateOnBoot: true
+    cpuMode: host-passthrough
+    ssh:
+      addToKnownHosts: true
+    os:
+      distro: ${DISTRO}
+  nodes:
+    loadBalancer:
+      default:
+        cpu: 1
+        ram: 1
+        mainDiskSize: 8
+      instances:
+        - id: 1
+          ip: 192.168.113.100
+    master:
+      default:
+        cpu: 2
+        ram: 2
+        mainDiskSize: 16
+      instances:
+        - id: 1
+          ip: 192.168.113.10
+    worker:
+      default:
+        cpu: 2
+        ram: 2
+        mainDiskSize: 16
+      instances:
+        - id: 1
+          ip: 192.168.113.20
+        - id: 2
+          ip: 192.168.113.21
+
+kubernetes:
+  version: ${K8S_VERSION}
+  networkPlugin: ${NETWORK_PLUGIN}
+EOF
+
+echo "Config:"
+echo "---"
+cat config.yaml
+echo "---"
+
+# Apply config and export kubeconfig.
+mkdir -p "${HOME}/.kube"
+kubitect apply --config config.yaml
+kubitect export kubeconfig --cluster "${CLUSTER}" > "${HOME}/.kube/config"
+
+echo "==> DEBUG: Cluster info"
+kubectl cluster-info
+kubectl get nodes
diff --git a/scripts/deploy-node.sh b/scripts/deploy-node.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+set -eu
+
+# Check input arguments.
+if [ "${1:-}" = "" ] || [ "${2:-}" = "" ] || [ "${3:-}" = "" ] || [ "${4:-}" = "" ]; then
+    echo "Usage: ${0} <cluster_name> <distro> <network_plugin> <k8s_version>"
+    exit 1
+fi
+
+CLUSTER="${1}"
+DISTRO="${2}"
+NETWORK_PLUGIN="${3}"
+K8S_VERSION="${4}"
+
+echo "==> DEPLOY: Cluster (Single Node) ${DISTRO}/${NETWORK_PLUGIN}/${K8S_VERSION}"
+
+# Create config.
+cat <<-EOF > config.yaml
+hosts:
+  - name: localhost
+    connection:
+      type: local
+
+cluster:
+  name: ${CLUSTER}
+  network:
+    mode: nat
+    cidr: 192.168.113.0/24
+  nodeTemplate:
+    user: k8s
+    updateOnBoot: true
+    cpuMode: host-passthrough
+    ssh:
+      addToKnownHosts: true
+    os:
+      distro: ${DISTRO}
+  nodes:
+    master:
+      default:
+        cpu: 2
+        ram: 4
+        mainDiskSize: 32
+      instances:
+        - id: 1
+          ip: 192.168.113.10
+
+kubernetes:
+  version: ${K8S_VERSION}
+  networkPlugin: ${NETWORK_PLUGIN}
+EOF
+
+echo "Config:"
+echo "---"
+cat config.yaml
+echo "---"
+
+# Apply config and export kubeconfig.
+mkdir -p "${HOME}/.kube"
+kubitect apply --config config.yaml
+kubitect export kubeconfig --cluster "${CLUSTER}" > "${HOME}/.kube/config"
+
+echo "==> DEBUG: Cluster info"
+kubectl cluster-info
+kubectl get nodes