ci: Add github action to run e2e test with kubernetes cluster.

Signed-off-by: Lan Liang <[email protected]>
xline-kv · Mar 17, 2024 · 24babde · 24babde
1 parent 75e51f4
commit 24babde
Show file tree

Hide file tree

Showing 5 changed files with 152 additions and 0 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -109,3 +109,94 @@ jobs:
       - uses: actions/checkout@v4
       - name: Check Spelling
         uses: crate-ci/typos@master
+
+  build:
+    name: build 
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Configure sccache
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Build xline image
+        run: |
+          docker run -q --rm -v $(pwd):/xline \
+            -e SCCACHE_GHA_ENABLED=on \
+            -e ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL} \
+            -e ACTIONS_RUNTIME_TOKEN=${ACTIONS_RUNTIME_TOKEN} \
+            ghcr.io/xline-kv/build-env:latest \
+            cargo build --release --bin xline --bin benchmark --bin validation_lock_client
+
+          sudo apt-get install -y --force-yes expect
+          cd scripts
+          cp ../target/release/{xline,benchmark,validation_lock_client} .
+          ldd ./xline
+          ldd ./benchmark
+          cp ../fixtures/{private,public}.pem .
+          docker build . -t ghcr.io/xline-kv/xline:master
+
+          cd .. 
+          pwd
+          mkdir -p _output
+          docker save -o _output/xline.tar ghcr.io/xline-kv/xline:master
+          ls _output
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: xline
+          path: _output/xline.tar
+
+  e2e:
+    name: e2e_kube
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s:
+          - v1.27.3
+          - v1.28.7
+          - v1.29.2
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: xline
+
+      - name: build kind node image with xline 
+        run: |
+          docker build . -f ci/artifacts/kind-node-xline.Dockerfile -t ghcr.io/xline-kv/kind-node-xline:${{ matrix.k8s }}-master --build-arg K8S_VERSION=${{ matrix.k8s }}
+
+      - name: Create kubernetes cluster with kind
+        env:
+          K8S_VERSION: ${{ matrix.k8s }}
+        run: |
+          ./ci/scripts/kind.sh
+
+      - name: Get cluster status
+        run: |
+          kubectl run test --image httpd:2
+          kubectl wait --timeout=1m --for=condition=ready pods test
+          kubectl get pods -A -owide
+
+      - name: Export logs
+        if: failure()
+        run: |
+          mkdir -p _artifacts
+          docker ps
+          kind export logs  --loglevel=debug ./_artifacts/logs
+          ls _artifacts
+
+      - name: Upload logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: xline-kind-log-${{ matrix.k8s }}-${{ github.run_id }}
+          path: ./_artifacts/logs
diff --git a/ci/artifacts/kind-node-xline.Dockerfile b/ci/artifacts/kind-node-xline.Dockerfile
@@ -0,0 +1,8 @@
+ARG K8S_VERSION
+
+FROM kindest/node:${K8S_VERSION}
+
+RUN mkdir /tmp/kind
+COPY xline.tar /tmp/kind/
+RUN ( containerd -l warning & ) && ctr -n k8s.io images import --no-unpack /tmp/kind/*.tar
+RUN rm /tmp/kind/*.tar
diff --git a/ci/artifacts/kind.yaml b/ci/artifacts/kind.yaml
@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  image: ghcr.io/xline-kv/kind-node-xline:${K8S_VERSION}-master
+  extraMounts:
+    - hostPath: ${WORKSPACE}/ci/artifacts/xline-pod.yaml
+      containerPath: /etc/kubernetes/manifests/xline.yaml
+  kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      etcd:
+        external:
+          endpoints:
+          - http://kind-control-plane:2379
+- role: worker
+  image: ghcr.io/xline-kv/kind-node-xline:${K8S_VERSION}-master
+- role: worker
+  image: ghcr.io/xline-kv/kind-node-xline:${K8S_VERSION}-master
diff --git a/ci/artifacts/xline-pod.yaml b/ci/artifacts/xline-pod.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: xline
+  namespace: kube-system
+spec:
+  containers:
+    - name: xline
+      image: ghcr.io/xline-kv/xline:master
+      imagePullPolicy: IfNotPresent
+      command:
+        - /bin/sh
+        - -c
+        - xline --name node1 --members node1=kind-control-plane:2379 --data-dir /tmp/xline --storage-engine rocksdb --client-listen-urls=http://kind-control-plane:2379 --peer-listen-urls=http://kind-control-plane:2380,http://kind-control-plane:2381 --client-advertise-urls=http://kind-control-plane:2379 --peer-advertise-urls=http://kind-control-plane:2380,http://kind-control-plane:2381
+  hostNetwork: true
+status: {}
diff --git a/ci/scripts/kind.sh b/ci/scripts/kind.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+K8S_VERSION=${K8S_VERSION:-"v1.27.3"}
+KIND_VERSION=${KIND_VERSION:-"0.22.0"}
+
+wget -q https://github.com/kubernetes-sigs/kind/releases/download/v$KIND_VERSION/kind-linux-amd64
+chmod +x kind-linux-amd64 && mv kind-linux-amd64 /usr/local/bin/kind
+
+# print the config file
+WORKSPACE=$PWD envsubst
+
+WORKSPACE=$PWD envsubst < ci/artifacts/kind.yaml | kind create cluster -v7 --retain --wait 4m --config -
+kubectl wait node --all --for condition=ready