Kind E2E CI

.github/workflows/ci.yml

@@ -4,29 +4,27 @@ on:
   schedule:
     - cron: "00 19 * * *" # run ci periodically at 3 am
   pull_request:
-    branches: [main]
+    branches: [ main ]
 
-env:
-  GO_VERSION: '1.21'
 
 jobs:
   format:
     name: Format
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-    - name: Setup Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: ${{env.GO_VERSION}}
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
 
-    - name: Format code
-      uses: iamnotaturtle/[email protected]
-      with:
-        # This part is also where you can pass other options, for example:
-        only_changed: True
+      - name: Format code
+        uses: iamnotaturtle/[email protected]
+        with:
+          # This part is also where you can pass other options, for example:
+          only_changed: True
 
   lint:
     name: Lint Check
@@ -38,13 +36,28 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: ${{env.GO_VERSION}}
+          go-version-file: go.mod
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: v1.54.2
 
+  test:
+    name: Xline Operator Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version-file: go.mod
+
+      - name: Build and test
+        run: make test
+
   commit:
     name: Commit Message Validation
     runs-on: ubuntu-latest
@@ -66,8 +79,8 @@ jobs:
     name: Spell Check
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout Actions Repository
-      uses: actions/checkout@v2
+      - name: Checkout Actions Repository
+        uses: actions/checkout@v2
 
-    - name: Check Spelling
-      uses: crate-ci/typos@master
+      - name: Check Spelling
+        uses: crate-ci/typos@master

.github/workflows/e2e.yml

@@ -0,0 +1,38 @@
+name: E2E
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch: { }
+
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  KIND_VERSION: v0.20.0
+
+jobs:
+  validation:
+    name: 'Validation'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+      - name: 'Install Kind'
+        run: |
+          wget https://github.com/kubernetes-sigs/kind/releases/download/${{ env.KIND_VERSION }}/kind-linux-amd64
+          chmod +x kind-linux-amd64
+          mv kind-linux-amd64 /usr/local/bin/kind
+      - name: 'E2E CI'
+        env:
+          KIND_CLUSTER_IMAGE: kindest/node:v1.27.3
+        run: bash ./tests/e2e/e2e.sh -p ci
+      - name: clean
+        if: failure()
+        run: bash ./tests/e2e/e2e.sh -c

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ repos:
       - id: check-merge-conflict
       - id: check-symlinks
       - id: check-yaml
+        args: [ --allow-multiple-documents ]
       - id: end-of-file-fixer
       - id: mixed-line-ending
       - id: trailing-whitespace

internal/controller/suite_test.go

@@ -33,7 +33,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	xlinekvstoredatenlordcomv1alpha1 "github.com/xline-kv/xline-operator/api/v1alpha1"
+	xapi "github.com/xline-kv/xline-operator/api/v1alpha1"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -79,7 +79,7 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 
-	err = xlinekvstoredatenlordcomv1alpha1.AddToScheme(scheme.Scheme)
+	err = xapi.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
 	//+kubebuilder:scaffold:scheme

internal/transformer/xlinecluster_resource_test.go

@@ -11,14 +11,14 @@ import (
 )
 
 func TestXlineClusterFunc(t *testing.T) {
-	test_image := "xline-img:latest"
+	testImage := "xline-img:latest"
 	xlineCluster := xapi.XlineCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "xline",
 			Namespace: "default",
 		},
 		Spec: xapi.XlineClusterSpec{
-			Image:    &test_image,
+			Image:    &testImage,
 			Replicas: 3,
 		},
 	}
@@ -38,8 +38,8 @@ func TestXlineClusterFunc(t *testing.T) {
 	})
 
 	t.Run("GetXlineImage should work properly", func(t *testing.T) {
-		xline_image := *xlineCluster.Spec.Image
-		assert.Equal(t, xline_image, "xline-img:latest")
+		xlineImage := *xlineCluster.Spec.Image
+		assert.Equal(t, xlineImage, "xline-img:latest")
 	})
 
 	t.Run("GetMemberTopology should work properly", func(t *testing.T) {

tests/e2e/cases/cases.sh

@@ -0,0 +1,3 @@
+${__E2E_CASES__:=false} && return 0 || __E2E_CASES__=true
+
+source "$(dirname "${BASH_SOURCE[0]}")/ci.sh"

tests/e2e/cases/ci.sh

@@ -0,0 +1,126 @@
+${__E2E_CASES_CI__:=false} && return 0 || __E2E_CASES_CI__=true
+
+source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh"
+
+_TEST_CI_CLUSTER_NAME="my-xline-cluster"
+_TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts"
+_TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc"
+_TEST_CI_NAMESPACE="default"
+_TEST_CI_DNS_SUFFIX="svc.cluster.local"
+_TEST_CI_XLINE_PORT="2379"
+_TEST_CI_LOG_SYNC_TIMEOUT=60
+
+function test::ci::_mk_endpoints() {
+  local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}"
+  for ((i = 1; i < $1; i++)); do
+    endpoints="${endpoints},${_TEST_CI_STS_NAME}-${i}.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}"
+  done
+  echo "$endpoints"
+}
+
+function test::ci::_etcdctl_expect() {
+  log::debug "run command: etcdctl --endpoints=$1 $2"
+  got=$(testenv::util::etcdctl --endpoints="$1" "$2")
+  expect=$(echo -e "$3")
+  if [ "${got//$'\r'/}" == "$expect" ]; then
+    log::info "command run success"
+  else
+    log::error "command run failed"
+    log::error "expect: $expect"
+    log::error "got: $got"
+    return 1
+  fi
+}
+
+function test::ci::_install_CRD() {
+    make install
+    if [ $? -eq 0 ]; then
+        log::info "make install: create custom resource definition succeeded"
+    else
+        log::error "make install: create custom resource definition failed"
+    fi
+}
+
+function test::ci::_uninstall_CRD() {
+    make uninstall
+    if [ $? -eq 0 ]; then
+        log::info "make uninstall: remove custom resource definition succeeded"
+    else
+        log::error "make uninstall: remove custom resource definition failed"
+    fi
+}
+
+function test::ci::wait_all_xline_pod_ready() {
+  for ((i = 0; i < $1; i++)); do
+    log::info "wait pod/${_TEST_CI_STS_NAME}-${i} to be ready"
+    if ! k8s::kubectl wait --for=condition=Ready pod/${_TEST_CI_STS_NAME}-${i} --timeout=300s; then
+      log::fatal "Failed to wait for util to be ready"
+    fi
+  done
+}
+
+function test::ci::_start() {
+  log::info "starting controller"
+  pushd $(dirname "${BASH_SOURCE[0]}")/../../../
+  test::ci::_install_CRD
+  make run  >/dev/null 2>&1 &
+  log::info "controller started"
+  popd
+  log::info "starting xline cluster"
+  k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yml" >/dev/null 2>&1
+  k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
+}
+
+function test::ci::_teardown() {
+  log::info "stopping controller"
+  pushd $(dirname "${BASH_SOURCE[0]}")/../../../
+  test::ci::_uninstall_CRD
+  controller_pid=$(ps aux | grep "[g]o run ./cmd/main.go" | awk '{print $2}')
+  if [ -n "$controller_pid" ]; then
+    kill -9 $controller_pid
+  fi
+}
+
+function test::ci::_chaos() {
+  size=$1
+  iters=$2
+  max_kill=$((size / 2))
+  log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill"
+  for ((i = 0; i < iters; i++)); do
+    log::info "chaos: iter=$i"
+    endpoints=$(test::ci::_mk_endpoints size)
+    test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $?
+    test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $?
+    kill=$((RANDOM % max_kill))
+    log::info "chaos: kill=$kill"
+    for ((j = 0; j < kill; j++)); do
+      pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))"
+      log::info "chaos: kill pod=$pod"
+      k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null
+    done
+    test::ci::_etcdctl_expect "$endpoints" "put B $i" "OK" || return $?
+    test::ci::_etcdctl_expect "$endpoints" "get B" "B\n$i" || return $?
+    k8s::kubectl wait --for=jsonpath='{.status.readyReplicas}'="$size" sts/$_TEST_CI_CLUSTER_NAME --timeout=300s >/dev/null 2>&1
+    log::info "wait for log synchronization" && sleep $_TEST_CI_LOG_SYNC_TIMEOUT
+  done
+}
+
+function test::run::ci::basic_validation() {
+  test::ci::_start
+  test::ci::wait_all_xline_pod_ready 3
+  endpoints=$(test::ci::_mk_endpoints 3)
+  test::ci::_etcdctl_expect "$endpoints" "put A 1" "OK" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "get A" "A\n1" || return $?
+  endpoints=$(test::ci::_mk_endpoints 1)
+  test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $?
+  test::ci::_teardown
+}
+
+
+function test::run::ci::basic_chaos() {
+  test::ci::_start
+  test::ci::_chaos 3 5 || return $?
+  test::ci::_teardown
+}

tests/e2e/cases/manifests/cluster.yml

@@ -0,0 +1,8 @@
+apiVersion: xline.io.datenlord.com/v1alpha1
+kind: XlineCluster
+metadata:
+  name: my-xline-cluster
+spec:
+  image: phoenix500526/xline:v0.6.1
+  imagePullPolicy: IfNotPresent
+  replicas: 3

tests/e2e/common/common.sh

@@ -0,0 +1,4 @@
+${__E2E_COMMON__:=false} && return 0 || __E2E_COMMON__=true
+
+source "$(dirname "${BASH_SOURCE[0]}")/log.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/k8s.sh"