From 17b133427f0b724c9a1b6796e1ecae17f1db8636 Mon Sep 17 00:00:00 2001 From: Phoeniix Zhao Date: Sun, 26 Nov 2023 23:56:59 +0800 Subject: [PATCH] ci: add e2e test scripts Signed-off-by: Phoeniix Zhao --- .github/scripts/install_deps.sh | 6 + .github/workflows/e2e.yml | 32 +++++ internal/controller/suite_test.go | 4 +- tests/e2e/cases/cases.sh | 3 + tests/e2e/cases/ci.sh | 126 ++++++++++++++++++ tests/e2e/cases/manifests/cluster.yml | 9 ++ tests/e2e/common/common.sh | 4 + tests/e2e/common/k8s.sh | 45 +++++++ tests/e2e/common/log.sh | 26 ++++ tests/e2e/e2e.sh | 97 ++++++++++++++ tests/e2e/testenv/minikube.sh | 47 +++++++ tests/e2e/testenv/testenv.sh | 13 ++ tests/e2e/testenv/util/manifests/etcdctl.yaml | 15 +++ tests/e2e/testenv/util/util.sh | 60 +++++++++ 14 files changed, 485 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/install_deps.sh create mode 100644 .github/workflows/e2e.yml create mode 100644 tests/e2e/cases/cases.sh create mode 100644 tests/e2e/cases/ci.sh create mode 100644 tests/e2e/cases/manifests/cluster.yml create mode 100644 tests/e2e/common/common.sh create mode 100644 tests/e2e/common/k8s.sh create mode 100644 tests/e2e/common/log.sh create mode 100755 tests/e2e/e2e.sh create mode 100644 tests/e2e/testenv/minikube.sh create mode 100644 tests/e2e/testenv/testenv.sh create mode 100644 tests/e2e/testenv/util/manifests/etcdctl.yaml create mode 100644 tests/e2e/testenv/util/util.sh diff --git a/.github/scripts/install_deps.sh b/.github/scripts/install_deps.sh new file mode 100644 index 00000000..890ea238 --- /dev/null +++ b/.github/scripts/install_deps.sh @@ -0,0 +1,6 @@ +#!/bin/bash +apt-get install -y make expect libssl-dev + +# install minikube +curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +install -m 755 minikube /usr/local/bin/minikube diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..ebcb5f9f --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,32 @@ +name: E2E + +on: + pull_request: + branches: + - main + workflow_dispatch: { } + +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency +concurrency: + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + +jobs: + validation: + name: 'Validation' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Dependencies + run: sudo bash ./.github/scripts/install_deps.sh + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: go/src/github.com/${{ github.repository }}/go.mod + go-version: 1.21.3 + - name: 'E2E CI' + run: bash ./tests/e2e/e2e.sh -p ci + - name: clean + if: failure() + run: bash ./tests/e2e/e2e.sh -c diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 3fd0cb28..80181fc6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -33,7 +33,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - xlinekvstoredatenlordcomv1alpha1 "github.com/xline-kv/xline-operator/api/v1alpha1" + xapi "github.com/xline-kv/xline-operator/api/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -79,7 +79,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = xlinekvstoredatenlordcomv1alpha1.AddToScheme(scheme.Scheme) + err = xapi.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme diff --git a/tests/e2e/cases/cases.sh b/tests/e2e/cases/cases.sh new file mode 100644 index 00000000..690d96de --- /dev/null +++ b/tests/e2e/cases/cases.sh @@ -0,0 +1,3 @@ +${__E2E_CASES__:=false} && return 0 || __E2E_CASES__=true + +source "$(dirname "${BASH_SOURCE[0]}")/ci.sh" diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh new file mode 100644 index 00000000..74a724f5 --- /dev/null +++ b/tests/e2e/cases/ci.sh @@ -0,0 +1,126 @@ +${__E2E_CASES_CI__:=false} && return 0 || __E2E_CASES_CI__=true + +source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh" + +_TEST_CI_CLUSTER_NAME="my-xline-cluster" +_TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts" +_TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc" +_TEST_CI_NAMESPACE="default" +_TEST_CI_DNS_SUFFIX="svc.cluster.local" +_TEST_CI_XLINE_PORT="2379" +_TEST_CI_LOG_SYNC_TIMEOUT=60 + +function test::ci::_mk_endpoints() { + local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" + for ((i = 1; i < $1; i++)); do + endpoints="${endpoints},${_TEST_CI_STS_NAME}-${i}.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" + done + echo "$endpoints" +} + +function test::ci::_etcdctl_expect() { + log::debug "run command: etcdctl --endpoints=$1 $2" + got=$(testenv::util::etcdctl --endpoints="$1" "$2") + expect=$(echo -e "$3") + if [ "${got//$'\r'/}" == "$expect" ]; then + log::info "command run success" + else + log::error "command run failed" + log::error "expect: $expect" + log::error "got: $got" + return 1 + fi +} + +function test::ci::_install_CRD() { + KUBECTL="minikube kubectl --" make install + if [ $? -eq 0 ]; then + log::info "make install: create custom resource definition succeeded" + else + log::error "make install: create custom resource definition failed" + fi +} + +function test::ci::_uninstall_CRD() { + KUBECTL="minikube kubectl --" make uninstall + if [ $? -eq 0 ]; then + log::info "make uninstall: remove custom resource definition succeeded" + else + log::error "make uninstall: remove custom resource definition failed" + fi +} + +function test::ci::wait_all_xline_pod_ready() { + for ((i = 0; i < $1; i++)); do + log::info "wait pod/${_TEST_CI_STS_NAME}-${i} to be ready" + if ! k8s::kubectl wait --for=condition=Ready pod/${_TEST_CI_STS_NAME}-${i} --timeout=300s; then + log::fatal "Failed to wait for util to be ready" + fi + done +} + +function test::ci::_start() { + log::info "starting controller" + pushd $(dirname "${BASH_SOURCE[0]}")/../../../ + test::ci::_install_CRD + KUBECTL="minikube kubectl --" make run >/dev/null 2>&1 & + log::info "controller started" + popd + log::info "starting xline cluster" + k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yml" >/dev/null 2>&1 + k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME +} + +function test::ci::_teardown() { + log::info "stopping controller" + pushd $(dirname "${BASH_SOURCE[0]}")/../../../ + test::ci::_uninstall_CRD + controller_pid=$(ps aux | grep "[g]o run ./cmd/main.go" | awk '{print $2}') + if [ -n "$controller_pid" ]; then + kill -9 $controller_pid + fi +} + +function test::ci::_chaos() { + size=$1 + iters=$2 + max_kill=$((size / 2)) + log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill" + for ((i = 0; i < iters; i++)); do + log::info "chaos: iter=$i" + endpoints=$(test::ci::_mk_endpoints size) + test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $? + kill=$((RANDOM % max_kill + 1)) + log::info "chaos: kill=$kill" + for ((j = 0; j < kill; j++)); do + pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))" + log::info "chaos: kill pod=$pod" + k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null + done + test::ci::_etcdctl_expect "$endpoints" "put B $i" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get B" "B\n$i" || return $? + k8s::kubectl wait --for=jsonpath='{.status.readyReplicas}'="$size" sts/$_TEST_CI_CLUSTER_NAME --timeout=300s >/dev/null 2>&1 + log::info "wait for log synchronization" && sleep $_TEST_CI_LOG_SYNC_TIMEOUT + done +} + +function test::run::ci::basic_validation() { + test::ci::_start + test::ci::wait_all_xline_pod_ready 3 + endpoints=$(test::ci::_mk_endpoints 3) + test::ci::_etcdctl_expect "$endpoints" "put A 1" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get A" "A\n1" || return $? + endpoints=$(test::ci::_mk_endpoints 1) + test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $? + test::ci::_teardown +} + + +function test::run::ci::basic_chaos() { + test::ci::_start + test::ci::_chaos 3 5 || return $? + test::ci::_teardown +} diff --git a/tests/e2e/cases/manifests/cluster.yml b/tests/e2e/cases/manifests/cluster.yml new file mode 100644 index 00000000..f7450c85 --- /dev/null +++ b/tests/e2e/cases/manifests/cluster.yml @@ -0,0 +1,9 @@ +apiVersion: xline.kvstore.datenlord.com/v1alpha1 +kind: XlineCluster +metadata: + name: my-xline-cluster +spec: + version: v0.6.0 + image: phoenix500526/xline + imagePullPolicy: IfNotPresent + replicas: 3 diff --git a/tests/e2e/common/common.sh b/tests/e2e/common/common.sh new file mode 100644 index 00000000..4f456e7e --- /dev/null +++ b/tests/e2e/common/common.sh @@ -0,0 +1,4 @@ +${__E2E_COMMON__:=false} && return 0 || __E2E_COMMON__=true + +source "$(dirname "${BASH_SOURCE[0]}")/log.sh" +source "$(dirname "${BASH_SOURCE[0]}")/k8s.sh" diff --git a/tests/e2e/common/k8s.sh b/tests/e2e/common/k8s.sh new file mode 100644 index 00000000..c72d44fb --- /dev/null +++ b/tests/e2e/common/k8s.sh @@ -0,0 +1,45 @@ +${__E2E_COMMON_K8S__:=false} && return 0 || __E2E_COMMON_K8S__=true + +# ENVIRONMENT VARIABLES: +# KUBECTL: path to kubectl binary +# KUBECTL_NAMESPACE: namespace to use for kubectl commands +# +# ARGUMENTS: +# $@: arguments to pass to kubectl +function k8s::kubectl() { + KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}" + local kubectl="${KUBECTL:-minikube kubectl --}" + ${kubectl} -n "${KUBECTL_NAMESPACE}" "$@" +} + +# ENVIRONMENT VARIABLES: +# KUBECTL: path to kubectl binary +# KUBECTL_NAMESPACE: namespace to use for kubectl commands +# +# ARGUMENTS: +# $1: resource type +# $2: resource name +function k8s::kubectl::resource_exist() { + KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}" + k8s::kubectl get "$1" "$2" >/dev/null 2>&1 +} + +# ENVIRONMENT VARIABLES: +# KUBECTL: path to kubectl binary +# KUBECTL_NAMESPACE: namespace to use for kubectl commands +# +# ARGUMENTS: +# $1: resource type +# $2: resource name +# $3: (optional) interval to check resource creation +function k8s::kubectl::wait_resource_creation() { + interval="${3:-5}" + KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}" + while true; do + if k8s::kubectl::resource_exist "$1" "$2"; then + break + fi + log::info "Waiting for $1/$2 ($KUBECTL_NAMESPACE) to be created" + sleep "$interval" + done +} diff --git a/tests/e2e/common/log.sh b/tests/e2e/common/log.sh new file mode 100644 index 00000000..30f406e0 --- /dev/null +++ b/tests/e2e/common/log.sh @@ -0,0 +1,26 @@ +${__E2E_COMMON_LOG__:=false} && return 0 || __E2E_COMMON_LOG__=true + +# ENVIRONMENT VARIABLES +# E2E_DEBUG: If set to true, debug messages will be printed to stdout +function log::debug() { + if [[ "${E2E_DEBUG:=false}" == "true" ]]; then + echo -e "\033[00;34m" "[DEBUG]" "$@" "\033[0m" + fi +} + +function log::info() { + echo -e "\033[00;32m" "[INFO]" "$@" "\033[0m" +} + +function log::warn() { + echo -e "\033[00;33m" "[WARN]" "$@" "\033[0m" +} + +function log::error() { + echo -e "\033[00;31m" "[ERROR]" "$@" "\033[0m" +} + +function log::fatal() { + echo -e "\033[00;31m" "[FATAL]" "$@" "\033[0m" + exit 1 +} diff --git a/tests/e2e/e2e.sh b/tests/e2e/e2e.sh new file mode 100755 index 00000000..098a50ac --- /dev/null +++ b/tests/e2e/e2e.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/common/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/testenv/testenv.sh" +source "$(dirname "${BASH_SOURCE[0]}")/cases/cases.sh" + +function setup() { + testenv::k8s::create + testenv::util::install +} + +function teardown() { + testenv::k8s::delete +} + +function list_test_cases() { + local -a functions + IFS=$'\n' read -d '' -ra functions <<<"$(compgen -A function | sort)" && unset IFS + local -a testcases=() + for func in "${functions[@]}"; do + if [[ "$func" =~ ^test::run:: ]]; then + testcase=${func#test::run::} + if [[ -n "${E2E_TEST_CASE_PREFIX:=}" && ${testcase} != "${E2E_TEST_CASE_PREFIX}"* ]]; then + continue + fi + testcases+=("$testcase") + fi + done + echo -n "${testcases[*]}" +} + +function run() { + local -a testcases=() + IFS=" " read -ra testcases <<<"$(list_test_cases)" && unset IFS + local failed=0 + local passed=0 + for testcase in "${testcases[@]}"; do + log::info "=== Running test case: $testcase ===" + if test::run::"$testcase"; then + log::info "Test case passed: $testcase" + ((passed++)) + else + log::error "Test case failed: $testcase" + ((failed++)) + fi + done + if ((failed > 0)); then + log::error "Failed test cases: $failed/${#testcases[@]}" + return "${failed}" + else + log::info "All test cases passed" + fi +} + +function help() { + echo "Xline Operator E2E Test Script" + echo "" + echo "Parameters:" + echo " -p Run selected test cases with prefix" + echo " -h Print this help" + echo " -l List all test cases" + echo " -c Clean the kind cluster." +} + +function main() { + while getopts "p:lhc" opt; do + case "$opt" in + p) + export E2E_TEST_CASE_PREFIX="$OPTARG" + log::info "Run selected test cases with prefix: $E2E_TEST_CASE_PREFIX" + ;; + l) + for testcase in $(list_test_cases); do + echo "$testcase" + done + exit 0 + ;; + c) + teardown + exit 0 + ;; + h) + help + exit 0 + ;; + ?) ;; + esac + done + + setup || return $? + run || return $? + teardown +} + +main "$@" diff --git a/tests/e2e/testenv/minikube.sh b/tests/e2e/testenv/minikube.sh new file mode 100644 index 00000000..216b1d83 --- /dev/null +++ b/tests/e2e/testenv/minikube.sh @@ -0,0 +1,47 @@ +${__E2E_TESTENV_MINIKUBE__:=false} && return 0 || __E2E_TESTENV_MINIKUBE__=true + +function testenv::k8s::minikube::_cluster_status() { + # return "Running", "Stopped" or nothing + minikube status | grep apiserver | awk -F : '{print $2}' | tr -d '[:space:]' +} + +# ENVIRONMENT VARIABLES: +# KIND_CLUSTER_IMAGE (optional): kind cluster image, default to _DEFAULT_KIND_IMAGE +function testenv::k8s::minikube::start() { + KUBEVERSION="${KUBEVERSION:-v1.23.3}" + status=$(testenv::k8s::minikube::_cluster_status) + if [ "$status" == "Running" ]; then + log::warn "minikube cluster already starts, skip strating" + else + log::info "Starting a minikube cluster ..." + if ! minikube start --kubernetes-version=${KUBEVERSION} --image-mirror-country='cn' --driver docker --image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers; then + log::fatal "Failed to start a minikube cluster" + fi + fi +} + +function testenv::k8s::minikube::stop() { + status=$(testenv::k8s::minikube::_cluster_status) + if [ "$status" == "Running" ]; then + log::info "Stopping minikube cluster" + if ! minikube delete; then + log::fatal "Failed to stop minikube cluster" + fi + else + log::warn "minikube cluster does not run, skip stopping" + fi +} + +function testenv::util::etcdctl() { + # shellcheck disable=SC2034 + local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}" + + # retry to avoid mysterious "Error from server: error dialing backend: EOF" error + for ((i = 0; i < ${RETRY_TIMES:-10}; i++)); do + if output=$(k8s::kubectl exec -i etcdctl -- env ETCDCTL_API=3 etcdctl $@ 2>&1); then + echo -e "$output" + return + fi + sleep "${RETRY_INTERVAL:-3}" + done +} diff --git a/tests/e2e/testenv/testenv.sh b/tests/e2e/testenv/testenv.sh new file mode 100644 index 00000000..519262d2 --- /dev/null +++ b/tests/e2e/testenv/testenv.sh @@ -0,0 +1,13 @@ +${__E2E_TESTENV__:=false} && return 0 || __E2E_TESTENV__=true + +source "$(dirname "${BASH_SOURCE[0]}")/minikube.sh" +source "$(dirname "${BASH_SOURCE[0]}")/util/util.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh" + +function testenv::k8s::create() { + testenv::k8s::minikube::start +} + +function testenv::k8s::delete() { + testenv::k8s::minikube::stop +} diff --git a/tests/e2e/testenv/util/manifests/etcdctl.yaml b/tests/e2e/testenv/util/manifests/etcdctl.yaml new file mode 100644 index 00000000..de10a399 --- /dev/null +++ b/tests/e2e/testenv/util/manifests/etcdctl.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: etcdctl +spec: + containers: + - name: etcdctl + image: ghcr.io/xline-kv/etcdctl:v3.5.9 + imagePullPolicy: IfNotPresent + command: + - bash + args: + - -c + - trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT; while true; do sleep 10 & wait; done + enableServiceLinks: false diff --git a/tests/e2e/testenv/util/util.sh b/tests/e2e/testenv/util/util.sh new file mode 100644 index 00000000..dfaf939e --- /dev/null +++ b/tests/e2e/testenv/util/util.sh @@ -0,0 +1,60 @@ +${__E2E_TESTENV_UTIL__:=false} && return 0 || __E2E_TESTENV_UTIL__=true + +_TEST_ENV_UTIL_PATH="$(dirname "${BASH_SOURCE[0]}")" +_UTIL_NAMESPACE="util" + +source "${_TEST_ENV_UTIL_PATH}/../../common/common.sh" + +function testenv::util::_relative_path() { + echo "${_TEST_ENV_UTIL_PATH}/$1" +} + +function testenv::util::_is_installed() { + k8s::kubectl::resource_exist namespace "${_UTIL_NAMESPACE}" +} + +function testenv::util::install() { + if ! testenv::util::_is_installed; then + log::info "Installing util" + if ! k8s::kubectl create namespace "${_UTIL_NAMESPACE}"; then + log::fatal "Failed to create namespace ${_UTIL_NAMESPACE}" + fi + # shellcheck disable=SC2034 + local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}" + k8s::kubectl::wait_resource_creation serviceaccount default + + if ! k8s::kubectl apply -f "$(testenv::util::_relative_path manifests)"; then + log::fatal "Failed to install util" + fi + if ! k8s::kubectl wait --for=condition=Ready pod/etcdctl --timeout=300s; then + log::fatal "Failed to wait for util to be ready" + fi + else + log::warn "Util already installed, skip installing" + fi +} + +function testenv::util::uninstall() { + if testenv::util::_is_installed; then + log::info "Uninstalling util" + if ! k8s::kubectl delete namespace "${_UTIL_NAMESPACE}"; then + log::fatal "Failed to delete namespace ${_UTIL_NAMESPACE}" + fi + else + log::warn "Util not installed, skip uninstalling" + fi +} + +function testenv::util::etcdctl() { + # shellcheck disable=SC2034 + local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}" + + # retry to avoid mysterious "Error from server: error dialing backend: EOF" error + for ((i = 0; i < ${RETRY_TIMES:-10}; i++)); do + if output=$(k8s::kubectl exec -i etcdctl -- env ETCDCTL_API=3 etcdctl $@ 2>&1); then + echo -e "$output" + return + fi + sleep "${RETRY_INTERVAL:-3}" + done +}