From 789ca9de2f561317f8452969b7689f8b125fb968 Mon Sep 17 00:00:00 2001 From: Phoeniix Zhao Date: Sun, 26 Nov 2023 23:56:59 +0800 Subject: [PATCH] ci: add e2e test with kind Signed-off-by: iGxnon --- .../aarch64-unknown-linux-gnu.Dockerfile | 9 -- .../x86_64-unknown-linux-gnu.Dockerfile | 6 - .github/workflows/e2e.yml | 40 ++++++ api/v1alpha1/zz_generated.deepcopy.go | 2 +- internal/controller/suite_test.go | 4 +- .../transformer/xlinecluster_resource_test.go | 8 +- tests/e2e/cases/cases.sh | 3 + tests/e2e/cases/ci.sh | 126 ++++++++++++++++++ tests/e2e/cases/manifests/cluster.yml | 8 ++ tests/e2e/common/common.sh | 4 + tests/e2e/common/k8s.sh | 57 ++++++++ tests/e2e/common/log.sh | 26 ++++ tests/e2e/e2e.sh | 98 ++++++++++++++ tests/e2e/testenv/k8s/kind.sh | 56 ++++++++ tests/e2e/testenv/testenv.sh | 26 ++++ tests/e2e/testenv/util/manifests/etcdctl.yaml | 15 +++ tests/e2e/testenv/util/util.sh | 60 +++++++++ 17 files changed, 526 insertions(+), 22 deletions(-) delete mode 100644 .github/scripts/aarch64-unknown-linux-gnu.Dockerfile delete mode 100644 .github/scripts/x86_64-unknown-linux-gnu.Dockerfile create mode 100644 .github/workflows/e2e.yml create mode 100644 tests/e2e/cases/cases.sh create mode 100644 tests/e2e/cases/ci.sh create mode 100644 tests/e2e/cases/manifests/cluster.yml create mode 100644 tests/e2e/common/common.sh create mode 100644 tests/e2e/common/k8s.sh create mode 100644 tests/e2e/common/log.sh create mode 100755 tests/e2e/e2e.sh create mode 100644 tests/e2e/testenv/k8s/kind.sh create mode 100644 tests/e2e/testenv/testenv.sh create mode 100644 tests/e2e/testenv/util/manifests/etcdctl.yaml create mode 100644 tests/e2e/testenv/util/util.sh diff --git a/.github/scripts/aarch64-unknown-linux-gnu.Dockerfile b/.github/scripts/aarch64-unknown-linux-gnu.Dockerfile deleted file mode 100644 index d140edcb..00000000 --- a/.github/scripts/aarch64-unknown-linux-gnu.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main@sha256:b4f5bf74812f9bb6516140d4b83d1f173c2d5ce0523f3e1c2253d99d851c734f - -ENV PKG_CONFIG_ALLOW_CROSS="true" - -RUN set-eux; dpkg --add-architecture arm64 && \ - apt-get update && \ - apt-get install --assume-yes clang-8 libclang-8-dev binutils-aarch64-linux-gnu zlib1g-dev:arm64 unzip wget libssl-dev:arm64 && \ - wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-x86_64.zip && \ - unzip protoc-21.10-linux-x86_64.zip -d /usr/local \ diff --git a/.github/scripts/x86_64-unknown-linux-gnu.Dockerfile b/.github/scripts/x86_64-unknown-linux-gnu.Dockerfile deleted file mode 100644 index d04bfb83..00000000 --- a/.github/scripts/x86_64-unknown-linux-gnu.Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main@sha256:bf0cd3027befe882feb5a2b4040dc6dbdcb799b25c5338342a03163cea43da1b - -RUN set-eux; apt-get update && \ - apt-get install --assume-yes clang libclang-dev unzip wget libssl-dev && \ - wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-x86_64.zip && \ - unzip protoc-21.10-linux-x86_64.zip -d /usr/local \ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..a5abb081 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,40 @@ +name: E2E + +on: + pull_request: + branches: + - main + workflow_dispatch: { } + +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency +concurrency: + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + KIND_VERSION: v0.20.0 + GO_VERSION: '1.21.3' + +jobs: + validation: + name: 'Validation' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: go/src/github.com/${{ github.repository }}/go.mod + go-version: ${{ env.GO_VERSION }} + - name: 'Install Kind' + run: | + wget https://github.com/kubernetes-sigs/kind/releases/download/${{ env.KIND_VERSION }}/kind-linux-amd64 + chmod +x kind-linux-amd64 + mv kind-linux-amd64 /usr/local/bin/kind + - name: 'E2E CI' + env: + KIND_CLUSTER_IMAGE: kindest/node:v1.27.3 + run: bash ./tests/e2e/e2e.sh -p ci + - name: clean + if: failure() + run: bash ./tests/e2e/e2e.sh -c diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6a7cc92b..e4cf3039 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/apps/v1" + "k8s.io/api/apps/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 3fd0cb28..80181fc6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -33,7 +33,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - xlinekvstoredatenlordcomv1alpha1 "github.com/xline-kv/xline-operator/api/v1alpha1" + xapi "github.com/xline-kv/xline-operator/api/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -79,7 +79,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = xlinekvstoredatenlordcomv1alpha1.AddToScheme(scheme.Scheme) + err = xapi.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme diff --git a/internal/transformer/xlinecluster_resource_test.go b/internal/transformer/xlinecluster_resource_test.go index 02d9e8fe..86ad5c5a 100644 --- a/internal/transformer/xlinecluster_resource_test.go +++ b/internal/transformer/xlinecluster_resource_test.go @@ -11,14 +11,14 @@ import ( ) func TestXlineClusterFunc(t *testing.T) { - test_image := "xline-img:latest" + testImage := "xline-img:latest" xlineCluster := xapi.XlineCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "xline", Namespace: "default", }, Spec: xapi.XlineClusterSpec{ - Image: &test_image, + Image: &testImage, Replicas: 3, }, } @@ -38,8 +38,8 @@ func TestXlineClusterFunc(t *testing.T) { }) t.Run("GetXlineImage should work properly", func(t *testing.T) { - xline_image := *xlineCluster.Spec.Image - assert.Equal(t, xline_image, "xline-img:latest") + xlineImage := *xlineCluster.Spec.Image + assert.Equal(t, xlineImage, "xline-img:latest") }) t.Run("GetMemberTopology should work properly", func(t *testing.T) { diff --git a/tests/e2e/cases/cases.sh b/tests/e2e/cases/cases.sh new file mode 100644 index 00000000..690d96de --- /dev/null +++ b/tests/e2e/cases/cases.sh @@ -0,0 +1,3 @@ +${__E2E_CASES__:=false} && return 0 || __E2E_CASES__=true + +source "$(dirname "${BASH_SOURCE[0]}")/ci.sh" diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh new file mode 100644 index 00000000..2b1da0ac --- /dev/null +++ b/tests/e2e/cases/ci.sh @@ -0,0 +1,126 @@ +${__E2E_CASES_CI__:=false} && return 0 || __E2E_CASES_CI__=true + +source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh" + +_TEST_CI_CLUSTER_NAME="my-xline-cluster" +_TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts" +_TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc" +_TEST_CI_NAMESPACE="default" +_TEST_CI_DNS_SUFFIX="svc.cluster.local" +_TEST_CI_XLINE_PORT="2379" +_TEST_CI_LOG_SYNC_TIMEOUT=60 + +function test::ci::_mk_endpoints() { + local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" + for ((i = 1; i < $1; i++)); do + endpoints="${endpoints},${_TEST_CI_STS_NAME}-${i}.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" + done + echo "$endpoints" +} + +function test::ci::_etcdctl_expect() { + log::debug "run command: etcdctl --endpoints=$1 $2" + got=$(testenv::util::etcdctl --endpoints="$1" "$2") + expect=$(echo -e "$3") + if [ "${got//$'\r'/}" == "$expect" ]; then + log::info "command run success" + else + log::error "command run failed" + log::error "expect: $expect" + log::error "got: $got" + return 1 + fi +} + +function test::ci::_install_CRD() { + make install + if [ $? -eq 0 ]; then + log::info "make install: create custom resource definition succeeded" + else + log::error "make install: create custom resource definition failed" + fi +} + +function test::ci::_uninstall_CRD() { + make uninstall + if [ $? -eq 0 ]; then + log::info "make uninstall: remove custom resource definition succeeded" + else + log::error "make uninstall: remove custom resource definition failed" + fi +} + +function test::ci::wait_all_xline_pod_ready() { + for ((i = 0; i < $1; i++)); do + log::info "wait pod/${_TEST_CI_STS_NAME}-${i} to be ready" + if ! k8s::kubectl wait --for=condition=Ready pod/${_TEST_CI_STS_NAME}-${i} --timeout=300s; then + log::fatal "Failed to wait for util to be ready" + fi + done +} + +function test::ci::_start() { + log::info "starting controller" + pushd $(dirname "${BASH_SOURCE[0]}")/../../../ + test::ci::_install_CRD + make run >/dev/null 2>&1 & + log::info "controller started" + popd + log::info "starting xline cluster" + k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yml" >/dev/null 2>&1 + k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME +} + +function test::ci::_teardown() { + log::info "stopping controller" + pushd $(dirname "${BASH_SOURCE[0]}")/../../../ + test::ci::_uninstall_CRD + controller_pid=$(ps aux | grep "[g]o run ./cmd/main.go" | awk '{print $2}') + if [ -n "$controller_pid" ]; then + kill -9 $controller_pid + fi +} + +function test::ci::_chaos() { + size=$1 + iters=$2 + max_kill=$((size / 2)) + log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill" + for ((i = 0; i < iters; i++)); do + log::info "chaos: iter=$i" + endpoints=$(test::ci::_mk_endpoints size) + test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $? + kill=$((RANDOM % max_kill)) + log::info "chaos: kill=$kill" + for ((j = 0; j < kill; j++)); do + pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))" + log::info "chaos: kill pod=$pod" + k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null + done + test::ci::_etcdctl_expect "$endpoints" "put B $i" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get B" "B\n$i" || return $? + k8s::kubectl wait --for=jsonpath='{.status.readyReplicas}'="$size" sts/$_TEST_CI_CLUSTER_NAME --timeout=300s >/dev/null 2>&1 + log::info "wait for log synchronization" && sleep $_TEST_CI_LOG_SYNC_TIMEOUT + done +} + +function test::run::ci::basic_validation() { + test::ci::_start + test::ci::wait_all_xline_pod_ready 3 + endpoints=$(test::ci::_mk_endpoints 3) + test::ci::_etcdctl_expect "$endpoints" "put A 1" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get A" "A\n1" || return $? + endpoints=$(test::ci::_mk_endpoints 1) + test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $? + test::ci::_teardown +} + + +function test::run::ci::basic_chaos() { + test::ci::_start + test::ci::_chaos 3 5 || return $? + test::ci::_teardown +} diff --git a/tests/e2e/cases/manifests/cluster.yml b/tests/e2e/cases/manifests/cluster.yml new file mode 100644 index 00000000..b276b307 --- /dev/null +++ b/tests/e2e/cases/manifests/cluster.yml @@ -0,0 +1,8 @@ +apiVersion: xline.io.datenlord.com/v1alpha1 +kind: XlineCluster +metadata: + name: my-xline-cluster +spec: + image: phoenix500526/xline:v0.6.1 + imagePullPolicy: IfNotPresent + replicas: 3 diff --git a/tests/e2e/common/common.sh b/tests/e2e/common/common.sh new file mode 100644 index 00000000..4f456e7e --- /dev/null +++ b/tests/e2e/common/common.sh @@ -0,0 +1,4 @@ +${__E2E_COMMON__:=false} && return 0 || __E2E_COMMON__=true + +source "$(dirname "${BASH_SOURCE[0]}")/log.sh" +source "$(dirname "${BASH_SOURCE[0]}")/k8s.sh" diff --git a/tests/e2e/common/k8s.sh b/tests/e2e/common/k8s.sh new file mode 100644 index 00000000..fd1bee2d --- /dev/null +++ b/tests/e2e/common/k8s.sh @@ -0,0 +1,57 @@ +${__E2E_COMMON_K8S__:=false} && return 0 || __E2E_COMMON_K8S__=true + +# ENVIRONMENT VARIABLES: +# KUBECTL: path to kubectl binary +# KUBECTL_NAMESPACE: namespace to use for kubectl commands +# +# ARGUMENTS: +# $@: arguments to pass to kubectl +function k8s::kubectl() { + KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}" + local kubectl="${KUBECTL:-kubectl}" + ${kubectl} -n "${KUBECTL_NAMESPACE}" "$@" +} + +# ENVIRONMENT VARIABLES: +# KUBECTL: path to kubectl binary +# KUBECTL_NAMESPACE: namespace to use for kubectl commands +# +# ARGUMENTS: +# $1: resource type +# $2: resource name +function k8s::kubectl::resource_exist() { + KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}" + k8s::kubectl get "$1" "$2" >/dev/null 2>&1 +} + +# ENVIRONMENT VARIABLES: +# KUBECTL: path to kubectl binary +# KUBECTL_NAMESPACE: namespace to use for kubectl commands +# +# ARGUMENTS: +# $1: resource type +# $2: resource name +# $3: (optional) interval to check resource creation +function k8s::kubectl::wait_resource_creation() { + interval="${3:-5}" + KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}" + retry_limit="${4:-10}" + retry_count=0 + + while true; do + if k8s::kubectl::resource_exist "$1" "$2"; then + log::info "Resource $1/$2 ($KUBECTL_NAMESPACE) created" + break + fi + + if [ "$retry_count" -ge "$retry_limit" ]; then + log::error "Exceeded retry limit for $1/$2 ($KUBECTL_NAMESPACE)" + break + fi + + retry_count=$((retry_count + 1)) + + log::info "Waiting for $1/$2 ($KUBECTL_NAMESPACE) to be created" + sleep "$interval" + done +} diff --git a/tests/e2e/common/log.sh b/tests/e2e/common/log.sh new file mode 100644 index 00000000..30f406e0 --- /dev/null +++ b/tests/e2e/common/log.sh @@ -0,0 +1,26 @@ +${__E2E_COMMON_LOG__:=false} && return 0 || __E2E_COMMON_LOG__=true + +# ENVIRONMENT VARIABLES +# E2E_DEBUG: If set to true, debug messages will be printed to stdout +function log::debug() { + if [[ "${E2E_DEBUG:=false}" == "true" ]]; then + echo -e "\033[00;34m" "[DEBUG]" "$@" "\033[0m" + fi +} + +function log::info() { + echo -e "\033[00;32m" "[INFO]" "$@" "\033[0m" +} + +function log::warn() { + echo -e "\033[00;33m" "[WARN]" "$@" "\033[0m" +} + +function log::error() { + echo -e "\033[00;31m" "[ERROR]" "$@" "\033[0m" +} + +function log::fatal() { + echo -e "\033[00;31m" "[FATAL]" "$@" "\033[0m" + exit 1 +} diff --git a/tests/e2e/e2e.sh b/tests/e2e/e2e.sh new file mode 100755 index 00000000..29b736e2 --- /dev/null +++ b/tests/e2e/e2e.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/common/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/testenv/testenv.sh" +source "$(dirname "${BASH_SOURCE[0]}")/cases/cases.sh" + +function setup() { + testenv::k8s::create + testenv::k8s::load_images + testenv::util::install +} + +function teardown() { + testenv::k8s::delete +} + +function list_test_cases() { + local -a functions + IFS=$'\n' read -d '' -ra functions <<<"$(compgen -A function | sort)" && unset IFS + local -a testcases=() + for func in "${functions[@]}"; do + if [[ "$func" =~ ^test::run:: ]]; then + testcase=${func#test::run::} + if [[ -n "${E2E_TEST_CASE_PREFIX:=}" && ${testcase} != "${E2E_TEST_CASE_PREFIX}"* ]]; then + continue + fi + testcases+=("$testcase") + fi + done + echo -n "${testcases[*]}" +} + +function run() { + local -a testcases=() + IFS=" " read -ra testcases <<<"$(list_test_cases)" && unset IFS + local failed=0 + local passed=0 + for testcase in "${testcases[@]}"; do + log::info "=== Running test case: $testcase ===" + if test::run::"$testcase"; then + log::info "Test case passed: $testcase" + ((passed++)) + else + log::error "Test case failed: $testcase" + ((failed++)) + fi + done + if ((failed > 0)); then + log::error "Failed test cases: $failed/${#testcases[@]}" + return "${failed}" + else + log::info "All test cases passed" + fi +} + +function help() { + echo "Xline Operator E2E Test Script" + echo "" + echo "Parameters:" + echo " -p Run selected test cases with prefix" + echo " -h Print this help" + echo " -l List all test cases" + echo " -c Clean the kind cluster." +} + +function main() { + while getopts "p:lhc" opt; do + case "$opt" in + p) + export E2E_TEST_CASE_PREFIX="$OPTARG" + log::info "Run selected test cases with prefix: $E2E_TEST_CASE_PREFIX" + ;; + l) + for testcase in $(list_test_cases); do + echo "$testcase" + done + exit 0 + ;; + c) + teardown + exit 0 + ;; + h) + help + exit 0 + ;; + ?) ;; + esac + done + + setup || return $? + run || return $? + teardown +} + +main "$@" diff --git a/tests/e2e/testenv/k8s/kind.sh b/tests/e2e/testenv/k8s/kind.sh new file mode 100644 index 00000000..151eea3e --- /dev/null +++ b/tests/e2e/testenv/k8s/kind.sh @@ -0,0 +1,56 @@ +${__E2E_TESTENV_KIND__:=false} && return 0 || __E2E_TESTENV_KIND__=true + +_TEST_ENV_KIND_CLUSTER_NAME="e2e-kind" +_DEFAULT_KIND_IMAGE="kindest/node:v1.27.3" + +source "$(dirname "${BASH_SOURCE[0]}")/../../common/common.sh" + +function testenv::k8s::kind::_cluster_exists() { + kind get clusters -q | grep -w -q "${_TEST_ENV_KIND_CLUSTER_NAME}" +} + +# ENVIRONMENT VARIABLES: +# KIND_CLUSTER_IMAGE (optional): kind cluster image, default to _DEFAULT_KIND_IMAGE +function testenv::k8s::kind::create() { + local kind_image="${KIND_CLUSTER_IMAGE:-${_DEFAULT_KIND_IMAGE}}" + if ! testenv::k8s::kind::_cluster_exists; then + log::info "Creating kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + if ! kind create cluster --name "${_TEST_ENV_KIND_CLUSTER_NAME}" --image "${kind_image}"; then + log::fatal "Failed to create kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + fi + else + log::warn "Kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME} already exists, skip creating" + fi +} + +function testenv::k8s::kind::export() { + if testenv::k8s::kind::_cluster_exists; then + log::info "Exporting logs kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + kind export logs --name ${_TEST_ENV_KIND_CLUSTER_NAME} /tmp/xlineoperator/${_TEST_ENV_KIND_CLUSTER_NAME} + else + log::warn "Kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME} does not exist, skip export logs" + fi +} + +function testenv::k8s::kind::delete() { + if testenv::k8s::kind::_cluster_exists; then + log::info "Deleting kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + if ! kind delete cluster --name "${_TEST_ENV_KIND_CLUSTER_NAME}"; then + log::fatal "Failed to delete kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + fi + else + log::warn "Kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME} does not exist, skip deleting" + fi +} + +function testenv::k8s::kind::load_image() { + log::info "Loading local docker images:" "$@" + if ! testenv::k8s::kind::_cluster_exists; then + log::fatal "Kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME} does not exist, cannot load image" + fi + if kind load docker-image --name "${_TEST_ENV_KIND_CLUSTER_NAME}" "$@"; then + log::info "Successfully loaded image into kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + else + log::error "Failed to load image into kind cluster ${_TEST_ENV_KIND_CLUSTER_NAME}" + fi +} diff --git a/tests/e2e/testenv/testenv.sh b/tests/e2e/testenv/testenv.sh new file mode 100644 index 00000000..d2ba30d2 --- /dev/null +++ b/tests/e2e/testenv/testenv.sh @@ -0,0 +1,26 @@ +${__E2E_TESTENV__:=false} && return 0 || __E2E_TESTENV__=true + +source "$(dirname "${BASH_SOURCE[0]}")/k8s/kind.sh" +source "$(dirname "${BASH_SOURCE[0]}")/util/util.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh" + +function testenv::k8s::create() { + testenv::k8s::kind::create +} + +function testenv::k8s::delete() { + testenv::k8s::kind::export + testenv::k8s::kind::delete +} + +function testenv::k8s::load_images() { + # xline image + log::info "Loading images" + xline_image="phoenix500526/xline:v0.6.1" + docker pull "$xline_image" 2>/dev/null + testenv::k8s::kind::load_image "$xline_image" + # etcdctl image + etcdctl_image="ghcr.io/xline-kv/etcdctl:v3.5.9" + docker pull "$etcdctl_image" 2>/dev/null + testenv::k8s::kind::load_image "$etcdctl_image" +} diff --git a/tests/e2e/testenv/util/manifests/etcdctl.yaml b/tests/e2e/testenv/util/manifests/etcdctl.yaml new file mode 100644 index 00000000..de10a399 --- /dev/null +++ b/tests/e2e/testenv/util/manifests/etcdctl.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: etcdctl +spec: + containers: + - name: etcdctl + image: ghcr.io/xline-kv/etcdctl:v3.5.9 + imagePullPolicy: IfNotPresent + command: + - bash + args: + - -c + - trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT; while true; do sleep 10 & wait; done + enableServiceLinks: false diff --git a/tests/e2e/testenv/util/util.sh b/tests/e2e/testenv/util/util.sh new file mode 100644 index 00000000..dfaf939e --- /dev/null +++ b/tests/e2e/testenv/util/util.sh @@ -0,0 +1,60 @@ +${__E2E_TESTENV_UTIL__:=false} && return 0 || __E2E_TESTENV_UTIL__=true + +_TEST_ENV_UTIL_PATH="$(dirname "${BASH_SOURCE[0]}")" +_UTIL_NAMESPACE="util" + +source "${_TEST_ENV_UTIL_PATH}/../../common/common.sh" + +function testenv::util::_relative_path() { + echo "${_TEST_ENV_UTIL_PATH}/$1" +} + +function testenv::util::_is_installed() { + k8s::kubectl::resource_exist namespace "${_UTIL_NAMESPACE}" +} + +function testenv::util::install() { + if ! testenv::util::_is_installed; then + log::info "Installing util" + if ! k8s::kubectl create namespace "${_UTIL_NAMESPACE}"; then + log::fatal "Failed to create namespace ${_UTIL_NAMESPACE}" + fi + # shellcheck disable=SC2034 + local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}" + k8s::kubectl::wait_resource_creation serviceaccount default + + if ! k8s::kubectl apply -f "$(testenv::util::_relative_path manifests)"; then + log::fatal "Failed to install util" + fi + if ! k8s::kubectl wait --for=condition=Ready pod/etcdctl --timeout=300s; then + log::fatal "Failed to wait for util to be ready" + fi + else + log::warn "Util already installed, skip installing" + fi +} + +function testenv::util::uninstall() { + if testenv::util::_is_installed; then + log::info "Uninstalling util" + if ! k8s::kubectl delete namespace "${_UTIL_NAMESPACE}"; then + log::fatal "Failed to delete namespace ${_UTIL_NAMESPACE}" + fi + else + log::warn "Util not installed, skip uninstalling" + fi +} + +function testenv::util::etcdctl() { + # shellcheck disable=SC2034 + local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}" + + # retry to avoid mysterious "Error from server: error dialing backend: EOF" error + for ((i = 0; i < ${RETRY_TIMES:-10}; i++)); do + if output=$(k8s::kubectl exec -i etcdctl -- env ETCDCTL_API=3 etcdctl $@ 2>&1); then + echo -e "$output" + return + fi + sleep "${RETRY_INTERVAL:-3}" + done +}