Skip to content

Commit

Permalink
ci: add e2e test with kind
Browse files Browse the repository at this point in the history
Signed-off-by: iGxnon <[email protected]>
  • Loading branch information
Phoenix500526 authored and iGxnon committed Jan 22, 2024
1 parent 0098075 commit fe90a70
Show file tree
Hide file tree
Showing 18 changed files with 526 additions and 22 deletions.
9 changes: 0 additions & 9 deletions .github/scripts/aarch64-unknown-linux-gnu.Dockerfile

This file was deleted.

6 changes: 0 additions & 6 deletions .github/scripts/x86_64-unknown-linux-gnu.Dockerfile

This file was deleted.

39 changes: 39 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: E2E

on:
pull_request:
branches:
- main
workflow_dispatch: { }

# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency
concurrency:
group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

env:
KIND_VERSION: v0.20.0
GO_VERSION: '1.21.3'

jobs:
validation:
name: 'Validation'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version-file: go.mod
- name: 'Install Kind'
run: |
wget https://github.com/kubernetes-sigs/kind/releases/download/${{ env.KIND_VERSION }}/kind-linux-amd64
chmod +x kind-linux-amd64
mv kind-linux-amd64 /usr/local/bin/kind
- name: 'E2E CI'
env:
KIND_CLUSTER_IMAGE: kindest/node:v1.27.3
run: bash ./tests/e2e/e2e.sh -p ci
- name: clean
if: failure()
run: bash ./tests/e2e/e2e.sh -c
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ repos:
- id: check-merge-conflict
- id: check-symlinks
- id: check-yaml
args: [ --allow-multiple-documents ]
- id: end-of-file-fixer
- id: mixed-line-ending
- id: trailing-whitespace
Expand Down
2 changes: 1 addition & 1 deletion api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions internal/controller/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import (
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

xlinekvstoredatenlordcomv1alpha1 "github.com/xline-kv/xline-operator/api/v1alpha1"
xapi "github.com/xline-kv/xline-operator/api/v1alpha1"
//+kubebuilder:scaffold:imports
)

Expand Down Expand Up @@ -79,7 +79,7 @@ var _ = BeforeSuite(func() {
Expect(err).NotTo(HaveOccurred())
Expect(cfg).NotTo(BeNil())

err = xlinekvstoredatenlordcomv1alpha1.AddToScheme(scheme.Scheme)
err = xapi.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())

//+kubebuilder:scaffold:scheme
Expand Down
8 changes: 4 additions & 4 deletions internal/transformer/xlinecluster_resource_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ import (
)

func TestXlineClusterFunc(t *testing.T) {
test_image := "xline-img:latest"
testImage := "xline-img:latest"
xlineCluster := xapi.XlineCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "xline",
Namespace: "default",
},
Spec: xapi.XlineClusterSpec{
Image: &test_image,
Image: &testImage,
Replicas: 3,
},
}
Expand All @@ -38,8 +38,8 @@ func TestXlineClusterFunc(t *testing.T) {
})

t.Run("GetXlineImage should work properly", func(t *testing.T) {
xline_image := *xlineCluster.Spec.Image
assert.Equal(t, xline_image, "xline-img:latest")
xlineImage := *xlineCluster.Spec.Image
assert.Equal(t, xlineImage, "xline-img:latest")
})

t.Run("GetMemberTopology should work properly", func(t *testing.T) {
Expand Down
3 changes: 3 additions & 0 deletions tests/e2e/cases/cases.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
${__E2E_CASES__:=false} && return 0 || __E2E_CASES__=true

source "$(dirname "${BASH_SOURCE[0]}")/ci.sh"
126 changes: 126 additions & 0 deletions tests/e2e/cases/ci.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
${__E2E_CASES_CI__:=false} && return 0 || __E2E_CASES_CI__=true

source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh"
source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh"

_TEST_CI_CLUSTER_NAME="my-xline-cluster"
_TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts"
_TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc"
_TEST_CI_NAMESPACE="default"
_TEST_CI_DNS_SUFFIX="svc.cluster.local"
_TEST_CI_XLINE_PORT="2379"
_TEST_CI_LOG_SYNC_TIMEOUT=60

function test::ci::_mk_endpoints() {
local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}"
for ((i = 1; i < $1; i++)); do
endpoints="${endpoints},${_TEST_CI_STS_NAME}-${i}.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}"
done
echo "$endpoints"
}

function test::ci::_etcdctl_expect() {
log::debug "run command: etcdctl --endpoints=$1 $2"
got=$(testenv::util::etcdctl --endpoints="$1" "$2")
expect=$(echo -e "$3")
if [ "${got//$'\r'/}" == "$expect" ]; then
log::info "command run success"
else
log::error "command run failed"
log::error "expect: $expect"
log::error "got: $got"
return 1
fi
}

function test::ci::_install_CRD() {
make install
if [ $? -eq 0 ]; then
log::info "make install: create custom resource definition succeeded"
else
log::error "make install: create custom resource definition failed"
fi
}

function test::ci::_uninstall_CRD() {
make uninstall
if [ $? -eq 0 ]; then
log::info "make uninstall: remove custom resource definition succeeded"
else
log::error "make uninstall: remove custom resource definition failed"
fi
}

function test::ci::wait_all_xline_pod_ready() {
for ((i = 0; i < $1; i++)); do
log::info "wait pod/${_TEST_CI_STS_NAME}-${i} to be ready"
if ! k8s::kubectl wait --for=condition=Ready pod/${_TEST_CI_STS_NAME}-${i} --timeout=300s; then
log::fatal "Failed to wait for util to be ready"
fi
done
}

function test::ci::_start() {
log::info "starting controller"
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
test::ci::_install_CRD
make run >/dev/null 2>&1 &
log::info "controller started"
popd
log::info "starting xline cluster"
k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yml" >/dev/null 2>&1
k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
}

function test::ci::_teardown() {
log::info "stopping controller"
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
test::ci::_uninstall_CRD
controller_pid=$(ps aux | grep "[g]o run ./cmd/main.go" | awk '{print $2}')
if [ -n "$controller_pid" ]; then
kill -9 $controller_pid
fi
}

function test::ci::_chaos() {
size=$1
iters=$2
max_kill=$((size / 2))
log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill"
for ((i = 0; i < iters; i++)); do
log::info "chaos: iter=$i"
endpoints=$(test::ci::_mk_endpoints size)
test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $?
test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $?
kill=$((RANDOM % max_kill))
log::info "chaos: kill=$kill"
for ((j = 0; j < kill; j++)); do
pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))"
log::info "chaos: kill pod=$pod"
k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null
done
test::ci::_etcdctl_expect "$endpoints" "put B $i" "OK" || return $?
test::ci::_etcdctl_expect "$endpoints" "get B" "B\n$i" || return $?
k8s::kubectl wait --for=jsonpath='{.status.readyReplicas}'="$size" sts/$_TEST_CI_CLUSTER_NAME --timeout=300s >/dev/null 2>&1
log::info "wait for log synchronization" && sleep $_TEST_CI_LOG_SYNC_TIMEOUT
done
}

function test::run::ci::basic_validation() {
test::ci::_start
test::ci::wait_all_xline_pod_ready 3
endpoints=$(test::ci::_mk_endpoints 3)
test::ci::_etcdctl_expect "$endpoints" "put A 1" "OK" || return $?
test::ci::_etcdctl_expect "$endpoints" "get A" "A\n1" || return $?
endpoints=$(test::ci::_mk_endpoints 1)
test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $?
test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $?
test::ci::_teardown
}


function test::run::ci::basic_chaos() {
test::ci::_start
test::ci::_chaos 3 5 || return $?
test::ci::_teardown
}
8 changes: 8 additions & 0 deletions tests/e2e/cases/manifests/cluster.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: xline.io.datenlord.com/v1alpha1
kind: XlineCluster
metadata:
name: my-xline-cluster
spec:
image: phoenix500526/xline:v0.6.1
imagePullPolicy: IfNotPresent
replicas: 3
4 changes: 4 additions & 0 deletions tests/e2e/common/common.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
${__E2E_COMMON__:=false} && return 0 || __E2E_COMMON__=true

source "$(dirname "${BASH_SOURCE[0]}")/log.sh"
source "$(dirname "${BASH_SOURCE[0]}")/k8s.sh"
57 changes: 57 additions & 0 deletions tests/e2e/common/k8s.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
${__E2E_COMMON_K8S__:=false} && return 0 || __E2E_COMMON_K8S__=true

# ENVIRONMENT VARIABLES:
# KUBECTL: path to kubectl binary
# KUBECTL_NAMESPACE: namespace to use for kubectl commands
#
# ARGUMENTS:
# $@: arguments to pass to kubectl
function k8s::kubectl() {
KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}"
local kubectl="${KUBECTL:-kubectl}"
${kubectl} -n "${KUBECTL_NAMESPACE}" "$@"
}

# ENVIRONMENT VARIABLES:
# KUBECTL: path to kubectl binary
# KUBECTL_NAMESPACE: namespace to use for kubectl commands
#
# ARGUMENTS:
# $1: resource type
# $2: resource name
function k8s::kubectl::resource_exist() {
KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}"
k8s::kubectl get "$1" "$2" >/dev/null 2>&1
}

# ENVIRONMENT VARIABLES:
# KUBECTL: path to kubectl binary
# KUBECTL_NAMESPACE: namespace to use for kubectl commands
#
# ARGUMENTS:
# $1: resource type
# $2: resource name
# $3: (optional) interval to check resource creation
function k8s::kubectl::wait_resource_creation() {
interval="${3:-5}"
KUBECTL_NAMESPACE="${KUBECTL_NAMESPACE:-default}"
retry_limit="${4:-10}"
retry_count=0

while true; do
if k8s::kubectl::resource_exist "$1" "$2"; then
log::info "Resource $1/$2 ($KUBECTL_NAMESPACE) created"
break
fi

if [ "$retry_count" -ge "$retry_limit" ]; then
log::error "Exceeded retry limit for $1/$2 ($KUBECTL_NAMESPACE)"
break
fi

retry_count=$((retry_count + 1))

log::info "Waiting for $1/$2 ($KUBECTL_NAMESPACE) to be created"
sleep "$interval"
done
}
26 changes: 26 additions & 0 deletions tests/e2e/common/log.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
${__E2E_COMMON_LOG__:=false} && return 0 || __E2E_COMMON_LOG__=true

# ENVIRONMENT VARIABLES
# E2E_DEBUG: If set to true, debug messages will be printed to stdout
function log::debug() {
if [[ "${E2E_DEBUG:=false}" == "true" ]]; then
echo -e "\033[00;34m" "[DEBUG]" "$@" "\033[0m"
fi
}

function log::info() {
echo -e "\033[00;32m" "[INFO]" "$@" "\033[0m"
}

function log::warn() {
echo -e "\033[00;33m" "[WARN]" "$@" "\033[0m"
}

function log::error() {
echo -e "\033[00;31m" "[ERROR]" "$@" "\033[0m"
}

function log::fatal() {
echo -e "\033[00;31m" "[FATAL]" "$@" "\033[0m"
exit 1
}
Loading

0 comments on commit fe90a70

Please sign in to comment.