Skip to content

Commit

Permalink
fix: fix e2e test will fail sometime
Browse files Browse the repository at this point in the history 
Signed-off-by: Phoeniix Zhao <[email protected]>
  • Loading branch information
@Phoenix500526
Phoenix500526 committed Feb 5, 2024
1 parent a663f8c commit 2a0cf72
Showing 1 changed file with 47 additions and 28 deletions.
75 changes: 47 additions & 28 deletions tests/e2e/cases/ci.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ _TEST_CI_NAMESPACE="default"
_TEST_CI_DNS_SUFFIX="svc.cluster.local"
_TEST_CI_XLINE_PORT="2379"
_TEST_CI_STORAGECLASS_NAME="e2e-storage"
_TEST_CI_LOG_SYNC_TIMEOUT=60
_TEST_CI_LOG_SYNC_TIMEOUT=30

function test::ci::_mk_endpoints() {
local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}"
Expand All @@ -22,13 +22,13 @@ function test::ci::_mk_endpoints() {
}

function test::ci::_etcdctl_expect() {
log::debug "run command: etcdctl --endpoints=$1 $2"
log::info "run command: etcdctl --endpoints=$1 $2"
got=$(testenv::util::etcdctl --endpoints="$1" "$2")
expect=$(echo -e "$3")
if [ "${got//$'\r'/}" == "$expect" ]; then
log::info "command run success"
log::info "command $2 run success"
else
log::error "command run failed"
log::error "command $2 run failed"
log::error "expect: $expect"
log::error "got: $got"
return 1
Expand Down Expand Up @@ -60,12 +60,12 @@ function test::ci::_auth_validation() {
test::ci::_etcdctl_expect "$endpoints" "user add root:root" "User root created" || return $?
test::ci::_etcdctl_expect "$endpoints" "role add root" "Role root created" || return $?
test::ci::_etcdctl_expect "$endpoints" "user grant-role root root" "Role root is granted to user root" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root user list" "etcdserver: authentication is not enabled" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root user list" "etcdserver: authentication is not enabled" || return $?
test::ci::_etcdctl_expect "$endpoints" "auth enable" "Authentication Enabled" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root user add u:u" "User u created" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user u:u user add f:f" "etcdserver: permission denied" || return $?
test::ci::_etcdctl_match "$endpoints" "--user u:u user add f:f" "etcdserver: permission denied" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root role add r" "Role r created" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root user grant-role u r" "Role r is granted to user u" || return $?
test::ci::_etcdctl_expect "$endpoints""--user root:root role grant-permission r readwrite key1" "Role r updated" || return $?
Expand All @@ -80,19 +80,21 @@ function test::ci::_auth_validation() {
test::ci::_etcdctl_expect "$endpoints" "--user root:root role list" "r\nroot" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root user delete u" "User u deleted" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root role delete r" "Role r deleted" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root user get non_exist_user" "etcdserver: user name not found" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root user add root:root" "etcdserver: user name already exists" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root role get non_exist_role" "etcdserver: role name not found" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root role add root" "etcdserver: role name already exists" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $?
test::ci::test::ci::_etcdctl_match "$endpoints" "--user root:root user delete root" "etcdserver: invalid auth management" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root user get non_exist_user" "etcdserver: user name not found" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root user add root:root" "etcdserver: user name already exists" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root role get non_exist_role" "etcdserver: role name not found" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root role add root" "etcdserver: role name already exists" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $?
test::ci::_etcdctl_match "$endpoints" "--user root:root user delete root" "etcdserver: invalid auth management" || return $?
test::ci::_etcdctl_expect "$endpoints" "--user root:root auth disable" "Authentication Disabled" || return $?
log::info "auth validation test passed"
}

function test::ci::_install_CRD() {
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
make install
popd
if [ $? -eq 0 ]; then
log::info "make install: create custom resource definition succeeded"
else
Expand All @@ -101,7 +103,9 @@ function test::ci::_install_CRD() {
}

function test::ci::_uninstall_CRD() {
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
make uninstall
popd
if [ $? -eq 0 ]; then
log::info "make uninstall: remove custom resource definition succeeded"
else
Expand Down Expand Up @@ -157,12 +161,9 @@ function test::ci::_clean_pv() {
}

function test::ci::_start() {
log::info "starting controller"
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
test::ci::stop_controller
test::ci::_install_CRD
make run >/dev/null 2>&1 &
log::info "controller started"
popd
test::ci::start_controller
log::info "create xline auth key pairs"
k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1
k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME
Expand All @@ -172,14 +173,31 @@ function test::ci::_start() {
k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
}

function test::ci::_teardown() {
log::info "stopping controller"
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
test::ci::_uninstall_CRD
controller_pid=$(lsof -i:8081 | awk 'NR==2 {print $2}')
function test::ci::start_controller() {
log::info "starting controller"
pushd $(dirname "${BASH_SOURCE[0]}")/../../../
make run >/dev/null 2>&1 &
popd
sleep 5
if kill -0 $!; then
log::info "Controller started"
else
log::error "Controller cannot failed"
fi
}

function test::ci::stop_controller() {
controller_pid=$(lsof -i:8081 | grep main | awk '{print $2}')
if [ -n "$controller_pid" ]; then
kill -9 $controller_pid
fi
}


function test::ci::_teardown() {
log::info "stopping controller"
test::ci::_uninstall_CRD
test::ci::stop_controller
k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1
test::ci::wait_all_xline_pod_deleted 3
test::ci::_clean_pvc 3
Expand All @@ -190,16 +208,17 @@ function test::ci::_teardown() {
function test::ci::_chaos() {
size=$1
iters=$2
max_kill=$((size / 2))
log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill"
majority=$((size / 2 + 1))
fault_tolerance=$((size - majority))
log::info "chaos: size=$size, iters=$iters, fault_tolerance=$fault_tolerance"
for ((i = 0; i < iters; i++)); do
log::info "chaos: iter=$i"
endpoints=$(test::ci::_mk_endpoints size)
test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $?
test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $?
kill=$((RANDOM % max_kill))
kill=$((RANDOM % fault_tolerance + 1))
log::info "chaos: kill=$kill"
for ((j = 0; j < kill; j++)); do
for ((j = 0; j < $kill; j++)); do
pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))"
log::info "chaos: kill pod=$pod"
k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null
Expand Down

0 comments on commit 2a0cf72

Please sign in to comment.