From 86a4deb84e99023f5736e6edc3e8cf2bf5d5ebb7 Mon Sep 17 00:00:00 2001
From: Phoeniix Zhao <Phoenix500526@163.com>
Date: Wed, 20 Dec 2023 19:16:41 +0800
Subject: [PATCH 1/5] feat: use secret to specify xline auth key pair

Signed-off-by: Phoeniix Zhao <Phoenix500526@163.com>
---
 api/v1alpha1/xlinecluster_types.go            | 10 +++
 api/v1alpha1/zz_generated.deepcopy.go         | 40 ++++++++++++
 .../xline.io.datenlord.com_xlineclusters.yaml | 17 ++++++
 internal/transformer/xlinecluster_resource.go | 59 +++++++++++++++---
 tests/e2e/cases/ci.sh                         | 61 +++++++++++++++++++
 tests/e2e/cases/manifests/auth-cred.yaml      |  8 +++
 tests/e2e/cases/manifests/cluster.yaml        |  8 +++
 7 files changed, 193 insertions(+), 10 deletions(-)
 create mode 100644 tests/e2e/cases/manifests/auth-cred.yaml

diff --git a/api/v1alpha1/xlinecluster_types.go b/api/v1alpha1/xlinecluster_types.go
index 4b472378..b2f4c657 100644
--- a/api/v1alpha1/xlinecluster_types.go
+++ b/api/v1alpha1/xlinecluster_types.go
@@ -133,6 +133,16 @@ type XlineClusterSpec struct {
 	// The replicas of xline nodes
 	// +kubebuilder:validation:Minimum=3
 	Replicas int32 `json:"replicas"`
+
+	// The auth secret keys
+	AuthSecrets *XlineAuthSecret `json:"authSecret,omitempty"`
+}
+
+type XlineAuthSecret struct {
+	Name      *string `json:"name"`
+	MountPath *string `json:"mountPath"`
+	PubKey    *string `json:"pubKey"`
+	PriKey    *string `json:"priKey"`
 }
 
 func (s *XlineClusterSpec) BootArgs() []string {
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 6c2b1f8d..72d0388b 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -171,6 +171,41 @@ func (in *XlineArgs) DeepCopy() *XlineArgs {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *XlineAuthSecret) DeepCopyInto(out *XlineAuthSecret) {
+	*out = *in
+	if in.Name != nil {
+		in, out := &in.Name, &out.Name
+		*out = new(string)
+		**out = **in
+	}
+	if in.MountPath != nil {
+		in, out := &in.MountPath, &out.MountPath
+		*out = new(string)
+		**out = **in
+	}
+	if in.PubKey != nil {
+		in, out := &in.PubKey, &out.PubKey
+		*out = new(string)
+		**out = **in
+	}
+	if in.PriKey != nil {
+		in, out := &in.PriKey, &out.PriKey
+		*out = new(string)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineAuthSecret.
+func (in *XlineAuthSecret) DeepCopy() *XlineAuthSecret {
+	if in == nil {
+		return nil
+	}
+	out := new(XlineAuthSecret)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *XlineCluster) DeepCopyInto(out *XlineCluster) {
 	*out = *in
@@ -259,6 +294,11 @@ func (in *XlineClusterSpec) DeepCopyInto(out *XlineClusterSpec) {
 		**out = **in
 	}
 	in.BootstrapArgs.DeepCopyInto(&out.BootstrapArgs)
+	if in.AuthSecrets != nil {
+		in, out := &in.AuthSecrets, &out.AuthSecrets
+		*out = new(XlineAuthSecret)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineClusterSpec.
diff --git a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml
index a845968e..3fc8d020 100644
--- a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml
+++ b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml
@@ -35,6 +35,23 @@ spec:
           spec:
             description: XlineClusterSpec defines the desired state of XlineCluster
             properties:
+              authSecret:
+                description: The auth secret keys
+                properties:
+                  mountPath:
+                    type: string
+                  name:
+                    type: string
+                  priKey:
+                    type: string
+                  pubKey:
+                    type: string
+                required:
+                - mountPath
+                - name
+                - priKey
+                - pubKey
+                type: object
               bootstrapArgs:
                 description: / Xline container bootstrap arguments / Set additional
                   arguments except [`--name`, `--members`, `--storage-engine`, `--data-dir`]
diff --git a/internal/transformer/xlinecluster_resource.go b/internal/transformer/xlinecluster_resource.go
index 99907ac8..bb9c3b2b 100644
--- a/internal/transformer/xlinecluster_resource.go
+++ b/internal/transformer/xlinecluster_resource.go
@@ -46,6 +46,38 @@ func GetMemberTopology(stsRef types.NamespacedName, svcName string, replicas int
 	return strings.Join(members, ",")
 }
 
+func GetAuthSecretVolume(auth_sec *xapi.XlineAuthSecret) []corev1.Volume {
+	if auth_sec == nil {
+		return []corev1.Volume{}
+	}
+	return []corev1.Volume{
+		{Name: "auth-cred", VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: *auth_sec.Name,
+			},
+		}},
+	}
+}
+
+func GetAuthSecretVolumeMount(auth_sec *xapi.XlineAuthSecret) []corev1.VolumeMount {
+	if auth_sec == nil {
+		return []corev1.VolumeMount{}
+	}
+	return []corev1.VolumeMount{
+		{Name: "auth-cred", ReadOnly: true, MountPath: *auth_sec.MountPath},
+	}
+}
+
+func GetAuthSecretEnvVars(auth_sec *xapi.XlineAuthSecret) []corev1.EnvVar {
+	if auth_sec == nil {
+		return []corev1.EnvVar{}
+	}
+	return []corev1.EnvVar{
+		{Name: "AUTH_PUBLIC_KEY", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PubKey)},
+		{Name: "AUTH_PRIVATE_KEY", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PriKey)},
+	}
+}
+
 func MakeService(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.Service {
 	svcRef := GetServiceKey(cr.ObjKey())
 	svcLabel := GetXlineInstanceLabels(cr.ObjKey())
@@ -82,9 +114,21 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 		"--storage-engine", "rocksdb",
 		"--data-dir", DataDir,
 	}
-
 	initCmd = append(initCmd, cr.Spec.BootArgs()...)
 
+	envs := []corev1.EnvVar{
+		{Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))},
+		{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{
+			FieldRef: &corev1.ObjectFieldSelector{
+				FieldPath: "metadata.name",
+			},
+		}},
+	}
+	envs = append(envs, GetAuthSecretEnvVars(cr.Spec.AuthSecrets)...)
+
+	volumes := GetAuthSecretVolume(cr.Spec.AuthSecrets)
+	volumeMount := GetAuthSecretVolumeMount(cr.Spec.AuthSecrets)
+
 	// pod template: main container
 	mainContainer := corev1.Container{
 		Name:            "xline",
@@ -93,15 +137,9 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 		Ports: []corev1.ContainerPort{
 			{Name: "xline-port", ContainerPort: XlinePort},
 		},
-		Command: initCmd,
-		Env: []corev1.EnvVar{
-			{Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))},
-			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{
-				FieldRef: &corev1.ObjectFieldSelector{
-					FieldPath: "metadata.name",
-				},
-			}},
-		},
+		Command:      initCmd,
+		Env:          envs,
+		VolumeMounts: volumeMount,
 	}
 
 	// pod template
@@ -110,6 +148,7 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 			Labels: stsLabels,
 		},
 		Spec: corev1.PodSpec{
+			Volumes:    volumes,
 			Containers: []corev1.Container{mainContainer},
 		},
 	}
diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh
index 0695d42f..258bc010 100644
--- a/tests/e2e/cases/ci.sh
+++ b/tests/e2e/cases/ci.sh
@@ -6,6 +6,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh"
 _TEST_CI_CLUSTER_NAME="my-xline-cluster"
 _TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts"
 _TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc"
+_TEST_CI_SECRET_NAME="auth-cred"
 _TEST_CI_NAMESPACE="default"
 _TEST_CI_DNS_SUFFIX="svc.cluster.local"
 _TEST_CI_XLINE_PORT="2379"
@@ -33,6 +34,62 @@ function test::ci::_etcdctl_expect() {
   fi
 }
 
+# run a command with expect output, based on key word match
+# args:
+#   $1: endpoints
+#   $2: command to run
+#   $3: key word to match
+function test::ci::_etcdctl_match() {
+    log::debug "run command: etcdctl --endpoints=$1 $2"
+    got=$(testenv::util::etcdctl --endpoints="$1" "$2")
+    expect=$(echo -e ${3})
+    if echo "${got}" | grep -q "${expect}"; then
+      log::info  "command run success"
+    else
+      log::error  "command run failed"
+      log::error  "expect: ${expect}"
+      log::error "got: $got"
+      return 1
+    fi
+}
+
+function test::ci::_auth_validation() {
+  log::info "auth validation test running..."
+  endpoints=$(test::ci::_mk_endpoints 3)
+  test::ci::_etcdctl_expect "$endpoints" "user add root:root" "User root created" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "role add root" "Role root created" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "user grant-role root root" "Role root is granted to user root" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user list" "etcdserver: authentication is not enabled" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "auth enable" "Authentication Enabled" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user add u:u" "User u created" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user u:u user add f:f" "etcdserver: permission denied" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role add r" "Role r created" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user grant-role u r" "Role r is granted to user u" || return $?
+  test::ci::_etcdctl_expect "$endpoints""--user root:root role grant-permission r readwrite key1" "Role r updated" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u put key1 value1" "OK" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u get key1" "key1\nvalue1" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u role get r" "Role r\nKV Read:\n\tkey1\nKV Write:\n\tkey1" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u user get u" "User: u\nRoles: r" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "echo 'new_password' | --user root:root user passwd --interactive=false u" "Password updated" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role revoke-permission r key1" "Permission of key key1 is revoked from role r" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user revoke-role u r" "Role r is revoked from user u" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user list" "root\nu" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role list" "r\nroot" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user delete u" "User u deleted" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role delete r" "Role r deleted" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user get non_exist_user" "etcdserver: user name not found" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user add root:root" "etcdserver: user name already exists" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root role get non_exist_role" "etcdserver: role name not found" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root role add root" "etcdserver: role name already exists" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user delete root" "etcdserver: invalid auth management" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root auth disable" "Authentication Disabled" || return $?
+  log::info "auth validation test passed"
+}
+
 function test::ci::_install_CRD() {
     make install
     if [ $? -eq 0 ]; then
@@ -67,6 +124,9 @@ function test::ci::_start() {
   make run  >/dev/null 2>&1 &
   log::info "controller started"
   popd
+  log::info "create xline auth key pairs"
+  k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1
+  k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME
   log::info "starting xline cluster"
   k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1
   k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
@@ -115,6 +175,7 @@ function test::run::ci::basic_validation() {
   endpoints=$(test::ci::_mk_endpoints 1)
   test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $?
   test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $?
+  test::ci::_auth_validation
   test::ci::_teardown
 }
 
diff --git a/tests/e2e/cases/manifests/auth-cred.yaml b/tests/e2e/cases/manifests/auth-cred.yaml
new file mode 100644
index 00000000..8501b674
--- /dev/null
+++ b/tests/e2e/cases/manifests/auth-cred.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: auth-cred
+data:
+  auth-jwt.pri: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQ25BeHhTWEpZV1pDS3IKNmY2ajBIUlV3a2hYLzArR1hqRWNsV29MQTUrS1pBdVdNU3U4Yno2WCtJU2N2NHZOd09SbEdTV09ucnorOG1iMgpJMEY2dGVWWldmV0Zxc255V2s3SXhNK2g5eVRnN2FZLzg2ODVZZldUTDdmcFdxMS8zRm5pejRRYnNZRnV6QjFWCmdhWjVmRDJDU1lJS3pTRCtxVlNsWEYyNUpERkhWN2IyT2RIclgwVUtaT1RXWS9WRS8vU1R0K1BKS2RYOVIzcGwKa0d3QXpKSWtrY0FaeTB2aHZxVDNBU1RnWGNoTmVOOHdHWVliM1lpcmtxSXNRQjVYY3MxUjFXK3l6K0lyVmE2LwowV01jeUU2cXRKUFowbHZpeVQwbkhWL3Baalh1RDRCMGFqYS8xZmsvSG1YRFBNanBLMUJ1Q0JUU3RNL0tsY3JBCm9BeG8rWURoQWdNQkFBRUNnZ0VBSXlKaFkrWThZTXVDQzc1M0pra2xIK3ViUW4vZ1gva1N4ZHVjNm1KQnZ1QmIKRzZhT2Q5N0RRVDh6enJIeEhFRFhDM21sMEFJTzZtZGVSNnVWQzlhV1FCelByT1lJQStjQnFmVFZaVkpUdk1uaAo3cFE2S1kwMUYxaXpqUERaalF0ekVXYnNlTkwzMHJJMy9aUC96SkRaYzc0NUVFS2xEVTNjRThtQm9nQStLYTZ3CkdMb3pUOXFRZjhrbkJydHp4SDZTdnJacGZhUmxQOTVpczgyYjRJdVBocVlkRzdkVllGVEFMRTFNeVZyQ2JTNFkKS3l0ak5MZ3dwMWJJUXRXcnpNZWJCR29pVStEdkRjUlk4enZPZkZ1cER3cFlDdDNwMWFVNXd5WVlkcjc0ZXNWNwpqanFIajg5VWE2NUpISjNYbk1BYU1jNGRITTJGc0dxTXNPdi9EREtJblFLQmdRRGF3Y2tRRWVreDBRdVAzZUpQCkdXZFo4N29jK0ZWakRlM2JZaEFuQ2YveVhSSm9xY3M1dnIxbTF5Q1hGZnNqYlFGWUhXWFI5QVV0Tm41SEN3T1oKem9UMU12OTZmWEJWR1FPUmd6dmxVV1M0M3VLcGZJUERWdjJJNlpjS1NJUUFHT2djV1l2bUJEaFlxUEhnbXgzbwpWU3JOR1d0TGR5dzNyRDFKNk8rMVJ3dGJpd0tCZ1FERGNobVk1OUVYQmlUdmx5VDNRamwwdlpGTUhhK1RFbGJoCmlrTnRZbHRiVUh0YW1PWFp6cGRrL0tBN1gyZFlpMFFwVmZiYnBmUC9seTVsWXZnWndsOGg5ME9ib3BydStBQ00KbmRsS0JmTlFZQXJtV1k2YkoyQ3dGN2oxYVRDQ0hadVZ1WDYvcHpGVlN0UmNzc24xNXVvVmFJeUtkL01oSnpMRgpTM2VydFFrU3d3S0JnQW5pTVlSaFdzamVhZ2hRL1JXWHp6eVlMM041b05uOTJoNU1XdkI0bWpESUZiblcyaEM4CjFtL2NEbVBsSVZpalp5a2xBdUd1aGNGYU1mQmh4Z0xmK3MvZFF2KzB4U3VER3M4clA3eUhwZVpZWTZOR3RlbFEKZDlvRXU4ZENLWHlibzNrTWJxNnd5Qjd4V3lSTHZka3VaK1dtWFZ1bWdiL3VMMEswbklmek1zY3JBb0dBZUExZQpLODQ1WVNzbEJRYVNiazcvZS9YMWlndXlEV1QyZVJPMDF6dlRZZ1BOd1ppcGwyQ1BIamtQTTJrbTBmeTVvYXBzCk4vOTRJVWQ3K0VzU21zQUtMNUx5dEdidFJGeVIrYzM3NnJ3OCtPSUZ6L2l5NEJzUUNScUpRaldhMWxIWmY5NngKUElnMmhXMnhoRDlPVHYzSVM5NHNkZUc0Tm1VZGlwTVFyeWhFcW9FQ2dZRUFrdlhPZzY2SUFWVHJPNnFnb3lsNQo0Mm91ZmEvUUUrcU9BWW9RRXBteDNTWng2dE1reWNmQVFxVUhZY1hoVzFITmp5R2JiZy9zbDEzeWRkblBRcWlnCitPYnRRTlNJcUdaV0NjL0hJcU0vL3BQSTNNSFBoV0FSTU9tQWJrMEkxbVQwUUtodUZmU3VnVjJ4YjFEai9SdmYKMFZkQjh0eFkrNVd6NnpQMUYyZzQ2Z009Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
+  auth-jwt.pub: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFwd01jVWx5V0ZtUWlxK24rbzlCMApWTUpJVi85UGhsNHhISlZxQ3dPZmltUUxsakVydkc4K2wvaUVuTCtMemNEa1pSa2xqcDY4L3ZKbTlpTkJlclhsCldWbjFoYXJKOGxwT3lNVFBvZmNrNE8ybVAvT3ZPV0gxa3krMzZWcXRmOXhaNHMrRUc3R0Jic3dkVllHbWVYdzkKZ2ttQ0NzMGcvcWxVcFZ4ZHVTUXhSMWUyOWpuUjYxOUZDbVRrMW1QMVJQLzBrN2ZqeVNuVi9VZDZaWkJzQU15UwpKSkhBR2N0TDRiNms5d0VrNEYzSVRYamZNQm1HRzkySXE1S2lMRUFlVjNMTlVkVnZzcy9pSzFXdXY5RmpITWhPCnFyU1QyZEpiNHNrOUp4MWY2V1kxN2crQWRHbzJ2OVg1UHg1bHd6ekk2U3RRYmdnVTByVFB5cFhLd0tBTWFQbUEKNFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==
diff --git a/tests/e2e/cases/manifests/cluster.yaml b/tests/e2e/cases/manifests/cluster.yaml
index b276b307..4d3541af 100644
--- a/tests/e2e/cases/manifests/cluster.yaml
+++ b/tests/e2e/cases/manifests/cluster.yaml
@@ -6,3 +6,11 @@ spec:
   image: phoenix500526/xline:v0.6.1
   imagePullPolicy: IfNotPresent
   replicas: 3
+  authSecret:
+    name: auth-cred
+    mountPath: /tmp/auth-cred
+    pubKey: auth-jwt.pub
+    priKey: auth-jwt.pri
+  bootstrapArgs:
+    auth-private-key: /tmp/auth-cred/auth-jwt.pri
+    auth-public-key: /tmp/auth-cred/auth-jwt.pub

From 5387bc7cffc349e2a827c321ec96fb02aa8ad45e Mon Sep 17 00:00:00 2001
From: Phoeniix Zhao <Phoenix500526@163.com>
Date: Sat, 23 Dec 2023 22:43:51 +0800
Subject: [PATCH 2/5] feat: support pvc for xline

Signed-off-by: Phoeniix Zhao <Phoenix500526@163.com>
---
 api/v1alpha1/xlinecluster_types.go            |  8 ++++
 api/v1alpha1/zz_generated.deepcopy.go         |  6 +++
 .../xline.io.datenlord.com_xlineclusters.yaml | 46 +++++++++++++++++++
 internal/transformer/xlinecluster_resource.go | 19 +++++---
 internal/util/kubeutil.go                     | 19 ++++++++
 5 files changed, 92 insertions(+), 6 deletions(-)

diff --git a/api/v1alpha1/xlinecluster_types.go b/api/v1alpha1/xlinecluster_types.go
index b2f4c657..dcdeb629 100644
--- a/api/v1alpha1/xlinecluster_types.go
+++ b/api/v1alpha1/xlinecluster_types.go
@@ -136,6 +136,14 @@ type XlineClusterSpec struct {
 
 	// The auth secret keys
 	AuthSecrets *XlineAuthSecret `json:"authSecret,omitempty"`
+
+	// K8s storage-class-name of the Xline storage
+	// Defaults to Kubernetes default storage class.
+	// +optional
+	StorageClassName *string `json:"storageClassName"`
+
+	// Defines the specification of resource cpu, mem, storage.
+	corev1.ResourceRequirements `json:",inline"`
 }
 
 type XlineAuthSecret struct {
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 72d0388b..7252e8c1 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -299,6 +299,12 @@ func (in *XlineClusterSpec) DeepCopyInto(out *XlineClusterSpec) {
 		*out = new(XlineAuthSecret)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.StorageClassName != nil {
+		in, out := &in.StorageClassName, &out.StorageClassName
+		*out = new(string)
+		**out = **in
+	}
+	in.ResourceRequirements.DeepCopyInto(&out.ResourceRequirements)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineClusterSpec.
diff --git a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml
index 3fc8d020..c80a4320 100644
--- a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml
+++ b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml
@@ -155,17 +155,63 @@ spec:
                     pattern: \d+(us|ms|s|m|h|d)
                     type: string
                 type: object
+              claims:
+                description: "Claims lists the names of resources, defined in spec.resourceClaims,
+                  that are used by this container. \n This is an alpha field and requires
+                  enabling the DynamicResourceAllocation feature gate. \n This field
+                  is immutable. It can only be set for containers."
+                items:
+                  description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                  properties:
+                    name:
+                      description: Name must match the name of one entry in pod.spec.resourceClaims
+                        of the Pod where this field is used. It makes that resource
+                        available inside a container.
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
               image:
                 description: Xline cluster image
                 type: string
               imagePullPolicy:
                 description: ImagePullPolicy of Xline cluster Pods
                 type: string
+              limits:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: 'Limits describes the maximum amount of compute resources
+                  allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                type: object
               replicas:
                 description: The replicas of xline nodes
                 format: int32
                 minimum: 3
                 type: integer
+              requests:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: 'Requests describes the minimum amount of compute resources
+                  required. If Requests is omitted for a container, it defaults to
+                  Limits if that is explicitly specified, otherwise to an implementation-defined
+                  value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                type: object
+              storageClassName:
+                description: K8s storage-class-name of the Xline storage Defaults
+                  to Kubernetes default storage class.
+                type: string
             required:
             - replicas
             type: object
diff --git a/internal/transformer/xlinecluster_resource.go b/internal/transformer/xlinecluster_resource.go
index bb9c3b2b..9616e1fa 100644
--- a/internal/transformer/xlinecluster_resource.go
+++ b/internal/transformer/xlinecluster_resource.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	xapi "github.com/xline-kv/xline-operator/api/v1alpha1"
+	"github.com/xline-kv/xline-operator/internal/util"
 	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -127,7 +128,12 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 	envs = append(envs, GetAuthSecretEnvVars(cr.Spec.AuthSecrets)...)
 
 	volumes := GetAuthSecretVolume(cr.Spec.AuthSecrets)
-	volumeMount := GetAuthSecretVolumeMount(cr.Spec.AuthSecrets)
+	volumeMounts := GetAuthSecretVolumeMount(cr.Spec.AuthSecrets)
+	volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "xline-storage", MountPath: "/usr/local/xline/data-dir"})
+
+	pvcTemplates := []corev1.PersistentVolumeClaim{
+		util.NewReadWriteOncePVC("xline-storage", cr.Spec.StorageClassName, cr.Spec.Requests.Storage()),
+	}
 
 	// pod template: main container
 	mainContainer := corev1.Container{
@@ -139,7 +145,7 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 		},
 		Command:      initCmd,
 		Env:          envs,
-		VolumeMounts: volumeMount,
+		VolumeMounts: volumeMounts,
 	}
 
 	// pod template
@@ -163,10 +169,11 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 			Labels:    stsLabels,
 		},
 		Spec: appv1.StatefulSetSpec{
-			Replicas:    &cr.Spec.Replicas,
-			ServiceName: svcName,
-			Selector:    &metav1.LabelSelector{MatchLabels: stsLabels},
-			Template:    podTemplate,
+			Replicas:             &cr.Spec.Replicas,
+			ServiceName:          svcName,
+			Selector:             &metav1.LabelSelector{MatchLabels: stsLabels},
+			VolumeClaimTemplates: pvcTemplates,
+			Template:             podTemplate,
 		},
 	}
 
diff --git a/internal/util/kubeutil.go b/internal/util/kubeutil.go
index fc4a04f1..58d87cce 100644
--- a/internal/util/kubeutil.go
+++ b/internal/util/kubeutil.go
@@ -5,6 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 )
 
@@ -34,3 +37,19 @@ func Md5HashOr(obj any, fallback string) string {
 	}
 	return hash
 }
+
+func NewReadWriteOncePVC(name string, storageClassName *string, storageRequest *resource.Quantity) corev1.PersistentVolumeClaim {
+	pvc := corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+			StorageClassName: storageClassName,
+		},
+	}
+	if storageRequest != nil {
+		pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest}
+	}
+	return pvc
+}

From e3243e42d352789b77205256343fe1fe2b086e9e Mon Sep 17 00:00:00 2001
From: Phoeniix Zhao <Phoenix500526@163.com>
Date: Sat, 23 Dec 2023 22:51:48 +0800
Subject: [PATCH 3/5] test: add e2e test for xline pvc

Signed-off-by: Phoeniix Zhao <Phoenix500526@163.com>
---
 tests/e2e/cases/ci.sh                      | 47 +++++++++++++++++++-
 tests/e2e/cases/manifests/cluster.yaml     |  3 ++
 tests/e2e/cases/manifests/e2e-storage.yaml | 51 ++++++++++++++++++++++
 3 files changed, 100 insertions(+), 1 deletion(-)
 create mode 100644 tests/e2e/cases/manifests/e2e-storage.yaml

diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh
index 258bc010..7ff47d96 100644
--- a/tests/e2e/cases/ci.sh
+++ b/tests/e2e/cases/ci.sh
@@ -10,6 +10,7 @@ _TEST_CI_SECRET_NAME="auth-cred"
 _TEST_CI_NAMESPACE="default"
 _TEST_CI_DNS_SUFFIX="svc.cluster.local"
 _TEST_CI_XLINE_PORT="2379"
+_TEST_CI_STORAGECLASS_NAME="e2e-storage"
 _TEST_CI_LOG_SYNC_TIMEOUT=60
 
 function test::ci::_mk_endpoints() {
@@ -117,6 +118,44 @@ function test::ci::wait_all_xline_pod_ready() {
   done
 }
 
+function test::ci::wait_all_xline_pod_deleted() {
+  for ((i = 0; i < $1; i++)); do
+    log::info "wait pod/${_TEST_CI_STS_NAME}-${i} to be ready"
+    if ! k8s::kubectl wait --for=delete pod/${_TEST_CI_STS_NAME}-${i} --timeout=300s; then
+      log::fatal "Failed to wait for util to be ready"
+    fi
+  done
+}
+
+function test::ci::_prepare_pv() {
+  log::info "create persistent volume and storage class"
+  mkdir -p /tmp/host-500m-pv1 /tmp/host-500m-pv2 /tmp/host-500m-pv3
+  k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/e2e-storage.yaml" >/dev/null 2>&1
+  k8s::kubectl::wait_resource_creation storageclass $_TEST_CI_STORAGECLASS_NAME
+  k8s::kubectl::wait_resource_creation pv "host-500m-pv1"
+  k8s::kubectl::wait_resource_creation pv "host-500m-pv2"
+  k8s::kubectl::wait_resource_creation pv "host-500m-pv3"
+}
+
+function test::ci::_clean_pvc() {
+  for ((i = 0; i < $1; i++)); do
+    local pvc_name="xline-storage-${_TEST_CI_STS_NAME}-${i}"
+    log::info "deleting pvc $pvc_name ..."
+    k8s::kubectl delete pvc $pvc_name >/dev/null 2>&1
+    if ! k8s::kubectl wait --for=delete pvc/${pvc_name} --timeout=300s; then
+      log::fatal "Failed to wait for pvc/${pvc_name} to be deleted"
+    fi
+  done
+}
+
+function test::ci::_clean_pv() {
+  log::info "delete persistent volume claim"
+  log::info "delete persistent volume and storage class"
+  k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/e2e-storage.yaml"
+  log::info "pv has been deleted"
+  rm -rf /tmp/host-500m-pv1 /tmp/host-500m-pv2 /tmp/host-500m-pv3
+}
+
 function test::ci::_start() {
   log::info "starting controller"
   pushd $(dirname "${BASH_SOURCE[0]}")/../../../
@@ -127,6 +166,7 @@ function test::ci::_start() {
   log::info "create xline auth key pairs"
   k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1
   k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME
+  test::ci::_prepare_pv
   log::info "starting xline cluster"
   k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1
   k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
@@ -136,10 +176,15 @@ function test::ci::_teardown() {
   log::info "stopping controller"
   pushd $(dirname "${BASH_SOURCE[0]}")/../../../
   test::ci::_uninstall_CRD
-  controller_pid=$(ps aux | grep "[g]o run ./cmd/main.go" | awk '{print $2}')
+  controller_pid=$(lsof -i:8081 | awk 'NR==2 {print $2}')
   if [ -n "$controller_pid" ]; then
     kill -9 $controller_pid
   fi
+  k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1
+  test::ci::wait_all_xline_pod_deleted 3
+  test::ci::_clean_pvc 3
+  test::ci::_clean_pv
+  k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1
 }
 
 function test::ci::_chaos() {
diff --git a/tests/e2e/cases/manifests/cluster.yaml b/tests/e2e/cases/manifests/cluster.yaml
index 4d3541af..95cb9bef 100644
--- a/tests/e2e/cases/manifests/cluster.yaml
+++ b/tests/e2e/cases/manifests/cluster.yaml
@@ -6,6 +6,9 @@ spec:
   image: phoenix500526/xline:v0.6.1
   imagePullPolicy: IfNotPresent
   replicas: 3
+  storageClassName: "e2e-storage"
+  requests:
+    storage: 500Mi
   authSecret:
     name: auth-cred
     mountPath: /tmp/auth-cred
diff --git a/tests/e2e/cases/manifests/e2e-storage.yaml b/tests/e2e/cases/manifests/e2e-storage.yaml
new file mode 100644
index 00000000..09a4dd15
--- /dev/null
+++ b/tests/e2e/cases/manifests/e2e-storage.yaml
@@ -0,0 +1,51 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: "e2e-storage"
+provisioner: "kubernetes.io/no-provisioner"
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: host-500m-pv1
+
+spec:
+  storageClassName: e2e-storage
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 500Mi
+  hostPath:
+    path: /tmp/host-500m-pv1/
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: host-500m-pv2
+
+spec:
+  storageClassName: e2e-storage
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 500Mi
+  hostPath:
+    path: /tmp/host-500m-pv2/
+---
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: host-500m-pv3
+
+spec:
+  storageClassName: e2e-storage
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 500Mi
+  hostPath:
+    path: /tmp/host-500m-pv3/
+---

From 275ba2f3b9eddd7a783c88e6c45d7b2264171acf Mon Sep 17 00:00:00 2001
From: Phoeniix Zhao <Phoenix500526@163.com>
Date: Thu, 25 Jan 2024 21:08:37 +0800
Subject: [PATCH 4/5] fix: fix e2e test will fail sometime

Signed-off-by: Phoeniix Zhao <Phoenix500526@163.com>
---
 tests/e2e/cases/ci.sh | 55 +++++++++++++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 18 deletions(-)

diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh
index 7ff47d96..ff92685e 100644
--- a/tests/e2e/cases/ci.sh
+++ b/tests/e2e/cases/ci.sh
@@ -11,7 +11,7 @@ _TEST_CI_NAMESPACE="default"
 _TEST_CI_DNS_SUFFIX="svc.cluster.local"
 _TEST_CI_XLINE_PORT="2379"
 _TEST_CI_STORAGECLASS_NAME="e2e-storage"
-_TEST_CI_LOG_SYNC_TIMEOUT=60
+_TEST_CI_LOG_SYNC_TIMEOUT=30
 
 function test::ci::_mk_endpoints() {
   local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}"
@@ -22,13 +22,13 @@ function test::ci::_mk_endpoints() {
 }
 
 function test::ci::_etcdctl_expect() {
-  log::debug "run command: etcdctl --endpoints=$1 $2"
+  log::info "run command: etcdctl --endpoints=$1 $2"
   got=$(testenv::util::etcdctl --endpoints="$1" "$2")
   expect=$(echo -e "$3")
   if [ "${got//$'\r'/}" == "$expect" ]; then
-    log::info "command run success"
+    log::info "command $2 run success"
   else
-    log::error "command run failed"
+    log::error "command $2 run failed"
     log::error "expect: $expect"
     log::error "got: $got"
     return 1
@@ -92,7 +92,9 @@ function test::ci::_auth_validation() {
 }
 
 function test::ci::_install_CRD() {
+    pushd $(dirname "${BASH_SOURCE[0]}")/../../../
     make install
+    popd
     if [ $? -eq 0 ]; then
         log::info "make install: create custom resource definition succeeded"
     else
@@ -101,7 +103,9 @@ function test::ci::_install_CRD() {
 }
 
 function test::ci::_uninstall_CRD() {
+    pushd $(dirname "${BASH_SOURCE[0]}")/../../../
     make uninstall
+    popd
     if [ $? -eq 0 ]; then
         log::info "make uninstall: remove custom resource definition succeeded"
     else
@@ -157,12 +161,9 @@ function test::ci::_clean_pv() {
 }
 
 function test::ci::_start() {
-  log::info "starting controller"
-  pushd $(dirname "${BASH_SOURCE[0]}")/../../../
+  test::ci::stop_controller
   test::ci::_install_CRD
-  make run  >/dev/null 2>&1 &
-  log::info "controller started"
-  popd
+  test::ci::start_controller
   log::info "create xline auth key pairs"
   k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1
   k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME
@@ -172,14 +173,31 @@ function test::ci::_start() {
   k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
 }
 
-function test::ci::_teardown() {
-  log::info "stopping controller"
-  pushd $(dirname "${BASH_SOURCE[0]}")/../../../
-  test::ci::_uninstall_CRD
-  controller_pid=$(lsof -i:8081 | awk 'NR==2 {print $2}')
+function test::ci::start_controller() {
+    log::info "starting controller"
+    pushd $(dirname "${BASH_SOURCE[0]}")/../../../
+    make run >/dev/null 2>&1 &
+    popd
+    sleep 5
+    if kill -0 $!; then
+      log::info "Controller started"
+    else
+      log::error "Controller cannot failed"
+    fi
+}
+
+function test::ci::stop_controller() {
+  controller_pid=$(lsof -i:8081 | grep main | awk '{print $2}')
   if [ -n "$controller_pid" ]; then
     kill -9 $controller_pid
   fi
+}
+
+
+function test::ci::_teardown() {
+  log::info "stopping controller"
+  test::ci::_uninstall_CRD
+  test::ci::stop_controller
   k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1
   test::ci::wait_all_xline_pod_deleted 3
   test::ci::_clean_pvc 3
@@ -190,16 +208,17 @@ function test::ci::_teardown() {
 function test::ci::_chaos() {
   size=$1
   iters=$2
-  max_kill=$((size / 2))
-  log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill"
+  majority=$((size / 2 + 1))
+  fault_tolerance=$((size - majority))
+  log::info "chaos: size=$size, iters=$iters, fault_tolerance=$fault_tolerance"
   for ((i = 0; i < iters; i++)); do
     log::info "chaos: iter=$i"
     endpoints=$(test::ci::_mk_endpoints size)
     test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $?
     test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $?
-    kill=$((RANDOM % max_kill))
+    kill=$((RANDOM % fault_tolerance + 1))
     log::info "chaos: kill=$kill"
-    for ((j = 0; j < kill; j++)); do
+    for ((j = 0; j < $kill; j++)); do
       pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))"
       log::info "chaos: kill pod=$pod"
       k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null

From 5ba4d5e926e5653dc819668088faa4b3b0419ed4 Mon Sep 17 00:00:00 2001
From: Phoeniix Zhao <Phoenix500526@163.com>
Date: Sat, 27 Jan 2024 17:53:43 +0800
Subject: [PATCH 5/5] fix: fix basic_validation test failed problem

Signed-off-by: Phoeniix Zhao <Phoenix500526@163.com>
---
 tests/e2e/cases/ci.sh          | 120 +++++++++++++--------------------
 tests/e2e/testenv/util/util.sh |  48 +++++++++++--
 2 files changed, 87 insertions(+), 81 deletions(-)

diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh
index ff92685e..4fe44633 100644
--- a/tests/e2e/cases/ci.sh
+++ b/tests/e2e/cases/ci.sh
@@ -21,73 +21,42 @@ function test::ci::_mk_endpoints() {
   echo "$endpoints"
 }
 
-function test::ci::_etcdctl_expect() {
-  log::info "run command: etcdctl --endpoints=$1 $2"
-  got=$(testenv::util::etcdctl --endpoints="$1" "$2")
-  expect=$(echo -e "$3")
-  if [ "${got//$'\r'/}" == "$expect" ]; then
-    log::info "command $2 run success"
-  else
-    log::error "command $2 run failed"
-    log::error "expect: $expect"
-    log::error "got: $got"
-    return 1
-  fi
-}
-
-# run a command with expect output, based on key word match
-# args:
-#   $1: endpoints
-#   $2: command to run
-#   $3: key word to match
-function test::ci::_etcdctl_match() {
-    log::debug "run command: etcdctl --endpoints=$1 $2"
-    got=$(testenv::util::etcdctl --endpoints="$1" "$2")
-    expect=$(echo -e ${3})
-    if echo "${got}" | grep -q "${expect}"; then
-      log::info  "command run success"
-    else
-      log::error  "command run failed"
-      log::error  "expect: ${expect}"
-      log::error "got: $got"
-      return 1
-    fi
-}
-
 function test::ci::_auth_validation() {
   log::info "auth validation test running..."
   endpoints=$(test::ci::_mk_endpoints 3)
-  test::ci::_etcdctl_expect "$endpoints" "user add root:root" "User root created" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "role add root" "Role root created" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "user grant-role root root" "Role root is granted to user root" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root user list" "etcdserver: authentication is not enabled" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "auth enable" "Authentication Enabled" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root user add u:u" "User u created" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user u:u user add f:f" "etcdserver: permission denied" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root role add r" "Role r created" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root user grant-role u r" "Role r is granted to user u" || return $?
-  test::ci::_etcdctl_expect "$endpoints""--user root:root role grant-permission r readwrite key1" "Role r updated" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user u:u put key1 value1" "OK" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user u:u get key1" "key1\nvalue1" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user u:u role get r" "Role r\nKV Read:\n\tkey1\nKV Write:\n\tkey1" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user u:u user get u" "User: u\nRoles: r" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "echo 'new_password' | --user root:root user passwd --interactive=false u" "Password updated" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root role revoke-permission r key1" "Permission of key key1 is revoked from role r" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root user revoke-role u r" "Role r is revoked from user u" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root user list" "root\nu" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root role list" "r\nroot" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root user delete u" "User u deleted" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root role delete r" "Role r deleted" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root user get non_exist_user" "etcdserver: user name not found" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root user add root:root" "etcdserver: user name already exists" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root role get non_exist_role" "etcdserver: role name not found" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root role add root" "etcdserver: role name already exists" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $?
-  test::ci::_etcdctl_match "$endpoints" "--user root:root user delete root" "etcdserver: invalid auth management" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "--user root:root auth disable" "Authentication Disabled" || return $?
+  ETCDCTL=$(testenv::util::etcdctl $endpoints)
+
+  testenv::util::run_with_expect "${ETCDCTL} user add root:root" "User root created" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} role add root" "Role root created" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} user grant-role root root" "Role root is granted to user root" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root user list" "etcdserver: authentication is not enabled" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} auth enable" "Authentication Enabled" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root user add u:u" "User u created" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user u:u user add f:f" "etcdserver: permission denied" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root role add r" "Role r created" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root user grant-role u r" "Role r is granted to user u" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root role grant-permission r readwrite key1" "Role r updated" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user u:u put key1 value1" "OK" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user u:u get key1" "key1\nvalue1" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user u:u role get r" "Role r\nKV Read:\n\tkey1\nKV Write:\n\tkey1" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user u:u user get u" "User: u\nRoles: r" || return $?
+  testenv::util::run_with_expect "echo 'new_password' | ${ETCDCTL} --user root:root user passwd --interactive=false u" "Password updated" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root role revoke-permission r key1" "Permission of key key1 is revoked from role r" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root user revoke-role u r" "Role r is revoked from user u" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root user list" "root\nu" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root role list" "r\nroot" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root user delete u" "User u deleted" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root role delete r" "Role r deleted" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root user get non_exist_user" "etcdserver: user name not found" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root user add root:root" "etcdserver: user name already exists" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root role get non_exist_role" "etcdserver: role name not found" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root role add root" "etcdserver: role name already exists" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $?
+  testenv::util::run_with_match "${ETCDCTL} --user root:root user delete root" "etcdserver: invalid auth management" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} --user root:root auth disable" "Authentication Disabled" || return $?
   log::info "auth validation test passed"
 }
 
@@ -210,12 +179,13 @@ function test::ci::_chaos() {
   iters=$2
   majority=$((size / 2 + 1))
   fault_tolerance=$((size - majority))
+  endpoints=$(test::ci::_mk_endpoints $size)
+  ETCDCTL=$(testenv::util::etcdctl $endpoints)
   log::info "chaos: size=$size, iters=$iters, fault_tolerance=$fault_tolerance"
-  for ((i = 0; i < iters; i++)); do
+  for ((i = 0; i < $iters; i++)); do
     log::info "chaos: iter=$i"
-    endpoints=$(test::ci::_mk_endpoints size)
-    test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $?
-    test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $?
+    testenv::util::run_with_expect "${ETCDCTL} put A $i" "OK" || return $?
+    testenv::util::run_with_expect "${ETCDCTL} get A" "A\n$i" || return $?
     kill=$((RANDOM % fault_tolerance + 1))
     log::info "chaos: kill=$kill"
     for ((j = 0; j < $kill; j++)); do
@@ -223,8 +193,8 @@ function test::ci::_chaos() {
       log::info "chaos: kill pod=$pod"
       k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null
     done
-    test::ci::_etcdctl_expect "$endpoints" "put B $i" "OK" || return $?
-    test::ci::_etcdctl_expect "$endpoints" "get B" "B\n$i" || return $?
+    testenv::util::run_with_expect "${ETCDCTL} put B $i" "OK" || return $?
+    testenv::util::run_with_expect "${ETCDCTL} get B" "B\n$i" || return $?
     k8s::kubectl wait --for=jsonpath='{.status.readyReplicas}'="$size" sts/$_TEST_CI_CLUSTER_NAME --timeout=300s >/dev/null 2>&1
     log::info "wait for log synchronization" && sleep $_TEST_CI_LOG_SYNC_TIMEOUT
   done
@@ -234,11 +204,13 @@ function test::run::ci::basic_validation() {
   test::ci::_start
   test::ci::wait_all_xline_pod_ready 3
   endpoints=$(test::ci::_mk_endpoints 3)
-  test::ci::_etcdctl_expect "$endpoints" "put A 1" "OK" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "get A" "A\n1" || return $?
+  ETCDCTL=$(testenv::util::etcdctl $endpoints)
+  testenv::util::run_with_expect "${ETCDCTL} put A 1" "OK" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} get A" "A\n1" || return $?
   endpoints=$(test::ci::_mk_endpoints 1)
-  test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $?
-  test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $?
+  ETCDCTL=$(testenv::util::etcdctl $endpoints)
+  testenv::util::run_with_expect "${ETCDCTL} put A 2" "OK" || return $?
+  testenv::util::run_with_expect "${ETCDCTL} get A" "A\n2" || return $?
   test::ci::_auth_validation
   test::ci::_teardown
 }
diff --git a/tests/e2e/testenv/util/util.sh b/tests/e2e/testenv/util/util.sh
index dfaf939e..aed7f481 100644
--- a/tests/e2e/testenv/util/util.sh
+++ b/tests/e2e/testenv/util/util.sh
@@ -46,15 +46,49 @@ function testenv::util::uninstall() {
 }
 
 function testenv::util::etcdctl() {
-  # shellcheck disable=SC2034
-  local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}"
+  echo -e "kubectl exec -n util -i etcdctl -- env ETCDCTL_API=3 etcdctl --endpoints=$1"
+}
+
+function testenv::util::run_with_expect() {
+  cmd="$1"
+  expect=$(echo -e ${2})
+  # retry to avoid mysterious "Error from server: error dialing backend: EOF" error
+  for ((k = 0; k < ${RETRY_TIMES:-10}; k++)); do
+    output=$(eval ${cmd} 2>&1)
+    if [[ $output == *"timed out"* || $output == *"Request timeout"* || $output == *"context deadline exceeded"* ]]; then
+      sleep "${RETRY_INTERVAL:-3}"
+    elif [ "${output//$'\r'/}" == "$expect" ]; then
+      log::info "command $cmd run success"
+      return 0
+    else
+      log::error "command $cmd run failed"
+      log::error "expect: $expect"
+      log::error "got: $output"
+      return 1
+    fi
+  done
+}
 
+# run a command with expect output, based on key word match
+# args:
+#   $1: command to run
+#   $2: key word to match
+function testenv::util::run_with_match() {
+  cmd="$1"
+  expect=$(echo -e ${2})
   # retry to avoid mysterious "Error from server: error dialing backend: EOF" error
-  for ((i = 0; i < ${RETRY_TIMES:-10}; i++)); do
-    if output=$(k8s::kubectl exec -i etcdctl -- env ETCDCTL_API=3 etcdctl $@ 2>&1); then
-      echo -e "$output"
-      return
+  for ((n = 0; n < ${RETRY_TIMES:-10}; n++)); do
+    output=$(eval ${cmd} 2>&1)
+    if [[ $output == *"timed out"* || $output == *"Request timeout"* || $output == *"context deadline exceeded"* ]]; then
+      sleep "${RETRY_INTERVAL:-3}"
+    elif echo "${output}" | grep -q "${expect}"; then
+      log::info "command $cmd run success"
+      return 0
+    else
+      log::error "command $cmd run failed"
+      log::error "expect: $expect"
+      log::error "got: $output"
+      return 1
     fi
-    sleep "${RETRY_INTERVAL:-3}"
   done
 }