feat: use secret to specify xline auth key pair

Signed-off-by: Phoeniix Zhao <[email protected]>

api/v1alpha1/xlinecluster_types.go

@@ -133,6 +133,16 @@ type XlineClusterSpec struct {
 	// The replicas of xline nodes
 	// +kubebuilder:validation:Minimum=3
 	Replicas int32 `json:"replicas"`
+
+	// The auth secret keys
+	AuthSecrets *XlineAuthSecret `json:"authSecret,omitempty"`
+}
+
+type XlineAuthSecret struct {
+	Name      *string `json:"name"`
+	MountPath *string `json:"mountPath"`
+	PubKey    *string `json:"pubKey"`
+	PriKey    *string `json:"priKey"`
 }
 
 func (s *XlineClusterSpec) BootArgs() []string {

config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml

@@ -35,6 +35,23 @@ spec:
           spec:
             description: XlineClusterSpec defines the desired state of XlineCluster
             properties:
+              authSecret:
+                description: The auth secret keys
+                properties:
+                  mountPath:
+                    type: string
+                  name:
+                    type: string
+                  priKey:
+                    type: string
+                  pubKey:
+                    type: string
+                required:
+                - mountPath
+                - name
+                - priKey
+                - pubKey
+                type: object
               bootstrapArgs:
                 description: / Xline container bootstrap arguments / Set additional
                   arguments except [`--name`, `--members`, `--storage-engine`, `--data-dir`]

internal/transformer/xlinecluster_resource.go

@@ -46,6 +46,38 @@ func GetMemberTopology(stsRef types.NamespacedName, svcName string, replicas int
 	return strings.Join(members, ",")
 }
 
+func GetAuthSecretVolume(auth_sec *xapi.XlineAuthSecret) []corev1.Volume {
+	if auth_sec == nil {
+		return []corev1.Volume{}
+	}
+	return []corev1.Volume{
+		{Name: "auth-cred", VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: *auth_sec.Name,
+			},
+		}},
+	}
+}
+
+func GetAuthSecretVolumeMount(auth_sec *xapi.XlineAuthSecret) []corev1.VolumeMount {
+	if auth_sec == nil {
+		return []corev1.VolumeMount{}
+	}
+	return []corev1.VolumeMount{
+		{Name: "auth-cred", ReadOnly: true, MountPath: *auth_sec.MountPath},
+	}
+}
+
+func GetAuthSecretEnvVars(auth_sec *xapi.XlineAuthSecret) []corev1.EnvVar {
+	if auth_sec == nil {
+		return []corev1.EnvVar{}
+	}
+	return []corev1.EnvVar{
+		{Name: "AUTH_PUBLIC_KEY", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PubKey)},
+		{Name: "AUTH_PRIVATE_KEY", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PriKey)},
+	}
+}
+
 func MakeService(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.Service {
 	svcRef := GetServiceKey(cr.ObjKey())
 	svcLabel := GetXlineInstanceLabels(cr.ObjKey())
@@ -83,7 +115,19 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 		"--data-dir", DataDir,
 	}
 
+	envs := []corev1.EnvVar{
+		{Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))},
+		{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{
+			FieldRef: &corev1.ObjectFieldSelector{
+				FieldPath: "metadata.name",
+			},
+		}},
+	}
+	envs = append(envs, GetAuthSecretEnvVars(cr.Spec.AuthSecrets)...)
+
 	initCmd = append(initCmd, cr.Spec.BootArgs()...)
+	volumes := GetAuthSecretVolume(cr.Spec.AuthSecrets)
+	volumeMount := GetAuthSecretVolumeMount(cr.Spec.AuthSecrets)
 
 	// pod template: main container
 	mainContainer := corev1.Container{
@@ -93,15 +137,9 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 		Ports: []corev1.ContainerPort{
 			{Name: "xline-port", ContainerPort: XlinePort},
 		},
-		Command: initCmd,
-		Env: []corev1.EnvVar{
-			{Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))},
-			{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{
-				FieldRef: &corev1.ObjectFieldSelector{
-					FieldPath: "metadata.name",
-				},
-			}},
-		},
+		Command:      initCmd,
+		Env:          envs,
+		VolumeMounts: volumeMount,
 	}
 
 	// pod template
@@ -110,6 +148,7 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State
 			Labels: stsLabels,
 		},
 		Spec: corev1.PodSpec{
+			Volumes:    volumes,
 			Containers: []corev1.Container{mainContainer},
 		},
 	}

tests/e2e/cases/ci.sh

@@ -6,6 +6,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh"
 _TEST_CI_CLUSTER_NAME="my-xline-cluster"
 _TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts"
 _TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc"
+_TEST_CI_SECRET_NAME="auth-cred"
 _TEST_CI_NAMESPACE="default"
 _TEST_CI_DNS_SUFFIX="svc.cluster.local"
 _TEST_CI_XLINE_PORT="2379"
@@ -33,6 +34,62 @@ function test::ci::_etcdctl_expect() {
   fi
 }
 
+# run a command with expect output, based on key word match
+# args:
+#   $1: endpoints
+#   $2: command to run
+#   $3: key word to match
+function test::ci::_etcdctl_match() {
+    log::debug "run command: etcdctl --endpoints=$1 $2"
+    got=$(testenv::util::etcdctl --endpoints="$1" "$2")
+    expect=$(echo -e ${3})
+    if echo "${got}" | grep -q "${expect}"; then
+      log::info  "command run success"
+    else
+      log::error  "command run failed"
+      log::error  "expect: ${expect}"
+      log::error "got: $got"
+      return 1
+    fi
+}
+
+function test::ci::_auth_validation() {
+  log::info "auth validation test running..."
+  endpoints=$(test::ci::_mk_endpoints 3)
+  test::ci::_etcdctl_expect "$endpoints" "user add root:root" "User root created" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "role add root" "Role root created" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "user grant-role root root" "Role root is granted to user root" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user list" "etcdserver: authentication is not enabled" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "auth enable" "Authentication Enabled" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user add u:u" "User u created" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user u:u user add f:f" "etcdserver: permission denied" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role add r" "Role r created" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user grant-role u r" "Role r is granted to user u" || return $?
+  test::ci::_etcdctl_expect "$endpoints""--user root:root role grant-permission r readwrite key1" "Role r updated" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u put key1 value1" "OK" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u get key1" "key1\nvalue1" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u role get r" "Role r\nKV Read:\n\tkey1\nKV Write:\n\tkey1" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user u:u user get u" "User: u\nRoles: r" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "echo 'new_password' | --user root:root user passwd --interactive=false u" "Password updated" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role revoke-permission r key1" "Permission of key key1 is revoked from role r" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user revoke-role u r" "Role r is revoked from user u" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user list" "root\nu" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role list" "r\nroot" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root user delete u" "User u deleted" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root role delete r" "Role r deleted" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user get non_exist_user" "etcdserver: user name not found" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user add root:root" "etcdserver: user name already exists" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root role get non_exist_role" "etcdserver: role name not found" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root role add root" "etcdserver: role name already exists" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $?
+  test::ci::_etcdctl_match "$endpoints" "--user root:root user delete root" "etcdserver: invalid auth management" || return $?
+  test::ci::_etcdctl_expect "$endpoints" "--user root:root auth disable" "Authentication Disabled" || return $?
+  log::info "auth validation test passed"
+}
+
 function test::ci::_install_CRD() {
     make install
     if [ $? -eq 0 ]; then
@@ -67,6 +124,9 @@ function test::ci::_start() {
   make run  >/dev/null 2>&1 &
   log::info "controller started"
   popd
+  log::info "create xline auth key pairs"
+  k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1
+  k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME
   log::info "starting xline cluster"
   k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1
   k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME
@@ -115,6 +175,7 @@ function test::run::ci::basic_validation() {
   endpoints=$(test::ci::_mk_endpoints 1)
   test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $?
   test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $?
+  test::ci::_auth_validation
   test::ci::_teardown
 }
 

tests/e2e/cases/manifests/auth-cred.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: auth-cred
+data:
+  auth-jwt.pri: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQ25BeHhTWEpZV1pDS3IKNmY2ajBIUlV3a2hYLzArR1hqRWNsV29MQTUrS1pBdVdNU3U4Yno2WCtJU2N2NHZOd09SbEdTV09ucnorOG1iMgpJMEY2dGVWWldmV0Zxc255V2s3SXhNK2g5eVRnN2FZLzg2ODVZZldUTDdmcFdxMS8zRm5pejRRYnNZRnV6QjFWCmdhWjVmRDJDU1lJS3pTRCtxVlNsWEYyNUpERkhWN2IyT2RIclgwVUtaT1RXWS9WRS8vU1R0K1BKS2RYOVIzcGwKa0d3QXpKSWtrY0FaeTB2aHZxVDNBU1RnWGNoTmVOOHdHWVliM1lpcmtxSXNRQjVYY3MxUjFXK3l6K0lyVmE2LwowV01jeUU2cXRKUFowbHZpeVQwbkhWL3Baalh1RDRCMGFqYS8xZmsvSG1YRFBNanBLMUJ1Q0JUU3RNL0tsY3JBCm9BeG8rWURoQWdNQkFBRUNnZ0VBSXlKaFkrWThZTXVDQzc1M0pra2xIK3ViUW4vZ1gva1N4ZHVjNm1KQnZ1QmIKRzZhT2Q5N0RRVDh6enJIeEhFRFhDM21sMEFJTzZtZGVSNnVWQzlhV1FCelByT1lJQStjQnFmVFZaVkpUdk1uaAo3cFE2S1kwMUYxaXpqUERaalF0ekVXYnNlTkwzMHJJMy9aUC96SkRaYzc0NUVFS2xEVTNjRThtQm9nQStLYTZ3CkdMb3pUOXFRZjhrbkJydHp4SDZTdnJacGZhUmxQOTVpczgyYjRJdVBocVlkRzdkVllGVEFMRTFNeVZyQ2JTNFkKS3l0ak5MZ3dwMWJJUXRXcnpNZWJCR29pVStEdkRjUlk4enZPZkZ1cER3cFlDdDNwMWFVNXd5WVlkcjc0ZXNWNwpqanFIajg5VWE2NUpISjNYbk1BYU1jNGRITTJGc0dxTXNPdi9EREtJblFLQmdRRGF3Y2tRRWVreDBRdVAzZUpQCkdXZFo4N29jK0ZWakRlM2JZaEFuQ2YveVhSSm9xY3M1dnIxbTF5Q1hGZnNqYlFGWUhXWFI5QVV0Tm41SEN3T1oKem9UMU12OTZmWEJWR1FPUmd6dmxVV1M0M3VLcGZJUERWdjJJNlpjS1NJUUFHT2djV1l2bUJEaFlxUEhnbXgzbwpWU3JOR1d0TGR5dzNyRDFKNk8rMVJ3dGJpd0tCZ1FERGNobVk1OUVYQmlUdmx5VDNRamwwdlpGTUhhK1RFbGJoCmlrTnRZbHRiVUh0YW1PWFp6cGRrL0tBN1gyZFlpMFFwVmZiYnBmUC9seTVsWXZnWndsOGg5ME9ib3BydStBQ00KbmRsS0JmTlFZQXJtV1k2YkoyQ3dGN2oxYVRDQ0hadVZ1WDYvcHpGVlN0UmNzc24xNXVvVmFJeUtkL01oSnpMRgpTM2VydFFrU3d3S0JnQW5pTVlSaFdzamVhZ2hRL1JXWHp6eVlMM041b05uOTJoNU1XdkI0bWpESUZiblcyaEM4CjFtL2NEbVBsSVZpalp5a2xBdUd1aGNGYU1mQmh4Z0xmK3MvZFF2KzB4U3VER3M4clA3eUhwZVpZWTZOR3RlbFEKZDlvRXU4ZENLWHlibzNrTWJxNnd5Qjd4V3lSTHZka3VaK1dtWFZ1bWdiL3VMMEswbklmek1zY3JBb0dBZUExZQpLODQ1WVNzbEJRYVNiazcvZS9YMWlndXlEV1QyZVJPMDF6dlRZZ1BOd1ppcGwyQ1BIamtQTTJrbTBmeTVvYXBzCk4vOTRJVWQ3K0VzU21zQUtMNUx5dEdidFJGeVIrYzM3NnJ3OCtPSUZ6L2l5NEJzUUNScUpRaldhMWxIWmY5NngKUElnMmhXMnhoRDlPVHYzSVM5NHNkZUc0Tm1VZGlwTVFyeWhFcW9FQ2dZRUFrdlhPZzY2SUFWVHJPNnFnb3lsNQo0Mm91ZmEvUUUrcU9BWW9RRXBteDNTWng2dE1reWNmQVFxVUhZY1hoVzFITmp5R2JiZy9zbDEzeWRkblBRcWlnCitPYnRRTlNJcUdaV0NjL0hJcU0vL3BQSTNNSFBoV0FSTU9tQWJrMEkxbVQwUUtodUZmU3VnVjJ4YjFEai9SdmYKMFZkQjh0eFkrNVd6NnpQMUYyZzQ2Z009Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
+  auth-jwt.pub: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFwd01jVWx5V0ZtUWlxK24rbzlCMApWTUpJVi85UGhsNHhISlZxQ3dPZmltUUxsakVydkc4K2wvaUVuTCtMemNEa1pSa2xqcDY4L3ZKbTlpTkJlclhsCldWbjFoYXJKOGxwT3lNVFBvZmNrNE8ybVAvT3ZPV0gxa3krMzZWcXRmOXhaNHMrRUc3R0Jic3dkVllHbWVYdzkKZ2ttQ0NzMGcvcWxVcFZ4ZHVTUXhSMWUyOWpuUjYxOUZDbVRrMW1QMVJQLzBrN2ZqeVNuVi9VZDZaWkJzQU15UwpKSkhBR2N0TDRiNms5d0VrNEYzSVRYamZNQm1HRzkySXE1S2lMRUFlVjNMTlVkVnZzcy9pSzFXdXY5RmpITWhPCnFyU1QyZEpiNHNrOUp4MWY2V1kxN2crQWRHbzJ2OVg1UHg1bHd6ekk2U3RRYmdnVTByVFB5cFhLd0tBTWFQbUEKNFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==

tests/e2e/cases/manifests/cluster.yaml

@@ -6,3 +6,11 @@ spec:
   image: phoenix500526/xline:v0.6.1
   imagePullPolicy: IfNotPresent
   replicas: 3
+  authSecret:
+    name: auth-cred
+    mountPath: /tmp/auth-cred
+    pubKey: auth-jwt.pub
+    priKey: auth-jwt.pri
+  bootstrapArgs:
+    auth-private-key: /tmp/auth-cred/auth-jwt.pri
+    auth-public-key: /tmp/auth-cred/auth-jwt.pub