diff --git a/api/v1alpha1/xlinecluster_types.go b/api/v1alpha1/xlinecluster_types.go index 4b47237..b2f4c65 100644 --- a/api/v1alpha1/xlinecluster_types.go +++ b/api/v1alpha1/xlinecluster_types.go @@ -133,6 +133,16 @@ type XlineClusterSpec struct { // The replicas of xline nodes // +kubebuilder:validation:Minimum=3 Replicas int32 `json:"replicas"` + + // The auth secret keys + AuthSecrets *XlineAuthSecret `json:"authSecret,omitempty"` +} + +type XlineAuthSecret struct { + Name *string `json:"name"` + MountPath *string `json:"mountPath"` + PubKey *string `json:"pubKey"` + PriKey *string `json:"priKey"` } func (s *XlineClusterSpec) BootArgs() []string { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6c2b1f8..72d0388 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -171,6 +171,41 @@ func (in *XlineArgs) DeepCopy() *XlineArgs { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *XlineAuthSecret) DeepCopyInto(out *XlineAuthSecret) { + *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.MountPath != nil { + in, out := &in.MountPath, &out.MountPath + *out = new(string) + **out = **in + } + if in.PubKey != nil { + in, out := &in.PubKey, &out.PubKey + *out = new(string) + **out = **in + } + if in.PriKey != nil { + in, out := &in.PriKey, &out.PriKey + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineAuthSecret. +func (in *XlineAuthSecret) DeepCopy() *XlineAuthSecret { + if in == nil { + return nil + } + out := new(XlineAuthSecret) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *XlineCluster) DeepCopyInto(out *XlineCluster) { *out = *in @@ -259,6 +294,11 @@ func (in *XlineClusterSpec) DeepCopyInto(out *XlineClusterSpec) { **out = **in } in.BootstrapArgs.DeepCopyInto(&out.BootstrapArgs) + if in.AuthSecrets != nil { + in, out := &in.AuthSecrets, &out.AuthSecrets + *out = new(XlineAuthSecret) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineClusterSpec. diff --git a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml index a845968..3fc8d02 100644 --- a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml +++ b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml @@ -35,6 +35,23 @@ spec: spec: description: XlineClusterSpec defines the desired state of XlineCluster properties: + authSecret: + description: The auth secret keys + properties: + mountPath: + type: string + name: + type: string + priKey: + type: string + pubKey: + type: string + required: + - mountPath + - name + - priKey + - pubKey + type: object bootstrapArgs: description: / Xline container bootstrap arguments / Set additional arguments except [`--name`, `--members`, `--storage-engine`, `--data-dir`] diff --git a/internal/transformer/xlinecluster_resource.go b/internal/transformer/xlinecluster_resource.go index 99907ac..f43bd4f 100644 --- a/internal/transformer/xlinecluster_resource.go +++ b/internal/transformer/xlinecluster_resource.go @@ -46,6 +46,38 @@ func GetMemberTopology(stsRef types.NamespacedName, svcName string, replicas int return strings.Join(members, ",") } +func GetAuthSecretVolume(auth_sec *xapi.XlineAuthSecret) []corev1.Volume { + if auth_sec == nil { + return []corev1.Volume{} + } + return []corev1.Volume{ + {Name: "auth-cred", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: *auth_sec.Name, + }, + }}, + } +} + +func GetAuthSecretVolumeMount(auth_sec *xapi.XlineAuthSecret) []corev1.VolumeMount { + if auth_sec == nil { + return []corev1.VolumeMount{} + } + return []corev1.VolumeMount{ + {Name: "auth-cred", ReadOnly: true, MountPath: *auth_sec.MountPath}, + } +} + +func GetAuthSecretEnvVars(auth_sec *xapi.XlineAuthSecret) []corev1.EnvVar { + if auth_sec == nil { + return []corev1.EnvVar{} + } + return []corev1.EnvVar{ + {Name: "AUTH_PUBLIC_KEY", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PubKey)}, + {Name: "AUTH_PRIVATE_KEY", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PriKey)}, + } +} + func MakeService(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.Service { svcRef := GetServiceKey(cr.ObjKey()) svcLabel := GetXlineInstanceLabels(cr.ObjKey()) @@ -83,7 +115,19 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State "--data-dir", DataDir, } + envs := []corev1.EnvVar{ + {Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }}, + } + envs = append(envs, GetAuthSecretEnvVars(cr.Spec.AuthSecrets)...) + initCmd = append(initCmd, cr.Spec.BootArgs()...) + volumes := GetAuthSecretVolume(cr.Spec.AuthSecrets) + volumeMount := GetAuthSecretVolumeMount(cr.Spec.AuthSecrets) // pod template: main container mainContainer := corev1.Container{ @@ -93,15 +137,9 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State Ports: []corev1.ContainerPort{ {Name: "xline-port", ContainerPort: XlinePort}, }, - Command: initCmd, - Env: []corev1.EnvVar{ - {Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))}, - {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }}, - }, + Command: initCmd, + Env: envs, + VolumeMounts: volumeMount, } // pod template @@ -110,6 +148,7 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State Labels: stsLabels, }, Spec: corev1.PodSpec{ + Volumes: volumes, Containers: []corev1.Container{mainContainer}, }, } diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh index 0695d42..258bc01 100644 --- a/tests/e2e/cases/ci.sh +++ b/tests/e2e/cases/ci.sh @@ -6,6 +6,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh" _TEST_CI_CLUSTER_NAME="my-xline-cluster" _TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts" _TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc" +_TEST_CI_SECRET_NAME="auth-cred" _TEST_CI_NAMESPACE="default" _TEST_CI_DNS_SUFFIX="svc.cluster.local" _TEST_CI_XLINE_PORT="2379" @@ -33,6 +34,62 @@ function test::ci::_etcdctl_expect() { fi } +# run a command with expect output, based on key word match +# args: +# $1: endpoints +# $2: command to run +# $3: key word to match +function test::ci::_etcdctl_match() { + log::debug "run command: etcdctl --endpoints=$1 $2" + got=$(testenv::util::etcdctl --endpoints="$1" "$2") + expect=$(echo -e ${3}) + if echo "${got}" | grep -q "${expect}"; then + log::info "command run success" + else + log::error "command run failed" + log::error "expect: ${expect}" + log::error "got: $got" + return 1 + fi +} + +function test::ci::_auth_validation() { + log::info "auth validation test running..." + endpoints=$(test::ci::_mk_endpoints 3) + test::ci::_etcdctl_expect "$endpoints" "user add root:root" "User root created" || return $? + test::ci::_etcdctl_expect "$endpoints" "role add root" "Role root created" || return $? + test::ci::_etcdctl_expect "$endpoints" "user grant-role root root" "Role root is granted to user root" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root user list" "etcdserver: authentication is not enabled" || return $? + test::ci::_etcdctl_expect "$endpoints" "auth enable" "Authentication Enabled" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root user add u:u" "User u created" || return $? + test::ci::_etcdctl_match "$endpoints" "--user u:u user add f:f" "etcdserver: permission denied" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root role add r" "Role r created" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root user grant-role u r" "Role r is granted to user u" || return $? + test::ci::_etcdctl_expect "$endpoints""--user root:root role grant-permission r readwrite key1" "Role r updated" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user u:u put key1 value1" "OK" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user u:u get key1" "key1\nvalue1" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user u:u role get r" "Role r\nKV Read:\n\tkey1\nKV Write:\n\tkey1" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user u:u user get u" "User: u\nRoles: r" || return $? + test::ci::_etcdctl_expect "$endpoints" "echo 'new_password' | --user root:root user passwd --interactive=false u" "Password updated" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root role revoke-permission r key1" "Permission of key key1 is revoked from role r" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root user revoke-role u r" "Role r is revoked from user u" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root user list" "root\nu" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root role list" "r\nroot" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root user delete u" "User u deleted" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root role delete r" "Role r deleted" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root user get non_exist_user" "etcdserver: user name not found" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root user add root:root" "etcdserver: user name already exists" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root role get non_exist_role" "etcdserver: role name not found" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root role add root" "etcdserver: role name already exists" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $? + test::ci::_etcdctl_match "$endpoints" "--user root:root user delete root" "etcdserver: invalid auth management" || return $? + test::ci::_etcdctl_expect "$endpoints" "--user root:root auth disable" "Authentication Disabled" || return $? + log::info "auth validation test passed" +} + function test::ci::_install_CRD() { make install if [ $? -eq 0 ]; then @@ -67,6 +124,9 @@ function test::ci::_start() { make run >/dev/null 2>&1 & log::info "controller started" popd + log::info "create xline auth key pairs" + k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1 + k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME log::info "starting xline cluster" k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1 k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME @@ -115,6 +175,7 @@ function test::run::ci::basic_validation() { endpoints=$(test::ci::_mk_endpoints 1) test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $? test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $? + test::ci::_auth_validation test::ci::_teardown } diff --git a/tests/e2e/cases/manifests/auth-cred.yaml b/tests/e2e/cases/manifests/auth-cred.yaml new file mode 100644 index 0000000..8501b67 --- /dev/null +++ b/tests/e2e/cases/manifests/auth-cred.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + creationTimestamp: null + name: auth-cred +data: + auth-jwt.pri: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQ25BeHhTWEpZV1pDS3IKNmY2ajBIUlV3a2hYLzArR1hqRWNsV29MQTUrS1pBdVdNU3U4Yno2WCtJU2N2NHZOd09SbEdTV09ucnorOG1iMgpJMEY2dGVWWldmV0Zxc255V2s3SXhNK2g5eVRnN2FZLzg2ODVZZldUTDdmcFdxMS8zRm5pejRRYnNZRnV6QjFWCmdhWjVmRDJDU1lJS3pTRCtxVlNsWEYyNUpERkhWN2IyT2RIclgwVUtaT1RXWS9WRS8vU1R0K1BKS2RYOVIzcGwKa0d3QXpKSWtrY0FaeTB2aHZxVDNBU1RnWGNoTmVOOHdHWVliM1lpcmtxSXNRQjVYY3MxUjFXK3l6K0lyVmE2LwowV01jeUU2cXRKUFowbHZpeVQwbkhWL3Baalh1RDRCMGFqYS8xZmsvSG1YRFBNanBLMUJ1Q0JUU3RNL0tsY3JBCm9BeG8rWURoQWdNQkFBRUNnZ0VBSXlKaFkrWThZTXVDQzc1M0pra2xIK3ViUW4vZ1gva1N4ZHVjNm1KQnZ1QmIKRzZhT2Q5N0RRVDh6enJIeEhFRFhDM21sMEFJTzZtZGVSNnVWQzlhV1FCelByT1lJQStjQnFmVFZaVkpUdk1uaAo3cFE2S1kwMUYxaXpqUERaalF0ekVXYnNlTkwzMHJJMy9aUC96SkRaYzc0NUVFS2xEVTNjRThtQm9nQStLYTZ3CkdMb3pUOXFRZjhrbkJydHp4SDZTdnJacGZhUmxQOTVpczgyYjRJdVBocVlkRzdkVllGVEFMRTFNeVZyQ2JTNFkKS3l0ak5MZ3dwMWJJUXRXcnpNZWJCR29pVStEdkRjUlk4enZPZkZ1cER3cFlDdDNwMWFVNXd5WVlkcjc0ZXNWNwpqanFIajg5VWE2NUpISjNYbk1BYU1jNGRITTJGc0dxTXNPdi9EREtJblFLQmdRRGF3Y2tRRWVreDBRdVAzZUpQCkdXZFo4N29jK0ZWakRlM2JZaEFuQ2YveVhSSm9xY3M1dnIxbTF5Q1hGZnNqYlFGWUhXWFI5QVV0Tm41SEN3T1oKem9UMU12OTZmWEJWR1FPUmd6dmxVV1M0M3VLcGZJUERWdjJJNlpjS1NJUUFHT2djV1l2bUJEaFlxUEhnbXgzbwpWU3JOR1d0TGR5dzNyRDFKNk8rMVJ3dGJpd0tCZ1FERGNobVk1OUVYQmlUdmx5VDNRamwwdlpGTUhhK1RFbGJoCmlrTnRZbHRiVUh0YW1PWFp6cGRrL0tBN1gyZFlpMFFwVmZiYnBmUC9seTVsWXZnWndsOGg5ME9ib3BydStBQ00KbmRsS0JmTlFZQXJtV1k2YkoyQ3dGN2oxYVRDQ0hadVZ1WDYvcHpGVlN0UmNzc24xNXVvVmFJeUtkL01oSnpMRgpTM2VydFFrU3d3S0JnQW5pTVlSaFdzamVhZ2hRL1JXWHp6eVlMM041b05uOTJoNU1XdkI0bWpESUZiblcyaEM4CjFtL2NEbVBsSVZpalp5a2xBdUd1aGNGYU1mQmh4Z0xmK3MvZFF2KzB4U3VER3M4clA3eUhwZVpZWTZOR3RlbFEKZDlvRXU4ZENLWHlibzNrTWJxNnd5Qjd4V3lSTHZka3VaK1dtWFZ1bWdiL3VMMEswbklmek1zY3JBb0dBZUExZQpLODQ1WVNzbEJRYVNiazcvZS9YMWlndXlEV1QyZVJPMDF6dlRZZ1BOd1ppcGwyQ1BIamtQTTJrbTBmeTVvYXBzCk4vOTRJVWQ3K0VzU21zQUtMNUx5dEdidFJGeVIrYzM3NnJ3OCtPSUZ6L2l5NEJzUUNScUpRaldhMWxIWmY5NngKUElnMmhXMnhoRDlPVHYzSVM5NHNkZUc0Tm1VZGlwTVFyeWhFcW9FQ2dZRUFrdlhPZzY2SUFWVHJPNnFnb3lsNQo0Mm91ZmEvUUUrcU9BWW9RRXBteDNTWng2dE1reWNmQVFxVUhZY1hoVzFITmp5R2JiZy9zbDEzeWRkblBRcWlnCitPYnRRTlNJcUdaV0NjL0hJcU0vL3BQSTNNSFBoV0FSTU9tQWJrMEkxbVQwUUtodUZmU3VnVjJ4YjFEai9SdmYKMFZkQjh0eFkrNVd6NnpQMUYyZzQ2Z009Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K + auth-jwt.pub: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFwd01jVWx5V0ZtUWlxK24rbzlCMApWTUpJVi85UGhsNHhISlZxQ3dPZmltUUxsakVydkc4K2wvaUVuTCtMemNEa1pSa2xqcDY4L3ZKbTlpTkJlclhsCldWbjFoYXJKOGxwT3lNVFBvZmNrNE8ybVAvT3ZPV0gxa3krMzZWcXRmOXhaNHMrRUc3R0Jic3dkVllHbWVYdzkKZ2ttQ0NzMGcvcWxVcFZ4ZHVTUXhSMWUyOWpuUjYxOUZDbVRrMW1QMVJQLzBrN2ZqeVNuVi9VZDZaWkJzQU15UwpKSkhBR2N0TDRiNms5d0VrNEYzSVRYamZNQm1HRzkySXE1S2lMRUFlVjNMTlVkVnZzcy9pSzFXdXY5RmpITWhPCnFyU1QyZEpiNHNrOUp4MWY2V1kxN2crQWRHbzJ2OVg1UHg1bHd6ekk2U3RRYmdnVTByVFB5cFhLd0tBTWFQbUEKNFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg== diff --git a/tests/e2e/cases/manifests/cluster.yaml b/tests/e2e/cases/manifests/cluster.yaml index b276b30..71b516a 100644 --- a/tests/e2e/cases/manifests/cluster.yaml +++ b/tests/e2e/cases/manifests/cluster.yaml @@ -6,3 +6,8 @@ spec: image: phoenix500526/xline:v0.6.1 imagePullPolicy: IfNotPresent replicas: 3 + authSecret: + name: auth-cred + mountPath: /tmp/auth-cred + pubKey: auth-jwt.pub + priKey: auth-jwt.pri