Merge pull request #25 from yue9944882/feat/copying-seret

Feat: Copying secrets via manager instead of relying on ManagedServiceAccount's projection

charts/addon-manager/crds/clustergatewayconfigurations.yaml

@@ -70,12 +70,28 @@ spec:
                 type: string
               installNamespace:
                 type: string
+              secretManagement:
+                properties:
+                  managedServiceAccount:
+                    properties:
+                      name:
+                        default: cluster-gateway
+                        type: string
+                    type: object
+                  type:
+                    default: ManagedServiceAccount
+                    enum:
+                    - Manual
+                    - ManagedServiceAccount
+                    type: string
+                type: object
               secretNamespace:
                 type: string
             required:
             - egress
             - image
             - installNamespace
+            - secretManagement
             - secretNamespace
             type: object
           status:

charts/addon-manager/templates/clustergatewayconfiguration.yaml

@@ -6,6 +6,10 @@ spec:
   image: {{ .Values.clusterGateway.image }}
   installNamespace: {{ .Values.clusterGateway.installNamespace }}
   secretNamespace: {{ .Values.clusterGateway.secretNamespace }}
+  secretManagement:
+    type: ManagedServiceAccount
+    managedServiceAccount:
+      name: cluster-gateway
   egress:
     type: ClusterProxy
     clusterProxy:

charts/addon-manager/values.yaml

@@ -1,5 +1,5 @@
 # Image of the cluster-gateway instances
-image: yue9944882/cluster-gateway-addon-manager:v1.1.6.ocm.5
+image: yue9944882/cluster-gateway-addon-manager:v1.1.6.sec
 
 clusterGateway:
   image: oamdev/cluster-gateway:v1.1.6

go.mod

@@ -30,6 +30,6 @@ require (
 )
 
 replace (
-	open-cluster-management.io/managed-serviceaccount v0.0.0 => github.com/yue9944882/managed-serviceaccount v0.0.31
+	open-cluster-management.io/managed-serviceaccount v0.0.0 => github.com/yue9944882/managed-serviceaccount v0.0.32
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.24
 )

go.sum

@@ -837,8 +837,8 @@ github.com/yeya24/promlinter v0.1.0/go.mod h1:rs5vtZzeBHqqMwXqFScncpCF6u06lezhZe
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
-github.com/yue9944882/managed-serviceaccount v0.0.31 h1:PDUR80gcc/xecF5BvJzntq3efL+oixfSiRyZDl8UqN8=
-github.com/yue9944882/managed-serviceaccount v0.0.31/go.mod h1:pibwxJlKDE0r59JGF8IGw4zaxSNA0Z20t9QmRL5wYaM=
+github.com/yue9944882/managed-serviceaccount v0.0.32 h1:QZRO4aisEotalPqtxUCynUBSAYoZcfou8N7qmgvXMAo=
+github.com/yue9944882/managed-serviceaccount v0.0.32/go.mod h1:D0DMSNH92z1aVpBexiYXdbrgnv+tG6alxqPSym8DLOk=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

hack/crd/bases/proxy.open-cluster-management.io_clustergatewayconfigurations.yaml

@@ -70,12 +70,28 @@ spec:
                 type: string
               installNamespace:
                 type: string
+              secretManagement:
+                properties:
+                  managedServiceAccount:
+                    properties:
+                      name:
+                        default: cluster-gateway
+                        type: string
+                    type: object
+                  type:
+                    default: ManagedServiceAccount
+                    enum:
+                    - Manual
+                    - ManagedServiceAccount
+                    type: string
+                type: object
               secretNamespace:
                 type: string
             required:
             - egress
             - image
             - installNamespace
+            - secretManagement
             - secretNamespace
             type: object
           status:

pkg/addon/agent/addon.go

@@ -1,23 +1,12 @@
 package agent
 
 import (
-	"context"
-	"fmt"
-	"time"
-
-	proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1"
 	"github.com/oam-dev/cluster-gateway/pkg/common"
-	"github.com/pkg/errors"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"open-cluster-management.io/addon-framework/pkg/agent"
 	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
-	ocmauthv1alpha1 "open-cluster-management.io/managed-serviceaccount/api/v1alpha1"
-	"open-cluster-management.io/managed-serviceaccount/pkg/addon/agent/health"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -36,68 +25,6 @@ type clusterGatewayAddonManager struct {
 }
 
 func (c *clusterGatewayAddonManager) Manifests(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) {
-	if len(addon.Status.AddOnConfiguration.CRName) == 0 {
-		return nil, fmt.Errorf("no gateway configuration bond to ManagedClusterAddOn")
-	}
-	gatewayConfig := &proxyv1alpha1.ClusterGatewayConfiguration{}
-	if err := c.client.Get(context.TODO(), types.NamespacedName{
-		Name: addon.Status.AddOnConfiguration.CRName,
-	}, gatewayConfig); err != nil {
-		return nil, fmt.Errorf("failed getting gateway configuration bond to ManagedClusterAddOn")
-	}
-	endpointType := "Const"
-	if gatewayConfig.Spec.Egress.Type == proxyv1alpha1.EgressTypeClusterProxy {
-		endpointType = "ClusterProxy"
-	}
-	msa := &ocmauthv1alpha1.ManagedServiceAccount{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "authentication.open-cluster-management.io/v1alpha1",
-			Kind:       "ManagedServiceAccount",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: cluster.Name,
-			Name:      common.AddonName,
-			OwnerReferences: []metav1.OwnerReference{
-				{
-					APIVersion: addonv1alpha1.GroupVersion.String(),
-					Kind:       "ManagedClusterAddOn",
-					UID:        addon.UID,
-					Name:       addon.Name,
-				},
-			},
-		},
-		Spec: ocmauthv1alpha1.ManagedServiceAccountSpec{
-			Projected: ocmauthv1alpha1.ManagedServiceAccountProjected{
-				Type: ocmauthv1alpha1.ProjectionTypeSecret,
-				Secret: &ocmauthv1alpha1.ProjectedSecret{
-					Labels: map[string]string{
-						"cluster.core.oam.dev/cluster-credential-type": "ServiceAccountToken",
-						"cluster.core.oam.dev/cluster-endpoint-type":   endpointType,
-					},
-					Namespace: gatewayConfig.Spec.SecretNamespace,
-					Name:      cluster.Name,
-				},
-			},
-			Rotation: ocmauthv1alpha1.ManagedServiceAccountRotation{
-				Enabled: true,
-				Validity: metav1.Duration{
-					Duration: time.Hour * 24 * 180,
-				},
-			},
-		},
-	}
-	if err := c.client.Create(context.TODO(), msa); err != nil {
-		if !apierrors.IsAlreadyExists(err) {
-			return nil, errors.Wrapf(err, "failed creating managed serviceaccount for cluster %v", cluster.Name)
-		}
-	}
-
-	leaseUpdater, err := health.NewAddonHealthUpdater(c.clientConfig, cluster.Name)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed creating lease updater")
-	}
-	go leaseUpdater.Start(context.Background())
-
 	return nil, nil
 }
 

pkg/addon/controllers/installer.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"time"
 
+	clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1"
 	proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1"
 	"github.com/oam-dev/cluster-gateway/pkg/common"
 	"github.com/oam-dev/cluster-gateway/pkg/event"
@@ -28,6 +29,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	ocmauthv1alpha1 "open-cluster-management.io/managed-serviceaccount/api/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -55,6 +57,11 @@ func SetupClusterGatewayInstallerWithManager(mgr ctrl.Manager, caPair *crypto.CA
 			&event.ClusterGatewayConfigurationHandler{
 				Client: mgr.GetClient(),
 			}).
+		Watches(
+			&source.Kind{
+				Type: &corev1.Secret{},
+			},
+			&event.SecretHandler{}).
 		Complete(installer)
 }
 
@@ -81,6 +88,11 @@ func (c *ClusterGatewayInstaller) Reconcile(ctx context.Context, request reconci
 		}
 		return reconcile.Result{}, errors.Wrapf(err, "failed to get cluster-management-addon: %v", request.Name)
 	}
+	if addon.Name != common.AddonName {
+		// skip
+		return reconcile.Result{}, nil
+	}
+
 	if addon.Spec.AddOnConfiguration.CRDName != common.ClusterGatewayConfigurationCRDName {
 		// skip
 		return reconcile.Result{}, nil
@@ -102,9 +114,12 @@ func (c *ClusterGatewayInstaller) Reconcile(ctx context.Context, request reconci
 	if err := c.ensureNamespace(clusterGatewayConfiguration.Spec.SecretNamespace); err != nil {
 		return reconcile.Result{}, errors.Wrapf(err, "failed to ensure required namespace")
 	}
-	if err := c.ensureProxySecrets(clusterGatewayConfiguration); err != nil {
+	if err := c.ensureClusterProxySecrets(clusterGatewayConfiguration); err != nil {
 		return reconcile.Result{}, errors.Wrapf(err, "failed to ensure required proxy client related credentials")
 	}
+	if err := c.ensureSecretManagement(addon, clusterGatewayConfiguration); err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to configure secret management")
+	}
 
 	sans := []string{
 		ServiceNameClusterGateway,
@@ -196,7 +211,7 @@ func (c *ClusterGatewayInstaller) ensureAPIService(addon *addonv1alpha1.ClusterM
 	return nil
 }
 
-func (c *ClusterGatewayInstaller) ensureProxySecrets(config *proxyv1alpha1.ClusterGatewayConfiguration) error {
+func (c *ClusterGatewayInstaller) ensureClusterProxySecrets(config *proxyv1alpha1.ClusterGatewayConfiguration) error {
 	if config.Spec.Egress.Type != proxyv1alpha1.EgressTypeClusterProxy {
 		return nil
 	}
@@ -232,6 +247,106 @@ func (c *ClusterGatewayInstaller) ensureProxySecrets(config *proxyv1alpha1.Clust
 	return nil
 }
 
+func (c *ClusterGatewayInstaller) ensureSecretManagement(clusterAddon *addonv1alpha1.ClusterManagementAddOn, config *proxyv1alpha1.ClusterGatewayConfiguration) error {
+	if config.Spec.SecretManagement.Type != proxyv1alpha1.SecretManagementTypeManagedServiceAccount {
+		return nil
+	}
+	addonList := &addonv1alpha1.ManagedClusterAddOnList{}
+	if err := c.client.List(context.TODO(), addonList); err != nil {
+		return errors.Wrapf(err, "failed to list managed cluster addons")
+	}
+	clusterGatewayAddon := make([]*addonv1alpha1.ManagedClusterAddOn, 0)
+	for _, addon := range addonList.Items {
+		addon := addon
+		if addon.Name == common.AddonName {
+			clusterGatewayAddon = append(clusterGatewayAddon, &addon)
+		}
+	}
+	for _, addon := range clusterGatewayAddon {
+		managedServiceAccount := buildManagedServiceAccount(addon)
+		if err := c.client.Create(context.TODO(), managedServiceAccount); err != nil {
+			if !apierrors.IsAlreadyExists(err) {
+				return errors.Wrapf(err, "failed to create managed serviceaccount")
+			}
+		}
+
+		if err := c.copySecretForManagedServiceAccount(
+			clusterAddon,
+			config,
+			addon.Namespace); err != nil {
+			return errors.Wrapf(err, "failed to copy secret from managed serviceaccount")
+		}
+	}
+	return nil
+}
+
+func (c *ClusterGatewayInstaller) copySecretForManagedServiceAccount(addon *addonv1alpha1.ClusterManagementAddOn, config *proxyv1alpha1.ClusterGatewayConfiguration, clusterName string) error {
+	endpointType := clusterv1alpha1.ClusterEndpointTypeConst
+	if config.Spec.Egress.Type == proxyv1alpha1.EgressTypeClusterProxy {
+		endpointType = clusterv1alpha1.ClusterEndpointTypeClusterProxy
+	}
+	gatewaySecretNamespace := config.Spec.SecretNamespace
+	secretName := config.Spec.SecretManagement.ManagedServiceAccount.Name
+
+	secret, err := c.secretLister.Secrets(clusterName).
+		Get(secretName)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to get token secret")
+		}
+		return nil
+	}
+	currentSecret, err := c.secretLister.Secrets(gatewaySecretNamespace).Get(clusterName)
+	shouldCreate := false
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to get the cluster secret")
+		}
+		shouldCreate = true
+	}
+	if shouldCreate {
+		if _, err := c.nativeClient.CoreV1().Secrets(gatewaySecretNamespace).
+			Create(context.TODO(),
+				&corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: gatewaySecretNamespace,
+						Name:      clusterName,
+						Labels: map[string]string{
+							clusterv1alpha1.LabelKeyClusterCredentialType: string(clusterv1alpha1.CredentialTypeServiceAccountToken),
+							clusterv1alpha1.LabelKeyClusterEndpointType:   endpointType,
+						},
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								APIVersion: addonv1alpha1.GroupVersion.String(),
+								Kind:       "ClusterManagementAddOn",
+								UID:        addon.UID,
+								Name:       addon.Name,
+							},
+						},
+					},
+					Type: corev1.SecretTypeOpaque,
+					Data: map[string][]byte{
+						corev1.ServiceAccountRootCAKey: secret.Data[corev1.ServiceAccountRootCAKey],
+						corev1.ServiceAccountTokenKey:  secret.Data[corev1.ServiceAccountTokenKey],
+					},
+				},
+				metav1.CreateOptions{}); err != nil {
+			return errors.Wrapf(err, "failed to create the cluster secret")
+		}
+	} else {
+		if bytes.Equal(secret.Data[corev1.ServiceAccountTokenKey], currentSecret.Data[corev1.ServiceAccountTokenKey]) {
+			return nil // no need for an update
+		}
+		currentSecret.Data[corev1.ServiceAccountRootCAKey] = secret.Data[corev1.ServiceAccountRootCAKey]
+		currentSecret.Data[corev1.ServiceAccountTokenKey] = secret.Data[corev1.ServiceAccountTokenKey]
+		if _, err := c.nativeClient.CoreV1().Secrets(gatewaySecretNamespace).
+			Update(context.TODO(), currentSecret, metav1.UpdateOptions{}); err != nil {
+			return errors.Wrapf(err, "failed to update the cluster secret")
+		}
+	}
+	return nil
+}
+
 func newServiceAccount(addon *addonv1alpha1.ClusterManagementAddOn, namespace string) *corev1.ServiceAccount {
 	return &corev1.ServiceAccount{
 		TypeMeta: metav1.TypeMeta{
@@ -567,5 +682,33 @@ func newAPFClusterRoleBinding(addon *addonv1alpha1.ClusterManagementAddOn, names
 			},
 		},
 	}
+}
 
+func buildManagedServiceAccount(addon *addonv1alpha1.ManagedClusterAddOn) *ocmauthv1alpha1.ManagedServiceAccount {
+	return &ocmauthv1alpha1.ManagedServiceAccount{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "authentication.open-cluster-management.io/v1alpha1",
+			Kind:       "ManagedServiceAccount",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: addon.Namespace,
+			Name:      common.AddonName,
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: addonv1alpha1.GroupVersion.String(),
+					Kind:       "ManagedClusterAddOn",
+					UID:        addon.UID,
+					Name:       addon.Name,
+				},
+			},
+		},
+		Spec: ocmauthv1alpha1.ManagedServiceAccountSpec{
+			Rotation: ocmauthv1alpha1.ManagedServiceAccountRotation{
+				Enabled: true,
+				Validity: metav1.Duration{
+					Duration: time.Hour * 24 * 180,
+				},
+			},
+		},
+	}
 }