SA does not create secret automatically in OCP 4.15 with IBM vmware

Signed-off-by: Xiangjing Li <[email protected]>

pkg/controller/spoketoken/spoke_toke_controller_test.go

@@ -24,6 +24,7 @@ import (
  kerrors "k8s.io/apimachinery/pkg/api/errors"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/types"
+ "k8s.io/klog/v2"
  "sigs.k8s.io/controller-runtime/pkg/manager"
  "sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
@@ -99,6 +100,13 @@ var (
  },
  Type: corev1.SecretTypeServiceAccountToken,
  }
+
+ sa2 = &corev1.ServiceAccount{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "application-manager",
+ Namespace: "open-cluster-management-agent-addon",
+ },
+ }
 )
 
 var expectedRequest = reconcile.Request{NamespacedName: sakey}
@@ -179,9 +187,10 @@ func TestReconcile(t *testing.T) {
 
  g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest)))
 
- theDeletedSecret := &corev1.Secret{}
-
  // Check that cluster1/cluster1-cluster-secret is deleted upon the deletion of the service account.
+ time.Sleep(time.Second * 2)
+
+ theDeletedSecret := &corev1.Secret{}
  g.Expect(kerrors.IsNotFound(c.Get(context.TODO(), secretkey, theDeletedSecret))).To(gomega.BeTrue())
 
  // Verify that service accounts other than open-cluster-management-agent-addon/klusterlet-addon-appmgr-
@@ -198,4 +207,59 @@ func TestReconcile(t *testing.T) {
  defer c.Delete(context.TODO(), saToBeIgnored)
 
  g.Eventually(requests, timeout).ShouldNot(gomega.Receive(gomega.Equal(unexpectedRequest)))
+
+ // Test case 3: create a SA without secret list, expect the spoke token controller to create the SA secret successfully
+ c.Delete(context.TODO(), sa1)
+ c.Delete(context.TODO(), secret1)
+
+ g.Expect(c.Create(context.TODO(), sa2)).NotTo(gomega.HaveOccurred())
+ defer c.Delete(context.TODO(), sa2)
+
+ g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest)))
+
+ // Check the secret open-cluster-management-agent-addon/application-manager is created.
+ time.Sleep(time.Second * 2)
+
+ saSecret := &corev1.Secret{}
+ g.Expect(c.Get(context.TODO(), sakey, saSecret)).NotTo(gomega.HaveOccurred())
+
+ klog.Infof("new SA secet: %#v", saSecret)
+
+ g.Expect(string(saSecret.Annotations["kubernetes.io/service-account.name"])).To(gomega.Equal(sakey.Name))
+ g.Expect(string(saSecret.Type)).To(gomega.Equal("kubernetes.io/service-account-token"))
+
+ // append the token to the SA secret, expect the cluster secret will be generated using the SA secret token
+ saSecret.Data = map[string][]byte{}
+ saSecret.Data["token"] = []byte("new-token-2")
+ g.Expect(c.Update(context.TODO(), saSecret)).NotTo(gomega.HaveOccurred())
+
+ g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest)))
+
+ // Check that cluster secret cluster1/cluster1-cluster-secret is created.
+ time.Sleep(time.Second * 2)
+
+ clusterSecret := &corev1.Secret{}
+ g.Expect(c.Get(context.TODO(), secretkey, clusterSecret)).NotTo(gomega.HaveOccurred())
+
+ // Verify the secret data
+ g.Expect(string(clusterSecret.Data["name"])).To(gomega.Equal(clusterName))
+ g.Expect(string(clusterSecret.Data["server"])).To(gomega.Equal(host))
+ g.Expect(string(clusterSecret.Data["config"])).NotTo(gomega.BeEmpty())
+
+ configData = &Config{}
+ g.Expect(json.Unmarshal(clusterSecret.Data["config"], configData)).NotTo(gomega.HaveOccurred())
+ g.Expect(configData.BearerToken).To(gomega.Equal(string(saSecret.Data["token"])))
+ g.Expect(configData.TLSClientConfig["insecure"]).To(gomega.BeTrue())
+
+ // delete the SA secret, expect the SA secret is regenerated
+ g.Expect(c.Delete(context.TODO(), saSecret)).NotTo(gomega.HaveOccurred())
+
+ g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest)))
+
+ time.Sleep(time.Second * 2)
+
+ newSaSecret := &corev1.Secret{}
+ g.Expect(c.Get(context.TODO(), sakey, newSaSecret)).NotTo(gomega.HaveOccurred())
+
+ klog.Infof("new SA secet: %#v", newSaSecret)
 }

pkg/controller/spoketoken/spoke_token_controller.go

@@ -17,6 +17,7 @@ package spoketoken
 import (
  "context"
  "encoding/json"
+ "fmt"
  "net/url"
  "strings"
  "time"
@@ -25,14 +26,18 @@ import (
  "github.com/pkg/errors"
  corev1 "k8s.io/api/core/v1"
  kerrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
  "k8s.io/apimachinery/pkg/types"
  "k8s.io/client-go/rest"
  "k8s.io/klog/v2"
  "sigs.k8s.io/controller-runtime/pkg/client"
  "sigs.k8s.io/controller-runtime/pkg/controller"
+ "sigs.k8s.io/controller-runtime/pkg/event"
  "sigs.k8s.io/controller-runtime/pkg/handler"
  "sigs.k8s.io/controller-runtime/pkg/manager"
+ "sigs.k8s.io/controller-runtime/pkg/predicate"
  "sigs.k8s.io/controller-runtime/pkg/reconcile"
  "sigs.k8s.io/controller-runtime/pkg/source"
 
@@ -43,6 +48,8 @@ const (
  secretSuffix = "-cluster-secret"
  requeuAfter = 5
  infrastructureConfigName = "cluster"
+ appAddonNS = "open-cluster-management-agent-addon"
+ appAddonName = "application-manager"
 )
 
 // Add creates a new agent token controller and adds it to the Manager if standalone is false.
@@ -74,6 +81,26 @@ func newReconciler(mgr manager.Manager, hubclient client.Client, syncid *types.N
  return rec
 }
 
+type applicationManagerSecretMapper struct {
+ client.Client
+}
+
+func (mapper *applicationManagerSecretMapper) Map(ctx context.Context, obj client.Object) []reconcile.Request {
+ var requests []reconcile.Request
+
+ // reconcile open-cluster-management-agent-addon/application-manager SA if its associated secret changes
+ requests = append(requests, reconcile.Request{
+ NamespacedName: types.NamespacedName{
+ Namespace: appAddonNS,
+ Name: appAddonName,
+ },
+ })
+
+ klog.Infof("app addon SA secret changed")
+
+ return requests
+}
+
 // add adds a new Controller to mgr with r as the reconcile.Reconciler
 func add(mgr manager.Manager, r reconcile.Reconciler) error {
  klog.Info("Adding klusterlet token controller.")
@@ -83,12 +110,23 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
  return err
  }
 
- // Watch for changes to klusterlet-addon-appmgr service account in open-cluster-management-agent-addon namespace.
+ // Watch for changes to open-cluster-management-agent-addon/application-manager service account.
  err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ServiceAccount{}), &handler.EnqueueRequestForObject{}, utils.ServiceAccountPredicateFunctions)
  if err != nil {
  return err
  }
 
+ // watch for changes to the secrets associated to the open-cluster-management-agent-addon/application-manager SA
+ saSecretMapper := &applicationManagerSecretMapper{mgr.GetClient()}
+ err = c.Watch(
+ source.Kind(mgr.GetCache(), &corev1.Secret{}),
+ handler.EnqueueRequestsFromMapFunc(saSecretMapper.Map),
+ applicationManagerSecretPredicateFunctions)
+
+ if err != nil {
+ return err
+ }
+
  return nil
 }
 
@@ -122,12 +160,12 @@ func (r *ReconcileAgentToken) Reconcile(ctx context.Context, request reconcile.R
 
  if err != nil {
  if kerrors.IsNotFound(err) {
- klog.Infof("%s is not found. Deleting the secret from the hub.", request.NamespacedName)
+ klog.Infof("SA %s is not found. Deleting the secret from the hub.", request.NamespacedName)
 
  err := r.hubclient.Delete(context.TODO(), r.prepareAgentTokenSecret(""))
 
  if err != nil {
- klog.Error("Failed to delete the secret from the hub.")
+ klog.Errorf("Failed to delete the secret from the hub. err: %v", err)
  return reconcile.Result{RequeueAfter: requeuAfter * time.Minute}, err
  }
 
@@ -144,7 +182,7 @@ func (r *ReconcileAgentToken) Reconcile(ctx context.Context, request reconcile.R
 
  if token == "" {
  klog.Error("Failed to find the service account token.")
- return reconcile.Result{}, errors.New("failed to find the klusterlet agent addon service account token secret")
+ return reconcile.Result{RequeueAfter: requeuAfter * time.Minute}, errors.New("failed to find the klusterlet agent addon service account token secret")
  }
 
  // Prepare the secret to be created/updated in the managed cluster namespace on the hub
@@ -245,21 +283,21 @@ func (r *ReconcileAgentToken) getServiceAccountTokenSecret() string {
  // Grab application-manager service account
  sa := &corev1.ServiceAccount{}
 
- err := r.Client.Get(context.TODO(), types.NamespacedName{Name: "application-manager", Namespace: "open-cluster-management-agent-addon"}, sa)
+ err := r.Client.Get(context.TODO(), types.NamespacedName{Name: appAddonName, Namespace: appAddonNS}, sa)
  if err != nil {
  klog.Error(err.Error())
  return ""
  }
 
- // loop through secrets to find application-manager-dockercfg secret
+ // first loop through secret list from the application-manager SA to find application-manager-dockercfg secret
  for _, secret := range sa.Secrets {
  if strings.HasPrefix(secret.Name, "application-manager-dockercfg") {
  klog.Info("found the application-manager-dockercfg secret " + secret.Name)
 
  // application-manager-token secret is owned by the dockercfg secret
  dockerSecret := &corev1.Secret{}
 
- err = r.Client.Get(context.TODO(), types.NamespacedName{Name: secret.Name, Namespace: "open-cluster-management-agent-addon"}, dockerSecret)
+ err = r.Client.Get(context.TODO(), types.NamespacedName{Name: secret.Name, Namespace: appAddonNS}, dockerSecret)
  if err != nil {
  klog.Error(err.Error())
  return ""
@@ -272,7 +310,99 @@ func (r *ReconcileAgentToken) getServiceAccountTokenSecret() string {
  }
  }
 
- return ""
+ // If not found, check the secret associated to the application-manager SA
+ err, saToken := r.createOrUpdateApplicationManagerSecret(sa)
+ if err != nil {
+ klog.Error(err.Error())
+ return ""
+ }
+
+ return saToken
+}
+
+func (r *ReconcileAgentToken) createOrUpdateApplicationManagerSecret(sa *corev1.ServiceAccount) (error, string) {
+ ApplicationManagerSecretList := &corev1.SecretList{}
+ listopts := &client.ListOptions{
+ Namespace: sa.Namespace,
+ }
+ err := r.Client.List(context.TODO(), ApplicationManagerSecretList, listopts)
+
+ if err != nil {
+ klog.Errorf("failed to list all secrets in application-manager SA NS: %v, err: %v", sa.Namespace, err)
+ return err, ""
+ }
+
+ for _, ApplicationManagerSecret := range ApplicationManagerSecretList.Items {
+ if ApplicationManagerSecret.Type != "kubernetes.io/service-account-token" {
+ continue
+ }
+
+ if ApplicationManagerSecret.Annotations != nil && ApplicationManagerSecret.Annotations["kubernetes.io/service-account.name"] == appAddonName {
+ // ApplicationManagerSecret.Data["token"] has been base64 decoded when fetched
+ if ApplicationManagerSecret.Data["token"] == nil {
+ klog.Errorf("secret token is empty, secret: %v/%v", ApplicationManagerSecret.Namespace, ApplicationManagerSecret.Name)
+ continue
+ }
+
+ return nil, string(ApplicationManagerSecret.Data["token"])
+ }
+ }
+
+ // if no secret is associated to the application-manager SA, create/update the application-manager secret associated to the application-manager SA
+ ApplicationManagerSecret := &corev1.Secret{}
+ ApplicationManagerSecretName := types.NamespacedName{Namespace: appAddonNS, Name: appAddonName}
+ err = r.Client.Get(context.TODO(), ApplicationManagerSecretName, ApplicationManagerSecret)
+
+ // if there exists the secret with the application-manager name, it is not associated to the application-manager SA, need to delete and re-create it
+ if err == nil {
+ err = r.Client.Delete(context.TODO(), ApplicationManagerSecret)
+ if err != nil {
+ klog.Errorf("failed to delete the invalid application-manager secret, err: %v", err)
+ return err, ""
+ }
+ }
+
+ // create the application-manager secret associated to the application-manager SA
+ saGVK := schema.GroupVersionKind{
+ Group: "",
+ Version: "v1",
+ Kind: "ServiceAccount",
+ }
+
+ owner := metav1.NewControllerRef(sa, saGVK)
+ ApplicationManagerSecret = &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: appAddonName,
+ Namespace: appAddonNS,
+ Annotations: map[string]string{
+ "kubernetes.io/service-account.name": appAddonName,
+ },
+ OwnerReferences: []metav1.OwnerReference{*owner},
+ },
+ Type: "kubernetes.io/service-account-token",
+ }
+
+ err = r.Create(context.TODO(), ApplicationManagerSecret)
+ if err != nil {
+ klog.Errorf("failed to create the new application-manager secret, err: %v", err)
+ return err, ""
+ }
+
+ klog.Infof("Application-manager secret %v created", ApplicationManagerSecretName)
+
+ newApplicationManagerSecret := &corev1.Secret{}
+ err = r.Client.Get(context.TODO(), ApplicationManagerSecretName, newApplicationManagerSecret)
+
+ if err != nil {
+ klog.Errorf("failed to get the new application-manager secret, err: %v", err)
+ return err, ""
+ }
+
+ if newApplicationManagerSecret.Data["token"] == nil {
+ return fmt.Errorf("new secret token is empty, secret: %v/%v", ApplicationManagerSecret.Namespace, ApplicationManagerSecret.Name), ""
+ }
+
+ return nil, string(newApplicationManagerSecret.Data["token"])
 }
 
 // getKubeAPIServerAddress - Get the API server address from OpenShift kubernetes cluster. This does not work with other kubernetes.
@@ -285,3 +415,58 @@ func (r *ReconcileAgentToken) getKubeAPIServerAddress() (string, error) {
 
  return infraConfig.Status.APIServerURL, nil
 }
+
+// detect if there is any change to the secret associated to the open-cluster-management-agent-addon/application-manager SA.
+var applicationManagerSecretPredicateFunctions = predicate.Funcs{
+ UpdateFunc: func(e event.UpdateEvent) bool {
+ newSecret, ok := e.ObjectNew.(*corev1.Secret)
+ if !ok {
+ return false
+ }
+
+ if newSecret.Namespace != appAddonNS {
+ return false
+ }
+
+ if newSecret.Type == "kubernetes.io/service-account-token" &&
+ newSecret.GetAnnotations()["kubernetes.io/service-account.name"] == appAddonName {
+ return true
+ }
+
+ return false
+ },
+ CreateFunc: func(e event.CreateEvent) bool {
+ newSecret, ok := e.Object.(*corev1.Secret)
+ if !ok {
+ return false
+ }
+
+ if newSecret.Namespace != appAddonNS {
+ return false
+ }
+
+ if newSecret.Type == "kubernetes.io/service-account-token" &&
+ newSecret.GetAnnotations()["kubernetes.io/service-account.name"] == appAddonName {
+ return true
+ }
+
+ return false
+ },
+ DeleteFunc: func(e event.DeleteEvent) bool {
+ newSecret, ok := e.Object.(*corev1.Secret)
+ if !ok {
+ return false
+ }
+
+ if newSecret.Namespace != appAddonNS {
+ return false
+ }
+
+ if newSecret.Type == "kubernetes.io/service-account-token" &&
+ newSecret.GetAnnotations()["kubernetes.io/service-account.name"] == appAddonName {
+ return true
+ }
+
+ return false
+ },
+}