diff --git a/controllers/tls/manager.go b/controllers/tls/manager.go index 7b8dfb76..29fb65d2 100644 --- a/controllers/tls/manager.go +++ b/controllers/tls/manager.go @@ -34,10 +34,9 @@ import ( ) const ( - certificateExpirationThreshold = 3 * 24 * time.Hour - certificateReconciliationThreshold = 4 * 24 * time.Hour - certificateValidity = 6 * 30 * 24 * time.Hour - PodUpdateAnnotationName = "capsule.clastix.io/updated" + certificateExpirationThreshold = 3 * 24 * time.Hour + certificateValidity = 6 * 30 * 24 * time.Hour + PodUpdateAnnotationName = "capsule.clastix.io/updated" ) type Reconciler struct { @@ -74,23 +73,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - - // Fetch the CA instance - certSecret := &corev1.Secret{} - - if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil { - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - +func (r Reconciler) ReconcileCertificates(ctx context.Context, certSecret *corev1.Secret) error { if r.shouldUpdateCertificate(certSecret) { r.Log.Info("Generating new TLS certificate") ca, err := cert.GenerateCertificateAuthority() if err != nil { - return reconcile.Result{}, err + return err } opts := cert.NewCertOpts(time.Now().Add(certificateValidity), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace)) @@ -99,7 +88,7 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R if err != nil { r.Log.Error(err, "Cannot generate new TLS certificate") - return reconcile.Result{}, err + return err } caCrt, _ := ca.CACertificatePem() @@ -120,7 +109,7 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R if err != nil { r.Log.Error(err, "cannot update Capsule TLS") - return reconcile.Result{}, err + return err } } @@ -129,12 +118,12 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R var ok bool if caBundle, ok = certSecret.Data[corev1.ServiceAccountRootCAKey]; !ok { - return reconcile.Result{}, fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName()) + return fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName()) } operatorPods, err := r.getOperatorPods(ctx) if err != nil { - return reconcile.Result{}, err + return err } r.Log.Info("Updating caBundle in webhooks and crd") @@ -161,6 +150,23 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R } if err := group.Wait(); err != nil { + return err + } + + return nil +} + +func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + + certSecret := &corev1.Secret{} + + if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil { + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + if err := r.ReconcileCertificates(ctx, certSecret); err != nil { return reconcile.Result{}, err } @@ -171,8 +177,8 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R } now := time.Now() - - rq := (time.Duration(certificate.NotAfter.Unix()-now.Unix()) * time.Second) - certificateReconciliationThreshold + requeueTime := certificate.NotAfter.Add(-(certificateExpirationThreshold - 1*time.Second)) + rq := requeueTime.Sub(now) r.Log.Info("Reconciliation completed, processing back in " + rq.String()) diff --git a/main.go b/main.go index 328fb6d2..724fabbe 100644 --- a/main.go +++ b/main.go @@ -11,12 +11,12 @@ import ( flag "github.com/spf13/pflag" "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilVersion "k8s.io/apimachinery/pkg/util/version" - "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" @@ -131,12 +131,6 @@ func main() { cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), configurationName) - clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) - if err != nil { - setupLog.Error(err, "unable to create kubernetes clientset") - os.Exit(1) - } - directClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{ Scheme: manager.GetScheme(), Mapper: manager.GetRESTMapper(), @@ -148,120 +142,125 @@ func main() { directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, configurationName) - if err = (&tlscontroller.Reconciler{ - Client: manager.GetClient(), + tlsReconciler := &tlscontroller.Reconciler{ + Client: directClient, Log: ctrl.Log.WithName("controllers").WithName("TLS"), Namespace: namespace, Configuration: directCfg, - }).SetupWithManager(manager); err != nil { + } + + if err = tlsReconciler.SetupWithManager(manager); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Namespace") os.Exit(1) } - tls, err := clientset.CoreV1().Secrets(namespace).Get(ctx, directCfg.TLSSecretName(), metav1.GetOptions{}) - if err != nil { + tlsCert := &corev1.Secret{} + + if err = directClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: directCfg.TLSSecretName()}, tlsCert); err != nil { setupLog.Error(err, "unable to get Capsule TLS secret") os.Exit(1) } - // nolint:nestif - if len(tls.Data) > 0 { - if err = (&tenantcontroller.Manager{ - RESTConfig: manager.GetConfig(), - Client: manager.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Tenant"), - Recorder: manager.GetEventRecorderFor("tenant-controller"), - }).SetupWithManager(manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Tenant") - os.Exit(1) - } - - if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil { - setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant") - os.Exit(1) - } - - if err = indexer.AddToManager(ctx, setupLog, manager); err != nil { - setupLog.Error(err, "unable to setup indexers") - os.Exit(1) - } - - var kubeVersion *utilVersion.Version - - if kubeVersion, err = utils.GetK8sVersion(); err != nil { - setupLog.Error(err, "unable to get kubernetes version") - os.Exit(1) - } - - // webhooks: the order matters, don't change it and just append - webhooksList := append( - make([]webhook.Webhook, 0), - route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()), - route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())), - route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), - route.PVC(pvc.Handler()), - route.Service(service.Handler()), - route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())), - route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()), - route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))), - route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()), - route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), - ) - - nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion) - if !nodeWebhookSupported { - setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735") - } - - if err = webhook.Register(manager, webhooksList...); err != nil { - setupLog.Error(err, "unable to setup webhooks") - os.Exit(1) - } - - rbacManager := &rbaccontroller.Manager{ - Log: ctrl.Log.WithName("controllers").WithName("Rbac"), - Configuration: cfg, - } - - if err = manager.Add(rbacManager); err != nil { - setupLog.Error(err, "unable to create cluster roles") - os.Exit(1) - } - - if err = rbacManager.SetupWithManager(ctx, manager, configurationName); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Rbac") - os.Exit(1) - } - - if err = (&servicelabelscontroller.ServicesLabelsReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"), - }).SetupWithManager(ctx, manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels") - os.Exit(1) - } - - if err = (&servicelabelscontroller.EndpointsLabelsReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"), - }).SetupWithManager(ctx, manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels") - os.Exit(1) - } - - if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"), - VersionMinor: kubeVersion.Minor(), - VersionMajor: kubeVersion.Major(), - }).SetupWithManager(ctx, manager); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") - } - - if err = (&configcontroller.Manager{ - Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"), - }).SetupWithManager(manager, configurationName); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") - os.Exit(1) - } - } else { - setupLog.Info("skip registering a tenant controller, missing CA secret") + + // Reconcile TLS certificates before starting controllers and webhooks + if err = tlsReconciler.ReconcileCertificates(ctx, tlsCert); err != nil { + setupLog.Error(err, "unable to reconcile Capsule TLS secret") + os.Exit(1) + } + + if err = (&tenantcontroller.Manager{ + RESTConfig: manager.GetConfig(), + Client: manager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Tenant"), + Recorder: manager.GetEventRecorderFor("tenant-controller"), + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Tenant") + os.Exit(1) + } + + if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil { + setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant") + os.Exit(1) + } + + if err = indexer.AddToManager(ctx, setupLog, manager); err != nil { + setupLog.Error(err, "unable to setup indexers") + os.Exit(1) + } + + var kubeVersion *utilVersion.Version + + if kubeVersion, err = utils.GetK8sVersion(); err != nil { + setupLog.Error(err, "unable to get kubernetes version") + os.Exit(1) + } + + // webhooks: the order matters, don't change it and just append + webhooksList := append( + make([]webhook.Webhook, 0), + route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()), + route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())), + route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), + route.PVC(pvc.Handler()), + route.Service(service.Handler()), + route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())), + route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()), + route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))), + route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()), + route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), + ) + + nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion) + if !nodeWebhookSupported { + setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735") + } + + if err = webhook.Register(manager, webhooksList...); err != nil { + setupLog.Error(err, "unable to setup webhooks") + os.Exit(1) + } + + rbacManager := &rbaccontroller.Manager{ + Log: ctrl.Log.WithName("controllers").WithName("Rbac"), + Configuration: cfg, + } + + if err = manager.Add(rbacManager); err != nil { + setupLog.Error(err, "unable to create cluster roles") + os.Exit(1) + } + + if err = rbacManager.SetupWithManager(ctx, manager, configurationName); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Rbac") + os.Exit(1) + } + + if err = (&servicelabelscontroller.ServicesLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"), + }).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels") + os.Exit(1) + } + + if err = (&servicelabelscontroller.EndpointsLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"), + }).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels") + os.Exit(1) + } + + if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"), + VersionMinor: kubeVersion.Minor(), + VersionMajor: kubeVersion.Major(), + }).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") + } + + if err = (&configcontroller.Manager{ + Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"), + }).SetupWithManager(manager, configurationName); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") + os.Exit(1) } setupLog.Info("starting manager") diff --git a/pkg/webhook/router.go b/pkg/webhook/router.go index e0df17b7..3f2ab521 100644 --- a/pkg/webhook/router.go +++ b/pkg/webhook/router.go @@ -5,7 +5,6 @@ package webhook import ( "context" - "io/ioutil" admissionv1 "k8s.io/api/admission/v1" "k8s.io/client-go/tools/record" @@ -16,12 +15,6 @@ import ( ) func Register(manager controllerruntime.Manager, webhookList ...Webhook) error { - // skipping webhook setup if certificate is missing - certData, _ := ioutil.ReadFile("/tmp/k8s-webhook-server/serving-certs/tls.crt") - if len(certData) == 0 { - return nil - } - recorder := manager.GetEventRecorderFor("tenant-webhook") server := manager.GetWebhookServer()