Skip to content

Commit

Permalink
feat: generate TLS certificates before starting controllers
Browse files Browse the repository at this point in the history
  • Loading branch information
@prometherion
Maksim Fedotov authored and prometherion committed Jun 8, 2022
1 parent 3738118 commit f1dc028
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 141 deletions.
50 changes: 28 additions & 22 deletions controllers/tls/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,9 @@ import (
)

const (
certificateExpirationThreshold = 3 * 24 * time.Hour
certificateReconciliationThreshold = 4 * 24 * time.Hour
certificateValidity = 6 * 30 * 24 * time.Hour
PodUpdateAnnotationName = "capsule.clastix.io/updated"
certificateExpirationThreshold = 3 * 24 * time.Hour
certificateValidity = 6 * 30 * 24 * time.Hour
PodUpdateAnnotationName = "capsule.clastix.io/updated"
)

type Reconciler struct {
Expand Down Expand Up @@ -74,23 +73,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}

func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)

// Fetch the CA instance
certSecret := &corev1.Secret{}

if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil {
// Error reading the object - requeue the request.
return reconcile.Result{}, err
}

func (r Reconciler) ReconcileCertificates(ctx context.Context, certSecret *corev1.Secret) error {
if r.shouldUpdateCertificate(certSecret) {
r.Log.Info("Generating new TLS certificate")

ca, err := cert.GenerateCertificateAuthority()
if err != nil {
return reconcile.Result{}, err
return err
}

opts := cert.NewCertOpts(time.Now().Add(certificateValidity), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace))
Expand All @@ -99,7 +88,7 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
if err != nil {
r.Log.Error(err, "Cannot generate new TLS certificate")

return reconcile.Result{}, err
return err
}

caCrt, _ := ca.CACertificatePem()
Expand All @@ -120,7 +109,7 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
if err != nil {
r.Log.Error(err, "cannot update Capsule TLS")

return reconcile.Result{}, err
return err
}
}

Expand All @@ -129,12 +118,12 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
var ok bool

if caBundle, ok = certSecret.Data[corev1.ServiceAccountRootCAKey]; !ok {
return reconcile.Result{}, fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
return fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
}

operatorPods, err := r.getOperatorPods(ctx)
if err != nil {
return reconcile.Result{}, err
return err
}

r.Log.Info("Updating caBundle in webhooks and crd")
Expand All @@ -161,6 +150,23 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
}

if err := group.Wait(); err != nil {
return err
}

return nil
}

func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)

certSecret := &corev1.Secret{}

if err := r.Client.Get(ctx, request.NamespacedName, certSecret); err != nil {
// Error reading the object - requeue the request.
return reconcile.Result{}, err
}

if err := r.ReconcileCertificates(ctx, certSecret); err != nil {
return reconcile.Result{}, err
}

Expand All @@ -171,8 +177,8 @@ func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.R
}

now := time.Now()

rq := (time.Duration(certificate.NotAfter.Unix()-now.Unix()) * time.Second) - certificateReconciliationThreshold
requeueTime := certificate.NotAfter.Add(-(certificateExpirationThreshold - 1*time.Second))
rq := requeueTime.Sub(now)

r.Log.Info("Reconciliation completed, processing back in " + rq.String())

Expand Down
223 changes: 111 additions & 112 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ import (

flag "github.com/spf13/pflag"
"go.uber.org/zap/zapcore"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilVersion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
ctrl "sigs.k8s.io/controller-runtime"
Expand Down Expand Up @@ -131,12 +131,6 @@ func main() {

cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), configurationName)

clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie())
if err != nil {
setupLog.Error(err, "unable to create kubernetes clientset")
os.Exit(1)
}

directClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{
Scheme: manager.GetScheme(),
Mapper: manager.GetRESTMapper(),
Expand All @@ -148,120 +142,125 @@ func main() {

directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, configurationName)

if err = (&tlscontroller.Reconciler{
Client: manager.GetClient(),
tlsReconciler := &tlscontroller.Reconciler{
Client: directClient,
Log: ctrl.Log.WithName("controllers").WithName("TLS"),
Namespace: namespace,
Configuration: directCfg,
}).SetupWithManager(manager); err != nil {
}

if err = tlsReconciler.SetupWithManager(manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Namespace")
os.Exit(1)
}

tls, err := clientset.CoreV1().Secrets(namespace).Get(ctx, directCfg.TLSSecretName(), metav1.GetOptions{})
if err != nil {
tlsCert := &corev1.Secret{}

if err = directClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: directCfg.TLSSecretName()}, tlsCert); err != nil {
setupLog.Error(err, "unable to get Capsule TLS secret")
os.Exit(1)
}
// nolint:nestif
if len(tls.Data) > 0 {
if err = (&tenantcontroller.Manager{
RESTConfig: manager.GetConfig(),
Client: manager.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("Tenant"),
Recorder: manager.GetEventRecorderFor("tenant-controller"),
}).SetupWithManager(manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Tenant")
os.Exit(1)
}

if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil {
setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant")
os.Exit(1)
}

if err = indexer.AddToManager(ctx, setupLog, manager); err != nil {
setupLog.Error(err, "unable to setup indexers")
os.Exit(1)
}

var kubeVersion *utilVersion.Version

if kubeVersion, err = utils.GetK8sVersion(); err != nil {
setupLog.Error(err, "unable to get kubernetes version")
os.Exit(1)
}

// webhooks: the order matters, don't change it and just append
webhooksList := append(
make([]webhook.Webhook, 0),
route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()),
route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())),
route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
route.PVC(pvc.Handler()),
route.Service(service.Handler()),
route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())),
route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()),
route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))),
route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()),
route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))),
)

nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion)
if !nodeWebhookSupported {
setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735")
}

if err = webhook.Register(manager, webhooksList...); err != nil {
setupLog.Error(err, "unable to setup webhooks")
os.Exit(1)
}

rbacManager := &rbaccontroller.Manager{
Log: ctrl.Log.WithName("controllers").WithName("Rbac"),
Configuration: cfg,
}

if err = manager.Add(rbacManager); err != nil {
setupLog.Error(err, "unable to create cluster roles")
os.Exit(1)
}

if err = rbacManager.SetupWithManager(ctx, manager, configurationName); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Rbac")
os.Exit(1)
}

if err = (&servicelabelscontroller.ServicesLabelsReconciler{
Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"),
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels")
os.Exit(1)
}

if err = (&servicelabelscontroller.EndpointsLabelsReconciler{
Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"),
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels")
os.Exit(1)
}

if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{
Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"),
VersionMinor: kubeVersion.Minor(),
VersionMajor: kubeVersion.Major(),
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels")
}

if err = (&configcontroller.Manager{
Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"),
}).SetupWithManager(manager, configurationName); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration")
os.Exit(1)
}
} else {
setupLog.Info("skip registering a tenant controller, missing CA secret")

// Reconcile TLS certificates before starting controllers and webhooks
if err = tlsReconciler.ReconcileCertificates(ctx, tlsCert); err != nil {
setupLog.Error(err, "unable to reconcile Capsule TLS secret")
os.Exit(1)
}

if err = (&tenantcontroller.Manager{
RESTConfig: manager.GetConfig(),
Client: manager.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("Tenant"),
Recorder: manager.GetEventRecorderFor("tenant-controller"),
}).SetupWithManager(manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Tenant")
os.Exit(1)
}

if err = (&capsulev1alpha1.Tenant{}).SetupWebhookWithManager(manager); err != nil {
setupLog.Error(err, "unable to create conversion webhook", "webhook", "Tenant")
os.Exit(1)
}

if err = indexer.AddToManager(ctx, setupLog, manager); err != nil {
setupLog.Error(err, "unable to setup indexers")
os.Exit(1)
}

var kubeVersion *utilVersion.Version

if kubeVersion, err = utils.GetK8sVersion(); err != nil {
setupLog.Error(err, "unable to get kubernetes version")
os.Exit(1)
}

// webhooks: the order matters, don't change it and just append
webhooksList := append(
make([]webhook.Webhook, 0),
route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()),
route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())),
route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
route.PVC(pvc.Handler()),
route.Service(service.Handler()),
route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())),
route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()),
route.OwnerReference(utils.InCapsuleGroups(cfg, ownerreference.Handler(cfg))),
route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler()),
route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))),
)

nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion)
if !nodeWebhookSupported {
setupLog.Info("Disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735")
}

if err = webhook.Register(manager, webhooksList...); err != nil {
setupLog.Error(err, "unable to setup webhooks")
os.Exit(1)
}

rbacManager := &rbaccontroller.Manager{
Log: ctrl.Log.WithName("controllers").WithName("Rbac"),
Configuration: cfg,
}

if err = manager.Add(rbacManager); err != nil {
setupLog.Error(err, "unable to create cluster roles")
os.Exit(1)
}

if err = rbacManager.SetupWithManager(ctx, manager, configurationName); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Rbac")
os.Exit(1)
}

if err = (&servicelabelscontroller.ServicesLabelsReconciler{
Log: ctrl.Log.WithName("controllers").WithName("ServiceLabels"),
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels")
os.Exit(1)
}

if err = (&servicelabelscontroller.EndpointsLabelsReconciler{
Log: ctrl.Log.WithName("controllers").WithName("EndpointLabels"),
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "EndpointLabels")
os.Exit(1)
}

if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{
Log: ctrl.Log.WithName("controllers").WithName("EndpointSliceLabels"),
VersionMinor: kubeVersion.Minor(),
VersionMajor: kubeVersion.Major(),
}).SetupWithManager(ctx, manager); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels")
}

if err = (&configcontroller.Manager{
Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"),
}).SetupWithManager(manager, configurationName); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration")
os.Exit(1)
}

setupLog.Info("starting manager")
Expand Down
7 changes: 0 additions & 7 deletions pkg/webhook/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package webhook

import (
"context"
"io/ioutil"

admissionv1 "k8s.io/api/admission/v1"
"k8s.io/client-go/tools/record"
Expand All @@ -16,12 +15,6 @@ import (
)

func Register(manager controllerruntime.Manager, webhookList ...Webhook) error {
// skipping webhook setup if certificate is missing
certData, _ := ioutil.ReadFile("/tmp/k8s-webhook-server/serving-certs/tls.crt")
if len(certData) == 0 {
return nil
}

recorder := manager.GetEventRecorderFor("tenant-webhook")

server := manager.GetWebhookServer()
Expand Down

0 comments on commit f1dc028

Please sign in to comment.