From ea88b102e5ae04272efb17187d01af3606e29ae9 Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Mon, 23 Jan 2023 14:26:44 +0100 Subject: [PATCH] feat: pv labelling and preventing cross-tenant mount --- controllers/pv/controller.go | 117 ++++++++++++++++++++++++++++++++++ main.go | 10 ++- pkg/webhook/pvc/errors.go | 48 ++++++++++++++ pkg/webhook/pvc/pv.go | 98 ++++++++++++++++++++++++++++ pkg/webhook/pvc/validating.go | 12 ++-- 5 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 controllers/pv/controller.go create mode 100644 pkg/webhook/pvc/pv.go diff --git a/controllers/pv/controller.go b/controllers/pv/controller.go new file mode 100644 index 00000000..3ef3ebae --- /dev/null +++ b/controllers/pv/controller.go @@ -0,0 +1,117 @@ +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package pv + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + log2 "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/clastix/capsule/api/v1beta2" + capsuleutils "github.com/clastix/capsule/pkg/utils" + webhookutils "github.com/clastix/capsule/pkg/webhook/utils" +) + +type Controller struct { + client client.Client + label string +} + +func (c *Controller) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := log2.FromContext(ctx) + + persistentVolume := corev1.PersistentVolume{} + if err := c.client.Get(ctx, request.NamespacedName, &persistentVolume); err != nil { + if errors.IsNotFound(err) { + log.Info("skipping reconciliation, resource may have been deleted") + + return reconcile.Result{}, nil + } + + log.Error(err, "cannot retrieve corev1.PersistentVolume") + + return reconcile.Result{}, err + } + + if persistentVolume.Spec.ClaimRef == nil { + log.Info("skipping reconciliation, missing claimRef") + + return reconcile.Result{}, nil + } + + tnt, err := webhookutils.TenantByStatusNamespace(ctx, c.client, persistentVolume.Spec.ClaimRef.Namespace) + if err != nil { + log.Error(err, "unable to retrieve Tenant from the claimRef") + + return reconcile.Result{}, err + } + + if tnt == nil { + log.Info("skipping reconciliation, PV is claimed by a PVC not managed in a Tenant") + + return reconcile.Result{}, nil + } + + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + pv := persistentVolume + + if err = c.client.Get(ctx, request.NamespacedName, &pv); err != nil { + return err + } + + labels := pv.GetLabels() + if labels == nil { + labels = map[string]string{} + } + + labels[c.label] = tnt.GetName() + + pv.SetLabels(labels) + + return c.client.Update(ctx, &pv) + }) + if retryErr != nil { + log.Error(retryErr, "unable to update PersistentVolume with Capsule label") + + return reconcile.Result{}, retryErr + } + + return reconcile.Result{}, nil +} + +func (c *Controller) SetupWithManager(mgr ctrl.Manager) error { + label, err := capsuleutils.GetTypeLabel(&capsulev1beta2.Tenant{}) + if err != nil { + return err + } + + c.client = mgr.GetClient() + c.label = label + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.PersistentVolume{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + pv, ok := object.(*corev1.PersistentVolume) + if !ok { + return false + } + + if pv.Spec.ClaimRef == nil { + return false + } + + labels := object.GetLabels() + _, ok = labels[c.label] + + return !ok + }))). + Complete(c) +} diff --git a/main.go b/main.go index 323f66a1..a69e8de4 100644 --- a/main.go +++ b/main.go @@ -30,6 +30,7 @@ import ( capsulev1beta1 "github.com/clastix/capsule/api/v1beta1" capsulev1beta2 "github.com/clastix/capsule/api/v1beta2" configcontroller "github.com/clastix/capsule/controllers/config" + "github.com/clastix/capsule/controllers/pv" rbaccontroller "github.com/clastix/capsule/controllers/rbac" "github.com/clastix/capsule/controllers/resources" servicelabelscontroller "github.com/clastix/capsule/controllers/servicelabels" @@ -95,7 +96,7 @@ func newDelegatingClient(cache cache.Cache, config *rest.Config, options client. return delegatingClient, nil } -// nolint:maintidx +// nolint:maintidx,cyclop func main() { var enableLeaderElection, version bool @@ -239,7 +240,7 @@ func main() { route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass(), pod.RuntimeClass()), route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.PatchHandler(), namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())), route.Ingress(ingress.Class(cfg, kubeVersion), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), - route.PVC(pvc.Handler()), + route.PVC(pvc.Validating(), pvc.PersistentVolumeReuse()), route.Service(service.Handler()), route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())), route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()), @@ -296,6 +297,11 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") } + if err = (&pv.Controller{}).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "PersistentVolume") + os.Exit(1) + } + if err = (&configcontroller.Manager{ Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"), }).SetupWithManager(manager, configurationName); err != nil { diff --git a/pkg/webhook/pvc/errors.go b/pkg/webhook/pvc/errors.go index f02789ae..11337172 100644 --- a/pkg/webhook/pvc/errors.go +++ b/pkg/webhook/pvc/errors.go @@ -43,3 +43,51 @@ func (f storageClassForbiddenError) Error() string { return utils.DefaultAllowedValuesErrorMessage(f.spec, msg) } + +type missingPVLabelsError struct { + name string +} + +func NewMissingPVLabelsError(name string) error { + return &missingPVLabelsError{name: name} +} + +func (m missingPVLabelsError) Error() string { + return fmt.Sprintf("PeristentVolume %s is missing any label, please, ask the Cluster Administrator to label it", m.name) +} + +type missingPVTenantLabelsError struct { + name string +} + +func NewMissingTenantPVLabelsError(name string) error { + return &missingPVTenantLabelsError{name: name} +} + +func (m missingPVTenantLabelsError) Error() string { + return fmt.Sprintf("PeristentVolume %s is missing the Capsule Tenant label, preventing a potential cross-tenant mount", m.name) +} + +type crossTenantPVMountError struct { + name string +} + +func NewCrossTenantPVMountError(name string) error { + return &crossTenantPVMountError{ + name: name, + } +} + +func (m crossTenantPVMountError) Error() string { + return fmt.Sprintf("PeristentVolume %s cannot be used by the following Tenant, preventing a cross-tenant mount", m.name) +} + +type pvSelectorError struct{} + +func NewPVSelectorError() error { + return &pvSelectorError{} +} + +func (m pvSelectorError) Error() string { + return "PersistentVolume selectors are not allowed since unable to prevent cross-tenant mount" +} diff --git a/pkg/webhook/pvc/pv.go b/pkg/webhook/pvc/pv.go new file mode 100644 index 00000000..06d65791 --- /dev/null +++ b/pkg/webhook/pvc/pv.go @@ -0,0 +1,98 @@ +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package pvc + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/clastix/capsule/api/v1beta2" + capsulewebhook "github.com/clastix/capsule/pkg/webhook" + "github.com/clastix/capsule/pkg/webhook/utils" +) + +type PV struct { + capsuleLabel string +} + +func PersistentVolumeReuse() capsulewebhook.Handler { + value, err := capsulev1beta2.GetTypeLabel(&capsulev1beta2.Tenant{}) + if err != nil { + panic(fmt.Sprintf("this shouldn't happen: %s", err.Error())) + } + + return &PV{ + capsuleLabel: value, + } +} + +func (p PV) OnCreate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + pvc := corev1.PersistentVolumeClaim{} + if err := decoder.Decode(req, &pvc); err != nil { + return utils.ErroredResponse(err) + } + + tnt, err := utils.TenantByStatusNamespace(ctx, client, pvc.GetNamespace()) + if err != nil { + return utils.ErroredResponse(err) + } + // PVC is not in a Tenant Namespace, skipping + if tnt == nil { + return nil + } + // A PersistentVolume selector cannot help in preventing a cross-tenant mount: + // thus, disallowing that in first place. + if pvc.Spec.Selector != nil { + return utils.ErroredResponse(NewPVSelectorError()) + } + // The PVC hasn't any volumeName pre-claimed, it can be skipped + if len(pvc.Spec.VolumeName) == 0 { + return nil + } + // Checking if the PV is labelled with the Tenant name + pv := corev1.PersistentVolume{} + if err = client.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil { + if errors.IsNotFound(err) { + err = fmt.Errorf("cannot create a PVC referring to a not yet existing PV") + } + + return utils.ErroredResponse(err) + } + + if pv.GetLabels() == nil { + return utils.ErroredResponse(NewMissingPVLabelsError(pv.GetName())) + } + + value, ok := pv.GetLabels()[p.capsuleLabel] + if !ok { + return utils.ErroredResponse(NewMissingTenantPVLabelsError(pv.GetName())) + } + + if value != p.capsuleLabel { + return utils.ErroredResponse(NewCrossTenantPVMountError(pv.GetName())) + } + + return nil + } +} + +func (p PV) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (p PV) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} diff --git a/pkg/webhook/pvc/validating.go b/pkg/webhook/pvc/validating.go index 52d2dc70..03b6fb2c 100644 --- a/pkg/webhook/pvc/validating.go +++ b/pkg/webhook/pvc/validating.go @@ -17,13 +17,13 @@ import ( "github.com/clastix/capsule/pkg/webhook/utils" ) -type handler struct{} +type validating struct{} -func Handler() capsulewebhook.Handler { - return &handler{} +func Validating() capsulewebhook.Handler { + return &validating{} } -func (h *handler) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { +func (h *validating) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { pvc := &corev1.PersistentVolumeClaim{} if err := decoder.Decode(req, pvc); err != nil { @@ -87,13 +87,13 @@ func (h *handler) OnCreate(c client.Client, decoder *admission.Decoder, recorder } } -func (h *handler) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { +func (h *validating) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return nil } } -func (h *handler) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { +func (h *validating) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return nil }