From 23be5c31d6e9d66934b75f7bd90808ceed3900cd Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Tue, 1 Sep 2020 10:26:06 -0700 Subject: [PATCH] Allow deletion pvc when tenant is deleted (#268) Add finalizer for the tenant. --- operator-kustomize/cluster-role.yaml | 8 ++ .../crds/minio.min.io_tenants.yaml | 6 + pkg/apis/minio.min.io/v1/names.go | 3 + pkg/apis/minio.min.io/v1/types.go | 4 + pkg/controller/cluster/main-controller.go | 125 ++++++++++++++++++ .../statefulsets/minio-statefulset.go | 9 ++ 6 files changed, 155 insertions(+) diff --git a/operator-kustomize/cluster-role.yaml b/operator-kustomize/cluster-role.yaml index 0be5a396dcf..7761908d08a 100644 --- a/operator-kustomize/cluster-role.yaml +++ b/operator-kustomize/cluster-role.yaml @@ -3,6 +3,14 @@ kind: ClusterRole metadata: name: minio-operator-role rules: + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - update + - list - apiGroups: - "" resources: diff --git a/operator-kustomize/crds/minio.min.io_tenants.yaml b/operator-kustomize/crds/minio.min.io_tenants.yaml index c8e8f5f541a..e4ad7f693fa 100644 --- a/operator-kustomize/crds/minio.min.io_tenants.yaml +++ b/operator-kustomize/crds/minio.min.io_tenants.yaml @@ -479,6 +479,12 @@ spec: importance of a Pod relative to other Pods. This is applied to MinIO pods only. Refer Kubernetes documentation for details https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass type: string + purgePVCOnTenantDelete: + description: Delete PVCs on tenant deletion. Defaults to (false) preserving + PVCs if tenant is deleted. PVCs will need cleanup post tenant deletion + if not set to true. If set to true ALL PVs for the tenant will be + deleted. + type: boolean requestAutoCert: description: 'RequestAutoCert allows user to enable Kubernetes based TLS cert generation and signing as explained here: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/' diff --git a/pkg/apis/minio.min.io/v1/names.go b/pkg/apis/minio.min.io/v1/names.go index 39cbc2e9d3a..12b2b4045a0 100644 --- a/pkg/apis/minio.min.io/v1/names.go +++ b/pkg/apis/minio.min.io/v1/names.go @@ -39,6 +39,9 @@ const ConsoleContainerName = "console" // InitContainerImage name for init container. const InitContainerImage = "busybox:1.32" +// Finalizer name used for resources operator wants to track for deletion +const Finalizer = "finalizer.minio.min.io" + // MinIO Related Names // MinIOStatefulSetNameForZone returns the name for MinIO StatefulSet diff --git a/pkg/apis/minio.min.io/v1/types.go b/pkg/apis/minio.min.io/v1/types.go index d33f41ec6c8..b2819c23d0b 100644 --- a/pkg/apis/minio.min.io/v1/types.go +++ b/pkg/apis/minio.min.io/v1/types.go @@ -51,6 +51,10 @@ type TenantScheduler struct { type TenantSpec struct { // Definition for Cluster in given MinIO cluster Zones []Zone `json:"zones"` + // Delete PVCs on tenant deletion. Defaults to (false) preserving PVCs if tenant is deleted. PVCs will + // need cleanup post tenant deletion if not set to true. If set to true ALL PVs for the tenant will be deleted. + // +optional + PurgePVCOnTenantDelete bool `json:"purgePVCOnTenantDelete,omitempty"` // Image defines the Tenant Docker image. // +optional Image string `json:"image,omitempty"` diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index 7b2d1ba95d0..b909c1ce5be 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -705,6 +705,13 @@ func (c *Controller) syncHandler(key string) error { } // Set any required default values and init Global variables nsName := types.NamespacedName{Namespace: namespace, Name: name} + + // Update tenant before setting defaults + mi, err = c.applyFinalizer(ctx, mi) + if err != nil { + return err + } + mi.EnsureDefaults() miniov1.InitGlobals(mi) @@ -723,6 +730,16 @@ func (c *Controller) syncHandler(key string) error { return err } + // Process deletion + if !mi.ObjectMeta.DeletionTimestamp.IsZero() { + klog.Infof("deleting tenant:%s/%s", mi.Namespace, mi.Name) + err = c.processDelete(ctx, mi) + if err != nil { + return err + } + return err + } + // check if both auto certificate creation and external secret with certificate is passed, // this is an error as only one of this is allowed in one Tenant if mi.AutoCert() && (mi.ExternalCert() || mi.ExternalClientCert() || mi.KESExternalCert() || mi.ConsoleExternalCert()) { @@ -887,6 +904,13 @@ func (c *Controller) syncHandler(key string) error { return err } } + + // Set the PVC purge behavior needed. + err = c.processPurgePVCsOnDeleteFlag(ctx, mi) + if err != nil { + return err + } + } // If the StatefulSet is not controlled by this Tenant resource, we should log @@ -1296,3 +1320,104 @@ func (c *Controller) checkAndCreateConsoleCSR(ctx context.Context, nsName types. } return nil } + +// getPodsForTenant returns a list of pods running on a given tenant +func (c *Controller) getPodsForTenant(tenant *miniov1.Tenant) ([]corev1.Pod, error) { + pods, err := c.kubeClientSet.CoreV1().Pods(tenant.Namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", miniov1.TenantLabel, tenant.Name), + }) + if err != nil { + return nil, err + } + return pods.Items, nil +} + +// processPurgePVCsOnDelete sets the owner reference for PVC to the tenant. This allows automatic deletion +// of the PVCs when the tenant is deleted. +func (c *Controller) processPurgePVCsOnDeleteFlag(ctx context.Context, mi *miniov1.Tenant) error { + + pods, err := c.getPodsForTenant(mi) + if err != nil { + // No pods created for tenant + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + for _, pod := range pods { + for _, pvc := range pod.Spec.Volumes { + if pvc.PersistentVolumeClaim != nil { + p, err := c.kubeClientSet.CoreV1().PersistentVolumeClaims(mi.Namespace).Get(ctx, pvc.PersistentVolumeClaim.ClaimName, metav1.GetOptions{}) + if err != nil { + // No claims found + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + update := true + if mi.Spec.PurgePVCOnTenantDelete { + // No need to add if already present + for _, o := range p.OwnerReferences { + if o.UID == mi.OwnerRef()[0].UID { + update = false + break + } + } + if update { + p.OwnerReferences = append(p.OwnerReferences, mi.OwnerRef()...) + } + } else { + update = false + // If UID is set remove and update + for i, o := range mi.OwnerReferences { + // Loop through ALL to remove the UID reference to tenant. + if o.UID == mi.OwnerRef()[0].UID { + p.OwnerReferences = append(p.OwnerReferences[:i], p.OwnerReferences[i+1:]...) + update = true + } + } + } + if update { + _, err = c.kubeClientSet.CoreV1().PersistentVolumeClaims(mi.Namespace).Update(ctx, p, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + } + } + } + return nil +} + +func (c *Controller) applyFinalizer(ctx context.Context, mi *miniov1.Tenant) (*miniov1.Tenant, error) { + found := false + for _, f := range mi.Finalizers { + if f == miniov1.Finalizer { + found = true + break + } + } + var err error + if !found { + mi.ObjectMeta.Finalizers = append(mi.ObjectMeta.Finalizers, miniov1.Finalizer) + mi, err = c.minioClientSet.MinioV1().Tenants(mi.Namespace).Update(ctx, mi, metav1.UpdateOptions{}) + } + return mi, err +} + +func (c *Controller) processDelete(ctx context.Context, mi *miniov1.Tenant) error { + // Set the PVC purge behavior needed. This will ensure that k8s correctly deletes all PVCs. + err := c.processPurgePVCsOnDeleteFlag(ctx, mi) + if err != nil { + return err + } + // Remove finalizer + for i, f := range mi.ObjectMeta.Finalizers { + if f == miniov1.Finalizer { + mi.ObjectMeta.Finalizers = append(mi.Finalizers[:i], mi.ObjectMeta.Finalizers[i+1:]...) + } + } + _, err = c.minioClientSet.MinioV1().Tenants(mi.Namespace).Update(ctx, mi, metav1.UpdateOptions{}) + return err +} diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index f133f311fb2..4333fb7884b 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -447,6 +447,15 @@ func NewForMinIOZone(t *miniov1.Tenant, wsSecret *v1.Secret, zone *miniov1.Zone, if zone.VolumeClaimTemplate != nil { pvClaim := *zone.VolumeClaimTemplate + if t.Spec.PurgePVCOnTenantDelete { + pvClaim.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(t, schema.GroupVersionKind{ + Group: miniov1.SchemeGroupVersion.Group, + Version: miniov1.SchemeGroupVersion.Version, + Kind: miniov1.MinIOCRDResourceKind, + }), + } + } name := pvClaim.Name for i := 0; i < int(zone.VolumesPerServer); i++ { pvClaim.Name = name + strconv.Itoa(i)