From 7f4d55f5519a97cca5bc3a066627eca01e4a1e7a Mon Sep 17 00:00:00 2001 From: Lukas Boettcher <1340215+lukasboettcher@users.noreply.github.com> Date: Fri, 24 May 2024 17:34:10 +0200 Subject: [PATCH] feat(metrics): provide metrics for tenant quotas Signed-off-by: Lukas Boettcher <1340215+lukasboettcher@users.noreply.github.com> --- controllers/tenant/cleanup.go | 50 ++++++++++++++++++++++++++++ controllers/tenant/manager.go | 9 +++++ controllers/tenant/resourcequotas.go | 26 +++++++++++++++ pkg/metrics/metrics.go | 30 +++++++++++++++++ 4 files changed, 115 insertions(+) create mode 100644 controllers/tenant/cleanup.go create mode 100644 pkg/metrics/metrics.go diff --git a/controllers/tenant/cleanup.go b/controllers/tenant/cleanup.go new file mode 100644 index 000000000..a173199f6 --- /dev/null +++ b/controllers/tenant/cleanup.go @@ -0,0 +1,50 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/metrics" +) + +func (r *Manager) handleMetricsCleanup(ctx context.Context, tenant *capsulev1beta2.Tenant) (err error) { + metricsFinalizer := "capsule.clastix.io/metrics-cleanup" + + // examine DeletionTimestamp to determine if object is under deletion + if tenant.ObjectMeta.DeletionTimestamp.IsZero() { //nolint:nestif + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // to registering our finalizer. + if !controllerutil.ContainsFinalizer(tenant, metricsFinalizer) { + controllerutil.AddFinalizer(tenant, metricsFinalizer) + + if err := r.Update(ctx, tenant); err != nil { + return err + } + } + } else { + // The object is being deleted + if controllerutil.ContainsFinalizer(tenant, metricsFinalizer) { + // our finalizer is present, so handle the removal of the metrics + r.Log.Info("Removing ResourceQuota metrics because Tenant " + tenant.Name + " is being deleted") + + // remove all metrics of the deleted tenant + metrics.TenantResourceUsage.DeletePartialMatch(map[string]string{"tenant": tenant.Name}) + metrics.TenantResourceLimit.DeletePartialMatch(map[string]string{"tenant": tenant.Name}) + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(tenant, metricsFinalizer) + + if err := r.Update(ctx, tenant); err != nil { + return err + } + } + } + + return nil +} diff --git a/controllers/tenant/manager.go b/controllers/tenant/manager.go index f797bfa54..6956743a5 100644 --- a/controllers/tenant/manager.go +++ b/controllers/tenant/manager.go @@ -133,6 +133,15 @@ func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ct return } + // Handle cleanup of metrics if the tenant was deleted + r.Log.Info("Setting up metrics cleanup") + + if err = r.handleMetricsCleanup(ctx, instance); err != nil { + r.Log.Error(err, "Cannot setup metrics cleanup") + + return + } + r.Log.Info("Tenant reconciling completed") return ctrl.Result{}, err diff --git a/controllers/tenant/resourcequotas.go b/controllers/tenant/resourcequotas.go index 5f5aca99d..2e5174040 100644 --- a/controllers/tenant/resourcequotas.go +++ b/controllers/tenant/resourcequotas.go @@ -23,6 +23,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/metrics" "github.com/projectcapsule/capsule/pkg/utils" ) @@ -51,6 +52,18 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 if typeLabel, err = utils.GetTypeLabel(&corev1.ResourceQuota{}); err != nil { return err } + + // Remove prior metrics, to avoid cleaning up for metrics of deleted ResourceQuotas + metrics.TenantResourceUsage.DeletePartialMatch(map[string]string{"tenant": tenant.Name}) + metrics.TenantResourceLimit.DeletePartialMatch(map[string]string{"tenant": tenant.Name}) + + // Expose the namespace quota and usage as metrics for the tenant + metrics.TenantResourceUsage.WithLabelValues(tenant.Name, "namespaces", "").Set(float64(tenant.Status.Size)) + + if tenant.Spec.NamespaceOptions != nil && tenant.Spec.NamespaceOptions.Quota != nil { + metrics.TenantResourceLimit.WithLabelValues(tenant.Name, "namespaces", "").Set(float64(*tenant.Spec.NamespaceOptions.Quota)) + } + //nolint:nestif if tenant.Spec.ResourceQuota.Scope == api.ResourceQuotaScopeTenant { group := new(errgroup.Group) @@ -102,6 +115,19 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 r.Log.Info("Computed " + name.String() + " quota for the whole Tenant is " + quantity.String()) + // Expose usage and limit metrics for the resource (name) of the ResourceQuota (index) + metrics.TenantResourceUsage.WithLabelValues( + tenant.Name, + name.String(), + strconv.Itoa(index), + ).Set(float64(quantity.MilliValue()) / 1000) + + metrics.TenantResourceLimit.WithLabelValues( + tenant.Name, + name.String(), + strconv.Itoa(index), + ).Set(float64(hardQuota.MilliValue()) / 1000) + switch quantity.Cmp(resourceQuota.Hard[name]) { case 0: // The Tenant is matching exactly the Quota: diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000..b9b06044a --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,30 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + metricsPrefix = "capsule_" + + TenantResourceUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: metricsPrefix + "tenant_resource_usage", + Help: "Current resource usage for a given resource in a tenant", + }, []string{"tenant", "resource", "resourcequotaindex"}) + + TenantResourceLimit = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: metricsPrefix + "tenant_resource_limit", + Help: "Current resource limit for a given resource in a tenant", + }, []string{"tenant", "resource", "resourcequotaindex"}) +) + +func init() { + metrics.Registry.MustRegister( + TenantResourceUsage, + TenantResourceLimit, + ) +}