diff --git a/Makefile b/Makefile index 7ccfe6c29..a068b8915 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,8 @@ dev-setup: {'op': 'replace', 'path': '/webhooks/5/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/pods\",'caBundle':\"$${CA_BUNDLE}\"}},\ {'op': 'replace', 'path': '/webhooks/6/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/persistentvolumeclaims\",'caBundle':\"$${CA_BUNDLE}\"}},\ {'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/services\",'caBundle':\"$${CA_BUNDLE}\"}},\ - {'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\ + {'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}},\ + {'op': 'replace', 'path': '/webhooks/9/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\ ]" && \ kubectl patch crd tenants.capsule.clastix.io \ --type='json' -p="[\ diff --git a/api/v1beta2/tenant_types.go b/api/v1beta2/tenant_types.go index 93e01b759..01c8e1a28 100644 --- a/api/v1beta2/tenant_types.go +++ b/api/v1beta2/tenant_types.go @@ -17,6 +17,8 @@ type TenantSpec struct { NamespaceOptions *NamespaceOptions `json:"namespaceOptions,omitempty"` // Specifies options for the Service, such as additional metadata or block of certain type of Services. Optional. ServiceOptions *api.ServiceOptions `json:"serviceOptions,omitempty"` + // Specifies options for the Pods, such additional metadata to enforce on each pod part of the tenant + PodOptions *api.PodOptions `json:"podOptions,omitempty"` // Specifies the allowed StorageClasses assigned to the Tenant. // Capsule assures that all PersistentVolumeClaim resources created in the Tenant can use only one of the allowed StorageClasses. // A default value can be specified, and all the PersistentVolumeClaim resources created will inherit the declared class. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 7245688f4..119085f48 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -716,6 +716,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = new(api.ServiceOptions) (*in).DeepCopyInto(*out) } + if in.PodOptions != nil { + in, out := &in.PodOptions, &out.PodOptions + *out = new(api.PodOptions) + (*in).DeepCopyInto(*out) + } if in.StorageClasses != nil { in, out := &in.StorageClasses, &out.StorageClasses *out = new(api.DefaultAllowedListSpec) diff --git a/charts/capsule/crds/tenant-crd.yaml b/charts/capsule/crds/tenant-crd.yaml index 314016fbd..8ae0d6ae1 100644 --- a/charts/capsule/crds/tenant-crd.yaml +++ b/charts/capsule/crds/tenant-crd.yaml @@ -632,6 +632,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. type: boolean @@ -1737,6 +1753,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object priorityClasses: description: Specifies the allowed priorityClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant @@ -2869,6 +2901,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. diff --git a/config/crd/bases/capsule.clastix.io_tenants.yaml b/config/crd/bases/capsule.clastix.io_tenants.yaml index 75c547236..7e1e00da5 100644 --- a/config/crd/bases/capsule.clastix.io_tenants.yaml +++ b/config/crd/bases/capsule.clastix.io_tenants.yaml @@ -2859,6 +2859,24 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pods, such additional metadata + to enforce on each pod part of the tenant + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule + operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. diff --git a/config/install.yaml b/config/install.yaml index d753fe848..6ffd10757 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -2437,6 +2437,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pods, such additional metadata to enforce on each pod part of the tenant + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. type: boolean diff --git a/controllers/podlabels/abstract.go b/controllers/podlabels/abstract.go new file mode 100644 index 000000000..00d442668 --- /dev/null +++ b/controllers/podlabels/abstract.go @@ -0,0 +1,117 @@ +package podlabels + +import ( + "context" + "fmt" + "github.com/go-logr/logr" + "github.com/projectcapsule/capsule/pkg/utils" + corev1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/pkg/errors" + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type abstractPodLabelsReconciler struct { + obj client.Object + client client.Client + log logr.Logger +} + +func (r *abstractPodLabelsReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + tenant, err := r.getTenant(ctx, request.NamespacedName, r.client) + if err != nil { + r.log.Error(err, fmt.Sprintf("Cannot get tenant for %T %s/%s", r.obj, request.Namespace, request.Name)) + + noTenantObjError := &NonTenantObjectError{} + noPodMetaError := &NoPodMetadataError{} + + if errors.As(err, &noTenantObjError) || errors.As(err, &noPodMetaError) { + return reconcile.Result{}, nil + } + + r.log.Error(err, fmt.Sprintf("Cannot sync %T %s/%s labels", r.obj, r.obj.GetNamespace(), r.obj.GetName())) + + return reconcile.Result{}, err + } + + err = r.client.Get(ctx, request.NamespacedName, r.obj) + if err != nil { + if apierr.IsNotFound(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, err + } + _, err = controllerutil.CreateOrUpdate(ctx, r.client, r.obj, func() (err error) { + r.obj.SetLabels(r.sync(r.obj.GetLabels(), tenant.Spec.PodOptions.AdditionalMetadata.Labels)) + r.obj.SetAnnotations(r.sync(r.obj.GetAnnotations(), tenant.Spec.PodOptions.AdditionalMetadata.Annotations)) + return nil + }) + return reconcile.Result{}, err +} + +func (r *abstractPodLabelsReconciler) getTenant(ctx context.Context, namespacedName types.NamespacedName, client client.Client) (*capsulev1beta2.Tenant, error) { + ns := &corev1.Namespace{} + tenant := &capsulev1beta2.Tenant{} + + if err := client.Get(ctx, types.NamespacedName{Name: namespacedName.Namespace}, ns); err != nil { + return nil, err + } + + capsuleLabel, _ := utils.GetTypeLabel(&capsulev1beta2.Tenant{}) + if _, ok := ns.GetLabels()[capsuleLabel]; !ok { + return nil, NewNonTenantObject(namespacedName.Name) + } + + if err := client.Get(ctx, types.NamespacedName{Name: ns.Labels[capsuleLabel]}, tenant); err != nil { + return nil, err + } + + if tenant.Spec.PodOptions == nil || tenant.Spec.PodOptions.AdditionalMetadata == nil { + return nil, NewNoPodMetadata(namespacedName.Name) + } + + return tenant, nil +} + +func (r *abstractPodLabelsReconciler) sync(available map[string]string, tenantSpec map[string]string) map[string]string { + if tenantSpec != nil { + if available == nil { + available = tenantSpec + } else { + for key, value := range tenantSpec { + if available[key] != value { + available[key] = value + } + } + } + } + + return available +} + +func (r *abstractPodLabelsReconciler) forOptionPerInstanceName(ctx context.Context) builder.ForOption { + return builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return r.IsNamespaceInTenant(ctx, object.GetNamespace()) + })) +} + +func (r *abstractPodLabelsReconciler) IsNamespaceInTenant(ctx context.Context, namespace string) bool { + tl := &capsulev1beta2.TenantList{} + if err := r.client.List(ctx, tl, client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(".status.namespaces", namespace), + }); err != nil { + return false + } + + return len(tl.Items) > 0 +} diff --git a/controllers/podlabels/errors.go b/controllers/podlabels/errors.go new file mode 100644 index 000000000..06ab8771c --- /dev/null +++ b/controllers/podlabels/errors.go @@ -0,0 +1,27 @@ +package podlabels + +import "fmt" + +type NonTenantObjectError struct { + objectName string +} + +func NewNonTenantObject(objectName string) error { + return &NonTenantObjectError{objectName: objectName} +} + +func (n NonTenantObjectError) Error() string { + return fmt.Sprintf("Skipping labels sync for %s as it doesn't belong to tenant", n.objectName) +} + +type NoPodMetadataError struct { + objectName string +} + +func NewNoPodMetadata(objectName string) error { + return &NoPodMetadataError{objectName: objectName} +} + +func (n NoPodMetadataError) Error() string { + return fmt.Sprintf("Skipping labels sync for %s because no AdditionalLabels or AdditionalAnnotations presents in Tenant spec", n.objectName) +} diff --git a/controllers/podlabels/pod.go b/controllers/podlabels/pod.go new file mode 100644 index 000000000..458c4ed0f --- /dev/null +++ b/controllers/podlabels/pod.go @@ -0,0 +1,26 @@ +package podlabels + +import ( + "context" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +type PodLabelsReconciler struct { + abstractPodLabelsReconciler + + Log logr.Logger +} + +func (r *PodLabelsReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + r.abstractPodLabelsReconciler = abstractPodLabelsReconciler{ + obj: &corev1.Pod{}, + client: mgr.GetClient(), + log: r.Log, + } + + return ctrl.NewControllerManagedBy(mgr). + For(r.abstractPodLabelsReconciler.obj, r.abstractPodLabelsReconciler.forOptionPerInstanceName(ctx)). + Complete(r) +} diff --git a/docs/content/general/crds-apis.md b/docs/content/general/crds-apis.md index 696a41eba..b222ebd2e 100644 --- a/docs/content/general/crds-apis.md +++ b/docs/content/general/crds-apis.md @@ -2963,6 +2963,13 @@ TenantSpec defines the desired state of Tenant. Specifies the label to control the placement of pods on a given pool of worker nodes. All namespaces created within the Tenant will have the node selector annotation. This annotation tells the Kubernetes scheduler to place pods on the nodes having the selector label. Optional.
false + + podOptions + object + + Specifies options for the Pods, such additional metadata to enforce on each pod part of the tenant
+ + false preventDeletion boolean @@ -4397,6 +4404,65 @@ NetworkPolicyPort describes a port to allow traffic on +### Tenant.spec.podOptions + + + +Specifies options for the Pods, such additional metadata to enforce on each pod part of the tenant + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
additionalMetadataobject + Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional.
+ 		false
+ + +### Tenant.spec.podOptions.additionalMetadata + + + +Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
annotationsmap[string]string +
+ 		false
labelsmap[string]string +
+ 		false
+ + ### Tenant.spec.priorityClasses diff --git a/e2e/pod_metadata_test.go b/e2e/pod_metadata_test.go new file mode 100644 index 000000000..98722cb75 --- /dev/null +++ b/e2e/pod_metadata_test.go @@ -0,0 +1,115 @@ +//go:build e2e + +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("adding metadata to Pod objects", func() { + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-metadata", + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: capsulev1beta2.OwnerListSpec{ + { + Name: "gatsby", + Kind: "User", + }, + }, + PodOptions: &api.PodOptions{ + AdditionalMetadata: &api.AdditionalMetadataSpec{ + Labels: map[string]string{ + "k8s.io/custom-label": "foo", + "clastix.io/custom-label": "bar", + }, + Annotations: map[string]string{ + "k8s.io/custom-annotation": "bizz", + "clastix.io/custom-annotation": "buzz", + }, + }, + }, + }, + } + + JustBeforeEach(func() { + EventuallyCreation(func() error { + + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + }) + + JustAfterEach(func() { + Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + }) + + It("should apply them to Pod", func() { + ns := NewNamespace("") + NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed()) + fmt.Sprint("namespace created") + //TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + fmt.Sprint("tenant contains list namespace") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-metadata", + Namespace: ns.GetName(), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + ImagePullPolicy: "IfNotPresent", + }, + }, + RestartPolicy: "Always", + }, + } + + EventuallyCreation(func() (err error) { + _, err = ownerClient(tnt.Spec.Owners[0]).CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{}) + + return + }).Should(Succeed()) + + By("checking additional labels", func() { + Eventually(func() (ok bool) { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: pod.GetName(), Namespace: ns.GetName()}, pod)).Should(Succeed()) + for k, v := range tnt.Spec.PodOptions.AdditionalMetadata.Labels { + ok, _ = HaveKeyWithValue(k, v).Match(pod.GetLabels()) + if !ok { + return false + } + } + return true + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + + By("checking additional annotations", func() { + Eventually(func() (ok bool) { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: pod.GetName(), Namespace: ns.GetName()}, pod)).Should(Succeed()) + for k, v := range tnt.Spec.PodOptions.AdditionalMetadata.Annotations { + ok, _ = HaveKeyWithValue(k, v).Match(pod.GetAnnotations()) + if !ok { + return false + } + } + return true + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + }) + +}) diff --git a/main.go b/main.go index ce6039796..464e232f5 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( capsulev1beta1 "github.com/projectcapsule/capsule/api/v1beta1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" configcontroller "github.com/projectcapsule/capsule/controllers/config" + podlabelscontroller "github.com/projectcapsule/capsule/controllers/podlabels" "github.com/projectcapsule/capsule/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/controllers/rbac" "github.com/projectcapsule/capsule/controllers/resources" @@ -291,6 +292,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") } + if err = (&podlabelscontroller.PodLabelsReconciler{ + Log: ctrl.Log.WithName("controllers").WithName("PodLabels"), + }).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "PodLabels") + os.Exit(1) + } + if err = (&pv.Controller{}).SetupWithManager(manager); err != nil { setupLog.Error(err, "unable to create controller", "controller", "PersistentVolume") os.Exit(1) diff --git a/pkg/api/pod_options.go b/pkg/api/pod_options.go new file mode 100644 index 000000000..cf00b122b --- /dev/null +++ b/pkg/api/pod_options.go @@ -0,0 +1,11 @@ +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package api + +// +kubebuilder:object:generate=true + +type PodOptions struct { + // Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + AdditionalMetadata *AdditionalMetadataSpec `json:"additionalMetadata,omitempty"` +} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 70c8265b9..d09c76c3e 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -213,6 +213,26 @@ func (in *NetworkPolicySpec) DeepCopy() *NetworkPolicySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodOptions) DeepCopyInto(out *PodOptions) { + *out = *in + if in.AdditionalMetadata != nil { + in, out := &in.AdditionalMetadata, &out.AdditionalMetadata + *out = new(AdditionalMetadataSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodOptions. +func (in *PodOptions) DeepCopy() *PodOptions { + if in == nil { + return nil + } + out := new(PodOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceQuotaSpec) DeepCopyInto(out *ResourceQuotaSpec) { *out = *in