From 2f86f4c9a8d30e21557198d489c0285aa84fbc59 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 12 Aug 2020 12:50:36 -0700 Subject: [PATCH] add webhook server implementation for tenants (#251) Current implementation provides a mechanism where the tenant loads an ENV such as its command line args remotely, this is to facilitate faster zone addition times. --- examples/tenant-console.yaml | 2 +- examples/tenant-kes.yaml | 2 +- examples/tenant-pod-security-policy.yaml | 2 +- go.mod | 2 + go.sum | 1 + kustomization.yaml | 3 +- operator-kustomize/service.yaml | 16 ++ pkg/apis/minio.min.io/v1/constants.go | 2 +- pkg/apis/minio.min.io/v1/helper.go | 15 ++ pkg/controller/cluster/console-csr.go | 2 +- pkg/controller/cluster/csr.go | 14 +- pkg/controller/cluster/kes-csr.go | 4 +- pkg/controller/cluster/main-controller.go | 215 ++++++++++++++---- pkg/controller/cluster/webhook-server.go | 65 ++++++ .../statefulsets/minio-statefulset.go | 31 ++- 15 files changed, 311 insertions(+), 65 deletions(-) create mode 100644 operator-kustomize/service.yaml create mode 100644 pkg/controller/cluster/webhook-server.go diff --git a/examples/tenant-console.yaml b/examples/tenant-console.yaml index d3accb33df8..b0f3107e546 100644 --- a/examples/tenant-console.yaml +++ b/examples/tenant-console.yaml @@ -37,7 +37,7 @@ spec: prometheus.io/port: "9000" prometheus.io/scrape: "true" ## Registry location and Tag to download MinIO Server image - image: minio/minio:RELEASE.2020-08-05T21-34-13Z + image: minio/minio:RELEASE.2020-08-08T04-50-06Z ## Secret with credentials to be used by MinIO instance. credsSecret: name: minio-creds-secret diff --git a/examples/tenant-kes.yaml b/examples/tenant-kes.yaml index 93f572cb802..58264b044b7 100644 --- a/examples/tenant-kes.yaml +++ b/examples/tenant-kes.yaml @@ -37,7 +37,7 @@ spec: prometheus.io/port: "9000" prometheus.io/scrape: "true" ## Registry location and Tag to download MinIO Server image - image: minio/minio:RELEASE.2020-08-05T21-34-13Z + image: minio/minio:RELEASE.2020-08-08T04-50-06Z ## Secret with credentials to be used by MinIO instance. credsSecret: name: minio-creds-secret diff --git a/examples/tenant-pod-security-policy.yaml b/examples/tenant-pod-security-policy.yaml index b626bbe3a5c..83389e8ceee 100644 --- a/examples/tenant-pod-security-policy.yaml +++ b/examples/tenant-pod-security-policy.yaml @@ -78,7 +78,7 @@ spec: prometheus.io/port: "9000" prometheus.io/scrape: "true" ## Registry location and Tag to download MinIO Server image - image: minio/minio:RELEASE.2020-08-05T21-34-13Z + image: minio/minio:RELEASE.2020-08-08T04-50-06Z ## Service account to be used for all the MinIO Pods serviceAccountName: minio-pods zones: diff --git a/go.mod b/go.mod index 2cccb66f6b9..ac069320a4b 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,11 @@ go 1.13 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/protobuf v1.4.2 // indirect github.com/google/go-cmp v0.4.1 // indirect + github.com/gorilla/mux v1.7.5-0.20200711200521-98cb6bf42e08 github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.10 // indirect github.com/minio/minio v0.0.0-20200723003940-b9be841fd222 diff --git a/go.sum b/go.sum index 2f962383e87..890d8ff38aa 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,7 @@ github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEo github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.5-0.20200711200521-98cb6bf42e08 h1:kPna6oIGlRXWmg/jkKfxbpvsl+0DHYnw1qQwN+6+gyA= github.com/gorilla/mux v1.7.5-0.20200711200521-98cb6bf42e08/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= diff --git a/kustomization.yaml b/kustomization.yaml index cb8c559600c..c3b6ecf73d1 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -5,7 +5,7 @@ kind: Kustomization images: - name: minio/k8s-operator newName: minio/k8s-operator - newTag: v3.0.10 + newTag: v3.0.11 namespace: minio-operator @@ -15,4 +15,5 @@ resources: - operator-kustomize/cluster-role.yaml - operator-kustomize/cluster-role-binding.yaml - operator-kustomize/crds/minio.min.io_tenants.yaml + - operator-kustomize/service.yaml - operator-kustomize/deployment.yaml diff --git a/operator-kustomize/service.yaml b/operator-kustomize/service.yaml new file mode 100644 index 00000000000..87e8d472451 --- /dev/null +++ b/operator-kustomize/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: operator + labels: + name: minio-operator + namespace: minio-operator +spec: + type: ClusterIP + ports: + - port: 4222 + name: http + - port: 4233 + name: https + selector: + name: minio-operator diff --git a/pkg/apis/minio.min.io/v1/constants.go b/pkg/apis/minio.min.io/v1/constants.go index c6b860cf1d3..cc3804a6c37 100644 --- a/pkg/apis/minio.min.io/v1/constants.go +++ b/pkg/apis/minio.min.io/v1/constants.go @@ -74,7 +74,7 @@ const MinIOVolumeMountPath = "/export" const MinIOVolumeSubPath = "" // DefaultMinIOImage specifies the default MinIO Docker hub image -const DefaultMinIOImage = "minio/minio:RELEASE.2020-08-05T21-34-13Z" +const DefaultMinIOImage = "minio/minio:RELEASE.2020-08-08T04-50-06Z" // DefaultMinIOUpdateURL specifies the default MinIO URL where binaries are // pulled from during MinIO upgrades diff --git a/pkg/apis/minio.min.io/v1/helper.go b/pkg/apis/minio.min.io/v1/helper.go index f90e43a853f..ecc88b12fcc 100644 --- a/pkg/apis/minio.min.io/v1/helper.go +++ b/pkg/apis/minio.min.io/v1/helper.go @@ -44,6 +44,21 @@ import ( "github.com/minio/minio/pkg/madmin" ) +// Webhook API constants +const ( + WebhookAPIVersion = "/webhook/v1" + WebhookDefaultPort = "4222" + WebhookOperatorSecret = "operator-webhook-secret" + WebhookOperatorUsername = "webhookUsername" + WebhookOperatorPassword = "webhookPassword" +) + +// List of webhook APIs +const ( + WebhookAPIGetenv = WebhookAPIVersion + "/getenv" + WebhookAPIBucketService = WebhookAPIVersion + "/bucketsrv" +) + type hostsTemplateValues struct { StatefulSet string CIService string diff --git a/pkg/controller/cluster/console-csr.go b/pkg/controller/cluster/console-csr.go index 2009346281e..fb417021fef 100644 --- a/pkg/controller/cluster/console-csr.go +++ b/pkg/controller/cluster/console-csr.go @@ -86,7 +86,7 @@ func (c *Controller) createConsoleTLSCSR(ctx context.Context, mi *miniov1.Tenant encodedPrivKey := pem.EncodeToMemory(&pem.Block{Type: privateKeyType, Bytes: privKeysBytes}) // Create secret for Console Deployment to use - err = c.createSecret(ctx, mi, mi.ConsolePodLabels(), mi.ConsoleTLSSecretName(), mi.Namespace, encodedPrivKey, certbytes) + err = c.createSecret(ctx, mi, mi.ConsolePodLabels(), mi.ConsoleTLSSecretName(), encodedPrivKey, certbytes) if err != nil { klog.Errorf("Unexpected error during the creation of the secret/%s: %v", mi.ConsoleTLSSecretName(), err) return err diff --git a/pkg/controller/cluster/csr.go b/pkg/controller/cluster/csr.go index 3872e3807dc..a4f5990b812 100644 --- a/pkg/controller/cluster/csr.go +++ b/pkg/controller/cluster/csr.go @@ -139,7 +139,7 @@ func (c *Controller) createCSR(ctx context.Context, mi *miniov1.Tenant) error { encodedPrivKey := pem.EncodeToMemory(&pem.Block{Type: privateKeyType, Bytes: privKeysBytes}) // Create secret for MinIO Statefulset to use - err = c.createSecret(ctx, mi, mi.MinIOPodLabels(), mi.MinIOTLSSecretName(), mi.Namespace, encodedPrivKey, certbytes) + err = c.createSecret(ctx, mi, mi.MinIOPodLabels(), mi.MinIOTLSSecretName(), encodedPrivKey, certbytes) if err != nil { klog.Errorf("Unexpected error during the creation of the secret/%s: %v", mi.MinIOTLSSecretName(), err) return err @@ -251,12 +251,12 @@ func (c *Controller) fetchCertificate(ctx context.Context, csrName string) ([]by } } -func (c *Controller) createSecret(ctx context.Context, mi *miniov1.Tenant, labels map[string]string, name, namespace string, pkBytes, certBytes []byte) error { +func (c *Controller) createSecret(ctx context.Context, mi *miniov1.Tenant, labels map[string]string, secretName string, pkBytes, certBytes []byte) error { secret := &corev1.Secret{ Type: "Opaque", ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, + Name: secretName, + Namespace: mi.Namespace, Labels: labels, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(mi, schema.GroupVersionKind{ @@ -271,10 +271,8 @@ func (c *Controller) createSecret(ctx context.Context, mi *miniov1.Tenant, label "public.crt": certBytes, }, } - if _, err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { - return err - } - return nil + _, err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Create(ctx, secret, metav1.CreateOptions{}) + return err } func parseCertificate(r io.Reader) (*x509.Certificate, error) { diff --git a/pkg/controller/cluster/kes-csr.go b/pkg/controller/cluster/kes-csr.go index 72748064ef3..15ca1bcb823 100644 --- a/pkg/controller/cluster/kes-csr.go +++ b/pkg/controller/cluster/kes-csr.go @@ -89,7 +89,7 @@ func (c *Controller) createKESTLSCSR(ctx context.Context, mi *miniov1.Tenant) er encodedPrivKey := pem.EncodeToMemory(&pem.Block{Type: privateKeyType, Bytes: privKeysBytes}) // Create secret for KES Statefulset to use - err = c.createSecret(ctx, mi, mi.KESPodLabels(), mi.KESTLSSecretName(), mi.Namespace, encodedPrivKey, certbytes) + err = c.createSecret(ctx, mi, mi.KESPodLabels(), mi.KESTLSSecretName(), encodedPrivKey, certbytes) if err != nil { klog.Errorf("Unexpected error during the creation of the secret/%s: %v", mi.KESTLSSecretName(), err) return err @@ -142,7 +142,7 @@ func (c *Controller) createMinIOClientTLSCSR(ctx context.Context, mi *miniov1.Te encodedPrivKey := pem.EncodeToMemory(&pem.Block{Type: privateKeyType, Bytes: privKeysBytes}) // Create secret for KES Statefulset to use - err = c.createSecret(ctx, mi, mi.MinIOPodLabels(), mi.MinIOClientTLSSecretName(), mi.Namespace, encodedPrivKey, certbytes) + err = c.createSecret(ctx, mi, mi.MinIOPodLabels(), mi.MinIOClientTLSSecretName(), encodedPrivKey, certbytes) if err != nil { klog.Errorf("Unexpected error during the creation of the secret/%s: %v", mi.MinIOClientTLSSecretName(), err) return err diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index 6fb85370d32..5aee687e0fb 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -23,6 +23,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "net/http" "strings" "time" @@ -30,10 +31,12 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -51,7 +54,10 @@ import ( "k8s.io/client-go/tools/record" queue "k8s.io/client-go/util/workqueue" - "github.com/minio/minio-go/v7/pkg/set" + "github.com/dgrijalva/jwt-go" + jwtreq "github.com/dgrijalva/jwt-go/request" + "github.com/gorilla/mux" + "github.com/minio/minio/pkg/auth" miniov1 "github.com/minio/operator/pkg/apis/minio.min.io/v1" clientset "github.com/minio/operator/pkg/client/clientset/versioned" minioscheme "github.com/minio/operator/pkg/client/clientset/versioned/scheme" @@ -89,7 +95,6 @@ const ( statusWaitingKESCert = "Waiting for KES TLS Certificate" statusWaitingConsoleCert = "Waiting for Console TLS Certificate" statusUpdatingMinIOVersion = "Updating MinIO Version" - statusUpdatingContainerArguments = "Updating Container Arguments" statusUpdatingConsoleVersion = "Updating Console Version" statusUpdatingResourceRequirements = "Updating Resource Requirements" statusUpdatingAffinity = "Updating Pod Affinity" @@ -153,6 +158,9 @@ type Controller struct { // Use a go template to render the hosts string hostsTemplate string + + // Webhook server instance + ws *http.Server } // NewController returns a new sample controller @@ -196,6 +204,9 @@ func NewController( hostsTemplate: hostsTemplate, } + // Initialize operator webhook handlers + controller.ws = configureWebhookServer(controller) + klog.Info("Setting up event handlers") // Set up an event handler for when Tenant resources change tenantInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -242,11 +253,138 @@ func NewController( return controller } +func (c *Controller) validateRequest(r *http.Request, secret *v1.Secret) error { + tokenStr, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(r) + if err != nil { + return err + } + + stdClaims := &jwt.StandardClaims{} + token, err := jwt.ParseWithClaims(tokenStr, stdClaims, func(token *jwt.Token) (interface{}, error) { + return secret.Data[miniov1.WebhookOperatorPassword], nil + }) + if err != nil { + return err + } + + if !token.Valid { + return fmt.Errorf(http.StatusText(http.StatusForbidden)) + } + + if stdClaims.Issuer != string(secret.Data[miniov1.WebhookOperatorUsername]) { + return fmt.Errorf(http.StatusText(http.StatusForbidden)) + } + + return nil +} + +func (c *Controller) applyOperatorWebhookSecret(ctx context.Context, mi *miniov1.Tenant) (*v1.Secret, error) { + secret, err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Get(ctx, miniov1.WebhookOperatorSecret, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + cred, err := auth.GetNewCredentials() + if err != nil { + return nil, err + } + secret = &corev1.Secret{ + Type: "Opaque", + ObjectMeta: metav1.ObjectMeta{ + Name: miniov1.WebhookOperatorSecret, + Namespace: mi.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(mi, schema.GroupVersionKind{ + Group: miniov1.SchemeGroupVersion.Group, + Version: miniov1.SchemeGroupVersion.Version, + Kind: miniov1.MinIOCRDResourceKind, + }), + }, + }, + Data: map[string][]byte{ + miniov1.WebhookOperatorUsername: []byte(cred.AccessKey), + miniov1.WebhookOperatorPassword: []byte(cred.SecretKey), + }, + } + return c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Create(ctx, secret, metav1.CreateOptions{}) + } + return nil, err + } + return secret, nil +} + +// Supported remote envs +const ( + envMinIOArgs = "MINIO_ARGS" +) + +// BucketSrvHandler - POST /webhook/api/v1/bucketsrv/{namespace}/{name}?bucket={bucket} +func (c *Controller) BucketSrvHandler(w http.ResponseWriter, r *http.Request) { +} + +// GetenvHandler - GET /webhook/api/v1/getenv/{namespace}/{name}?key={env} +func (c *Controller) GetenvHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + namespace := vars["namespace"] + name := vars["name"] + key := vars["key"] + + secret, err := c.kubeClientSet.CoreV1().Secrets(namespace).Get(r.Context(), + miniov1.WebhookOperatorSecret, metav1.GetOptions{}) + if err != nil { + http.Error(w, err.Error(), http.StatusForbidden) + return + } + + if err = c.validateRequest(r, secret); err != nil { + http.Error(w, err.Error(), http.StatusForbidden) + return + } + + // Get the Tenant resource with this namespace/name + mi, err := c.tenantsLister.Tenants(namespace).Get(name) + if err != nil { + if errors.IsNotFound(err) { + // The Tenant resource may no longer exist, in which case we stop processing. + http.Error(w, fmt.Sprintf("Tenant '%s' in work queue no longer exists", key), http.StatusNotFound) + return + } + http.Error(w, err.Error(), http.StatusForbidden) + return + } + + mi.EnsureDefaults() + miniov1.InitGlobals(mi) + + // Validate the MinIO Instance + if err = mi.Validate(); err != nil { + http.Error(w, err.Error(), http.StatusForbidden) + return + } + + switch key { + case envMinIOArgs: + args := strings.Join(statefulsets.GetContainerArgs(mi, c.hostsTemplate), " ") + klog.Infof("%s value is %s", key, args) + + _, _ = w.Write([]byte(args)) + w.(http.Flusher).Flush() + default: + http.Error(w, fmt.Sprintf("%s env key is not supported yet", key), http.StatusBadRequest) + return + } +} + // Start will set up the event handlers for types we are interested in, as well // as syncing informer caches and starting workers. It will block until stopCh // is closed, at which point it will shutdown the workqueue and wait for // workers to finish processing their current work items. func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error { + go func() { + if err := c.ws.ListenAndServe(); err != http.ErrServerClosed { + klog.Infof("HTTP server ListenAndServe: %v", err) + return + } + }() // Start the informer factories to begin populating the informer caches klog.Info("Starting Tenant controller") @@ -268,6 +406,12 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error { // Stop is called to shutdown the controller func (c *Controller) Stop() { + klog.Info("Stopping the minio controller webhook") + // Wait upto 5 secs and terminate all connections. + tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = c.ws.Shutdown(tctx) + cancel() + klog.Info("Stopping the minio controller") c.workqueue.ShutDown() } @@ -333,6 +477,17 @@ func (c *Controller) processNextWorkItem() bool { return true } +const slashSeparator = "/" + +func key2NamespaceName(key string) (namespace, name string) { + key = strings.TrimPrefix(key, slashSeparator) + m := strings.Index(key, slashSeparator) + if m < 0 { + return "", key + } + return key[:m], key[m+len(slashSeparator):] +} + // syncHandler compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Tenant resource // with the current status of the resource. @@ -345,12 +500,13 @@ func (c *Controller) syncHandler(key string) error { var consoleDeployment *appsv1.Deployment // Convert the namespace/name string into a distinct namespace and name - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { + if key == "" { runtime.HandleError(fmt.Errorf("Invalid resource key: %s", key)) return nil } + namespace, name := key2NamespaceName(key) + // Get the Tenant resource with this namespace/name mi, err := c.tenantsLister.Tenants(namespace).Get(name) if err != nil { @@ -376,6 +532,11 @@ func (c *Controller) syncHandler(key string) error { return err } + secret, err := c.applyOperatorWebhookSecret(ctx, mi) + if err != nil { + return err + } + // check if both auto certificate creation and external secret with certificate is passed, // this is an error as only one of this is allowed in one Tenant if mi.AutoCert() && (mi.ExternalCert() || mi.ExternalClientCert() || mi.KESExternalCert() || mi.ConsoleExternalCert()) { @@ -499,52 +660,32 @@ func (c *Controller) syncHandler(key string) error { if mi, err = c.updateTenantStatus(ctx, mi, statusProvisioningStatefulSet, 0); err != nil { return err } - ss = statefulsets.NewForMinIOZone(mi, &zone, hlSvc.Name, c.hostsTemplate) + + ss = statefulsets.NewForMinIOZone(mi, secret, &zone, hlSvc.Name, c.hostsTemplate) ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts) if err != nil { return err } - // Prepare for zone expansion by deleting existing pods, - // this help all the pods to synchronize faster. - if err := c.deleteTenantPods(ctx, mi); err != nil { - return err - } + + // Restart the services to fetch the new args, ignore any error. + _ = adminClnt.ServiceRestart(ctx) } else { return err } } else { - // If the number of the replicas on the Tenant resource is specified, and the - // number does not equal the current desired replicas on the StatefulSet, we - // should update the StatefulSet resource. - // If the status already indicates "statusUpdatingContainerArguments", no need for another - // thread to enter this block - we don't want to get in a race for deletion and creation of CSRs - if zone.Servers != *ss.Spec.Replicas && mi.Status.CurrentState != statusUpdatingContainerArguments { + if zone.Servers != *ss.Spec.Replicas { // warn the user that replica count of an existing zone can't be changed if mi, err = c.updateTenantStatus(ctx, mi, fmt.Sprintf("Can't modify server count for zone %s", zone.Name), 0); err != nil { return err } } - // verify the container arguments - currentArgsSet := set.CreateStringSet(ss.Spec.Template.Spec.Containers[0].Args...) - newArgsSet := set.CreateStringSet(statefulsets.GetContainerArgs(mi, c.hostsTemplate)...) - if !currentArgsSet.Equals(newArgsSet) { - if mi, err = c.updateTenantStatus(ctx, mi, statusUpdatingContainerArguments, ss.Status.Replicas); err != nil { - return err - } - klog.V(4).Infof("container arguments updates for zone %s", zone.Name) - ss = statefulsets.NewForMinIOZone(mi, &zone, hlSvc.Name, c.hostsTemplate) - if ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { - return err - } - } - if zone.Resources.String() != ss.Spec.Template.Spec.Containers[0].Resources.String() { if mi, err = c.updateTenantStatus(ctx, mi, statusUpdatingResourceRequirements, ss.Status.Replicas); err != nil { return err } klog.V(4).Infof("resource requirements updates for zone %s", zone.Name) - ss = statefulsets.NewForMinIOZone(mi, &zone, hlSvc.Name, c.hostsTemplate) + ss = statefulsets.NewForMinIOZone(mi, secret, &zone, hlSvc.Name, c.hostsTemplate) if ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { return err } @@ -555,7 +696,7 @@ func (c *Controller) syncHandler(key string) error { return err } klog.V(4).Infof("affinity update for zone %s", zone.Name) - ss = statefulsets.NewForMinIOZone(mi, &zone, hlSvc.Name, c.hostsTemplate) + ss = statefulsets.NewForMinIOZone(mi, secret, &zone, hlSvc.Name, c.hostsTemplate) if ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { return err } @@ -657,7 +798,7 @@ func (c *Controller) syncHandler(key string) error { for _, zone := range mi.Spec.Zones { // Now proceed to make the yaml changes for the tenant statefulset. - ss := statefulsets.NewForMinIOZone(mi, &zone, hlSvc.Name, c.hostsTemplate) + ss := statefulsets.NewForMinIOZone(mi, secret, &zone, hlSvc.Name, c.hostsTemplate) if _, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { return err } @@ -959,11 +1100,3 @@ func (c *Controller) checkAndCreateConsoleCSR(ctx context.Context, nsName types. } return nil } - -// deleteTenantPods deletes all the pods for a defined tenant. -func (c *Controller) deleteTenantPods(ctx context.Context, tenant *miniov1.Tenant) error { - listOpts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", miniov1.TenantLabel, tenant.Name), - } - return c.kubeClientSet.CoreV1().Pods(tenant.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts) -} diff --git a/pkg/controller/cluster/webhook-server.go b/pkg/controller/cluster/webhook-server.go new file mode 100644 index 00000000000..808a336ed49 --- /dev/null +++ b/pkg/controller/cluster/webhook-server.go @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020, MinIO, Inc. + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +package cluster + +import ( + "fmt" + "net/http" + "time" + + "github.com/gorilla/mux" + miniov1 "github.com/minio/operator/pkg/apis/minio.min.io/v1" +) + +// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example) +// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"] +func restQueries(keys ...string) []string { + var accumulator []string + for _, key := range keys { + accumulator = append(accumulator, key, "{"+key+":.*}") + } + return accumulator +} + +func configureWebhookServer(c *Controller) *http.Server { + router := mux.NewRouter().SkipClean(true).UseEncodedPath() + + router.Methods(http.MethodGet). + Path(miniov1.WebhookAPIGetenv + "/{namespace}/{name:.+}"). + HandlerFunc(c.GetenvHandler). + Queries(restQueries("key")...) + + router.Methods(http.MethodPost). + Path(miniov1.WebhookAPIBucketService + "/{namespace}/{name:.+}"). + HandlerFunc(c.BucketSrvHandler). + Queries(restQueries("bucket")...) + + router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Println(r) + }) + + s := &http.Server{ + Addr: ":" + miniov1.WebhookDefaultPort, + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + return s +} diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 060f04fbd24..4ecf00be149 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -24,6 +24,7 @@ import ( miniov1 "github.com/minio/operator/pkg/apis/minio.min.io/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" @@ -32,11 +33,13 @@ import ( // Returns the MinIO environment variables set in configuration. // If a user specifies a secret in the spec (for MinIO credentials) we use // that to set MINIO_ACCESS_KEY & MINIO_SECRET_KEY. -func minioEnvironmentVars(t *miniov1.Tenant) []corev1.EnvVar { +func minioEnvironmentVars(t *miniov1.Tenant, wsSecret *v1.Secret) []corev1.EnvVar { var envVars []corev1.EnvVar // Add all the environment variables envVars = append(envVars, t.Spec.Env...) + envScheme := "env" + // Enable `mc admin update` style updates to MinIO binaries // within the container, only operator is supposed to perform // these operations. @@ -46,6 +49,19 @@ func minioEnvironmentVars(t *miniov1.Tenant) []corev1.EnvVar { }, corev1.EnvVar{ Name: "MINIO_UPDATE_MINISIGN_PUBKEY", Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", + }, corev1.EnvVar{ + Name: "MINIO_ARGS", + Value: fmt.Sprintf("%s://%s:%s@%s:%s%s/%s/%s", + envScheme, + string(wsSecret.Data[miniov1.WebhookOperatorUsername]), + string(wsSecret.Data[miniov1.WebhookOperatorPassword]), + fmt.Sprintf("operator.minio-operator.svc.%s", + miniov1.ClusterDomain), + miniov1.WebhookDefaultPort, + miniov1.WebhookAPIGetenv, + t.Namespace, + t.Name, + ), }) // Add env variables from credentials secret, if no secret provided, dont use @@ -189,8 +205,8 @@ func probes(t *miniov1.Tenant) (liveness *corev1.Probe) { } // Builds the MinIO container for a Tenant. -func zoneMinioServerContainer(t *miniov1.Tenant, zone *miniov1.Zone, hostsTemplate string) corev1.Container { - args := GetContainerArgs(t, hostsTemplate) +func zoneMinioServerContainer(t *miniov1.Tenant, wsSecret *v1.Secret, zone *miniov1.Zone, hostsTemplate string) corev1.Container { + args := []string{"server", "--certs-dir", miniov1.MinIOCertPath} liveProbe := probes(t) @@ -205,7 +221,7 @@ func zoneMinioServerContainer(t *miniov1.Tenant, zone *miniov1.Zone, hostsTempla ImagePullPolicy: miniov1.DefaultImagePullPolicy, VolumeMounts: volumeMounts(t, zone), Args: args, - Env: minioEnvironmentVars(t), + Env: minioEnvironmentVars(t, wsSecret), Resources: zone.Resources, LivenessProbe: liveProbe, } @@ -213,8 +229,7 @@ func zoneMinioServerContainer(t *miniov1.Tenant, zone *miniov1.Zone, hostsTempla // GetContainerArgs returns the arguments that the MinIO container receives func GetContainerArgs(t *miniov1.Tenant, hostsTemplate string) []string { - args := []string{"server", "--certs-dir", miniov1.MinIOCertPath} - + var args []string if len(t.Spec.Zones) == 1 && t.Spec.Zones[0].Servers == 1 { // to run in standalone mode we must pass the path args = append(args, t.VolumePathForZone(&t.Spec.Zones[0])) @@ -242,7 +257,7 @@ func minioSecurityContext(t *miniov1.Tenant) *corev1.PodSecurityContext { } // NewForMinIOZone creates a new StatefulSet for the given Cluster. -func NewForMinIOZone(t *miniov1.Tenant, zone *miniov1.Zone, serviceName string, hostsTemplate string) *appsv1.StatefulSet { +func NewForMinIOZone(t *miniov1.Tenant, wsSecret *v1.Secret, zone *miniov1.Zone, serviceName string, hostsTemplate string) *appsv1.StatefulSet { var podVolumes []corev1.Volume var replicas = zone.Servers var serverCertSecret string @@ -345,7 +360,7 @@ func NewForMinIOZone(t *miniov1.Tenant, zone *miniov1.Zone, serviceName string, }) } - containers := []corev1.Container{zoneMinioServerContainer(t, zone, hostsTemplate)} + containers := []corev1.Container{zoneMinioServerContainer(t, wsSecret, zone, hostsTemplate)} ss := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{