From 2b7f4de2be695a65022bbbe03c5025a363eb404a Mon Sep 17 00:00:00 2001 From: jiuker Date: Fri, 18 Oct 2024 10:58:15 +0800 Subject: [PATCH 1/4] feat: support metrics scrape feat: support metrics scrape --- Makefile | 2 +- docs/tenant_crd.adoc | 6 ++ go.mod | 7 --- go.sum | 21 ------- .../templates/minio.min.io_tenants.yaml | 59 ++++++++++++++++++- .../templates/sts.min.io_policybindings.yaml | 2 +- pkg/apis/minio.min.io/v2/constants.go | 4 +- pkg/apis/minio.min.io/v2/types.go | 6 ++ pkg/resources/configmaps/prometheus.go | 35 ++++++----- resources/base/crds/minio.min.io_tenants.yaml | 59 ++++++++++++++++++- .../base/crds/sts.min.io_policybindings.yaml | 2 +- 11 files changed, 154 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 1524d478a98..24197d373a4 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ clean: @rm -rf dist/ regen-crd: - @go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0 + @go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.4 @${GOPATH}/bin/controller-gen crd:maxDescLen=0,generateEmbeddedObjectMeta=true webhook paths="./..." output:crd:artifacts:config=$(KUSTOMIZE_CRDS) @sed 's#namespace: minio-operator#namespace: {{ .Release.Namespace }}#g' resources/base/crds/minio.min.io_tenants.yaml > $(HELM_TEMPLATES)/minio.min.io_tenants.yaml @sed 's#namespace: minio-operator#namespace: {{ .Release.Namespace }}#g' resources/base/crds/sts.min.io_policybindings.yaml > $(HELM_TEMPLATES)/sts.min.io_policybindings.yaml diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index e01e63ed310..40bc0fba078 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -1143,6 +1143,12 @@ Directs the MinIO Operator to use prometheus operator. + Tenant scrape configuration will be added to prometheus managed by the prometheus-operator. +|*`prometheusOperatorScrape`* __string array__ +|*Optional* + + + +The name of the Prometheus instance to scrape metrics from. + |*`serviceAccountName`* __string__ |*Optional* + diff --git a/go.mod b/go.mod index 774fbd42e82..710444cdaa4 100644 --- a/go.mod +++ b/go.mod @@ -52,8 +52,6 @@ require ( require ( aead.dev/mem v0.2.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect @@ -61,9 +59,7 @@ require ( github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -102,7 +98,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -120,12 +115,10 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/tools v0.22.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 2710d19040e..1c50ff4fb17 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,8 @@ aead.dev/mem v0.2.0 h1:ufgkESS9+lHV/GUjxgc2ObF43FLZGSemh+W+y27QFMI= aead.dev/mem v0.2.0/go.mod h1:4qj+sh8fjDhlvne9gm/ZaMRIX9EkmDrKOLwmyDtoMWM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,14 +25,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= -github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -52,7 +44,6 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= @@ -176,12 +167,8 @@ github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0 h github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0/go.mod h1:wAR5JopumPtAZnu0Cjv2PSqV4p4QB09LMhc6fZZTXuA= github.com/prometheus-operator/prometheus-operator/pkg/client v0.74.0 h1:SyBTzvFuVshDNjDVALs6+NgOy3qh8/xlAsyqB1SzHbI= github.com/prometheus-operator/prometheus-operator/pkg/client v0.74.0/go.mod h1:FlcnLo14zQxL6P1yPrV22kYBqyAT0ZRRytv98+B7lBQ= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= -github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= @@ -250,8 +237,6 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= -golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -333,8 +318,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -358,8 +341,6 @@ gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= -k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= -k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= @@ -376,8 +357,6 @@ k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlL k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= -sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index e295126ce51..f9ffb49783f 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.4 operator.min.io/version: v6.0.4 name: tenants.minio.min.io spec: @@ -94,10 +94,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -474,6 +476,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -487,6 +496,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: format: int32 @@ -735,6 +745,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -742,6 +753,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -753,6 +765,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -761,6 +774,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -778,6 +792,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -1281,6 +1296,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -1387,6 +1403,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -1468,6 +1485,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -1585,6 +1604,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -2338,6 +2358,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -2412,6 +2434,8 @@ spec: type: integer type: array x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string sysctls: items: properties: @@ -2654,6 +2678,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -3264,6 +3289,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -3340,6 +3367,8 @@ spec: type: integer type: array x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string sysctls: items: properties: @@ -3653,6 +3682,10 @@ spec: type: string prometheusOperator: type: boolean + prometheusOperatorScrape: + items: + type: string + type: array readiness: properties: exec: @@ -3672,6 +3705,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4020,6 +4054,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4126,6 +4161,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4207,6 +4243,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -4324,6 +4362,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4448,6 +4487,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -4681,10 +4722,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -5061,6 +5104,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -5074,6 +5124,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: format: int32 @@ -5322,6 +5373,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -5329,6 +5381,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -5340,6 +5393,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -5348,6 +5402,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -5365,6 +5420,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -5458,6 +5514,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port diff --git a/helm/operator/templates/sts.min.io_policybindings.yaml b/helm/operator/templates/sts.min.io_policybindings.yaml index c1a3ac2446b..e39e3896b60 100644 --- a/helm/operator/templates/sts.min.io_policybindings.yaml +++ b/helm/operator/templates/sts.min.io_policybindings.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.4 operator.min.io/version: v6.0.4 name: policybindings.sts.min.io spec: diff --git a/pkg/apis/minio.min.io/v2/constants.go b/pkg/apis/minio.min.io/v2/constants.go index 61463b31b01..c57aede25d6 100644 --- a/pkg/apis/minio.min.io/v2/constants.go +++ b/pkg/apis/minio.min.io/v2/constants.go @@ -188,8 +188,8 @@ const StatefulSetPrefix = "ss" // StatefulSetLegacyPrefix by old operators const StatefulSetLegacyPrefix = "zone" -// MinIOPrometheusPathCluster is the path where MinIO tenant exposes cluster Prometheus metrics -const MinIOPrometheusPathCluster = "/minio/v2/metrics/cluster" +// MinIOPrometheusPathPrefix is the path prefix where MinIO tenant exposes cluster Prometheus metrics +const MinIOPrometheusPathPrefix = "/minio/v2/metrics/" // MinIOPrometheusScrapeInterval defines how frequently to scrape targets. const MinIOPrometheusScrapeInterval = 30 * time.Second diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 4644f72a58a..e44728d871f 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -278,6 +278,12 @@ type TenantSpec struct { PrometheusOperator bool `json:"prometheusOperator,omitempty"` // *Optional* + // + // The name of the Prometheus instance to scrape metrics from. + // + // +optional + PrometheusOperatorScrape []string `json:"prometheusOperatorScrape,omitempty"` + // *Optional* + + // // The https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes Service Account] to use for running MinIO pods created as part of the Tenant. + // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` diff --git a/pkg/resources/configmaps/prometheus.go b/pkg/resources/configmaps/prometheus.go index 82b1c5c6171..49b41e6bf93 100644 --- a/pkg/resources/configmaps/prometheus.go +++ b/pkg/resources/configmaps/prometheus.go @@ -80,22 +80,29 @@ func GetPrometheusConfig(t *miniov2.Tenant, accessKey, secretKey string) *Promet ScrapeInterval: miniov2.MinIOPrometheusScrapeInterval, EvaluationInterval: 30 * time.Second, }, - ScrapeConfigs: []ScrapeConfig{ - { - JobName: t.PrometheusConfigJobName(), - BearerToken: bearerToken, - MetricsPath: miniov2.MinIOPrometheusPathCluster, - Scheme: minioScheme, - TLSConfig: tlsConfig{ - CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", - }, - StaticConfigs: []staticConfig{ - { - Targets: []string{minioTargets}, - }, + ScrapeConfigs: []ScrapeConfig{}, + } + + if len(t.Spec.PrometheusOperatorScrape) == 0 { + t.Spec.PrometheusOperatorScrape = []string{"cluster"} + } + + for _, scrape := range t.Spec.PrometheusOperatorScrape { + promConfig.ScrapeConfigs = append(promConfig.ScrapeConfigs, ScrapeConfig{ + JobName: t.PrometheusConfigJobName(), + BearerToken: bearerToken, + MetricsPath: fmt.Sprintf("%s%s", miniov2.MinIOPrometheusPathPrefix, scrape), + Scheme: minioScheme, + TLSConfig: tlsConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []staticConfig{ + { + Targets: []string{minioTargets}, }, }, - }, + }) } + return promConfig } diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index e295126ce51..f9ffb49783f 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.4 operator.min.io/version: v6.0.4 name: tenants.minio.min.io spec: @@ -94,10 +94,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -474,6 +476,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -487,6 +496,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: format: int32 @@ -735,6 +745,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -742,6 +753,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -753,6 +765,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -761,6 +774,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -778,6 +792,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -1281,6 +1296,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -1387,6 +1403,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -1468,6 +1485,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -1585,6 +1604,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -2338,6 +2358,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -2412,6 +2434,8 @@ spec: type: integer type: array x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string sysctls: items: properties: @@ -2654,6 +2678,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -3264,6 +3289,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -3340,6 +3367,8 @@ spec: type: integer type: array x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string sysctls: items: properties: @@ -3653,6 +3682,10 @@ spec: type: string prometheusOperator: type: boolean + prometheusOperatorScrape: + items: + type: string + type: array readiness: properties: exec: @@ -3672,6 +3705,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4020,6 +4054,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4126,6 +4161,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4207,6 +4243,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -4324,6 +4362,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port @@ -4448,6 +4487,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -4681,10 +4722,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -5061,6 +5104,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -5074,6 +5124,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: format: int32 @@ -5322,6 +5373,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -5329,6 +5381,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -5340,6 +5393,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -5348,6 +5402,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -5365,6 +5420,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -5458,6 +5514,7 @@ spec: format: int32 type: integer service: + default: "" type: string required: - port diff --git a/resources/base/crds/sts.min.io_policybindings.yaml b/resources/base/crds/sts.min.io_policybindings.yaml index c1a3ac2446b..e39e3896b60 100644 --- a/resources/base/crds/sts.min.io_policybindings.yaml +++ b/resources/base/crds/sts.min.io_policybindings.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.4 operator.min.io/version: v6.0.4 name: policybindings.sts.min.io spec: From 483a69c0a310117a2d9ba0b0e31b5e86d7db4d29 Mon Sep 17 00:00:00 2001 From: jiuker Date: Fri, 18 Oct 2024 11:22:44 +0800 Subject: [PATCH 2/4] refactor refactor --- docs/tenant_crd.adoc | 2 +- helm/operator/templates/minio.min.io_tenants.yaml | 2 +- helm/operator/templates/sts.min.io_policybindings.yaml | 2 +- pkg/apis/minio.min.io/v2/constants.go | 3 --- pkg/apis/minio.min.io/v2/types.go | 2 +- pkg/resources/configmaps/prometheus.go | 8 ++++---- resources/base/crds/minio.min.io_tenants.yaml | 2 +- resources/base/crds/sts.min.io_policybindings.yaml | 2 +- 8 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index 40bc0fba078..cd901820779 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -1143,7 +1143,7 @@ Directs the MinIO Operator to use prometheus operator. + Tenant scrape configuration will be added to prometheus managed by the prometheus-operator. -|*`prometheusOperatorScrape`* __string array__ +|*`prometheusOperatorScrapeMetricsPath`* __string array__ |*Optional* + diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index f9ffb49783f..23c7d3a1943 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -3682,7 +3682,7 @@ spec: type: string prometheusOperator: type: boolean - prometheusOperatorScrape: + prometheusOperatorScrapeMetricsPath: items: type: string type: array diff --git a/helm/operator/templates/sts.min.io_policybindings.yaml b/helm/operator/templates/sts.min.io_policybindings.yaml index e39e3896b60..f9a1d7f86b4 100644 --- a/helm/operator/templates/sts.min.io_policybindings.yaml +++ b/helm/operator/templates/sts.min.io_policybindings.yaml @@ -4,7 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.4 - operator.min.io/version: v6.0.4 + operator.min.io/version: v5.0.15 name: policybindings.sts.min.io spec: group: sts.min.io diff --git a/pkg/apis/minio.min.io/v2/constants.go b/pkg/apis/minio.min.io/v2/constants.go index c57aede25d6..f298c65592a 100644 --- a/pkg/apis/minio.min.io/v2/constants.go +++ b/pkg/apis/minio.min.io/v2/constants.go @@ -188,9 +188,6 @@ const StatefulSetPrefix = "ss" // StatefulSetLegacyPrefix by old operators const StatefulSetLegacyPrefix = "zone" -// MinIOPrometheusPathPrefix is the path prefix where MinIO tenant exposes cluster Prometheus metrics -const MinIOPrometheusPathPrefix = "/minio/v2/metrics/" - // MinIOPrometheusScrapeInterval defines how frequently to scrape targets. const MinIOPrometheusScrapeInterval = 30 * time.Second diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index e44728d871f..b10d4dacc36 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -281,7 +281,7 @@ type TenantSpec struct { // The name of the Prometheus instance to scrape metrics from. // // +optional - PrometheusOperatorScrape []string `json:"prometheusOperatorScrape,omitempty"` + PrometheusOperatorScrapeMetricsPath []string `json:"prometheusOperatorScrapeMetricsPath,omitempty"` // *Optional* + // // The https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes Service Account] to use for running MinIO pods created as part of the Tenant. + diff --git a/pkg/resources/configmaps/prometheus.go b/pkg/resources/configmaps/prometheus.go index 49b41e6bf93..ed3f73da9ba 100644 --- a/pkg/resources/configmaps/prometheus.go +++ b/pkg/resources/configmaps/prometheus.go @@ -83,15 +83,15 @@ func GetPrometheusConfig(t *miniov2.Tenant, accessKey, secretKey string) *Promet ScrapeConfigs: []ScrapeConfig{}, } - if len(t.Spec.PrometheusOperatorScrape) == 0 { - t.Spec.PrometheusOperatorScrape = []string{"cluster"} + if len(t.Spec.PrometheusOperatorScrapeMetricsPath) == 0 { + t.Spec.PrometheusOperatorScrapeMetricsPath = []string{"/minio/v2/metrics/cluster"} } - for _, scrape := range t.Spec.PrometheusOperatorScrape { + for _, scrape := range t.Spec.PrometheusOperatorScrapeMetricsPath { promConfig.ScrapeConfigs = append(promConfig.ScrapeConfigs, ScrapeConfig{ JobName: t.PrometheusConfigJobName(), BearerToken: bearerToken, - MetricsPath: fmt.Sprintf("%s%s", miniov2.MinIOPrometheusPathPrefix, scrape), + MetricsPath: scrape, Scheme: minioScheme, TLSConfig: tlsConfig{ CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index f9ffb49783f..23c7d3a1943 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -3682,7 +3682,7 @@ spec: type: string prometheusOperator: type: boolean - prometheusOperatorScrape: + prometheusOperatorScrapeMetricsPath: items: type: string type: array diff --git a/resources/base/crds/sts.min.io_policybindings.yaml b/resources/base/crds/sts.min.io_policybindings.yaml index e39e3896b60..f9a1d7f86b4 100644 --- a/resources/base/crds/sts.min.io_policybindings.yaml +++ b/resources/base/crds/sts.min.io_policybindings.yaml @@ -4,7 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.16.4 - operator.min.io/version: v6.0.4 + operator.min.io/version: v5.0.15 name: policybindings.sts.min.io spec: group: sts.min.io From ad2bcb6e17d8dbf132af1011ffb12154c55c4ba1 Mon Sep 17 00:00:00 2001 From: jiuker Date: Fri, 18 Oct 2024 17:35:49 +0800 Subject: [PATCH 3/4] refactor refactor --- pkg/controller/prometheus.go | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/pkg/controller/prometheus.go b/pkg/controller/prometheus.go index 1b231992d84..67f9d9c0afb 100644 --- a/pkg/controller/prometheus.go +++ b/pkg/controller/prometheus.go @@ -17,6 +17,7 @@ package controller import ( "context" "errors" + "reflect" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" @@ -161,20 +162,20 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten return err } } else { - var scrapeConfigs []configmaps.ScrapeConfig + var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig err := yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs) if err != nil { return err } - // Check if the scrape config is already present - hasScrapeConfig := false + // get other scrape configs for _, sc := range scrapeConfigs { - if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() { - hasScrapeConfig = true - break + if sc.JobName != tenant.PrometheusOperatorAddlConfigJobName() { + exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } - if !hasScrapeConfig { + // add new scrape configs + exceptedScrapeConfigs = append(exceptedScrapeConfigs, promCfg.ScrapeConfigs...) + if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) { klog.Infof("Adding MinIO tenant Prometheus scrape config") scrapeConfigs = append(scrapeConfigs, promCfg.ScrapeConfigs...) scrapeCfgYaml, err := yaml.Marshal(scrapeConfigs) @@ -224,27 +225,20 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min return err } - var scrapeConfigs []configmaps.ScrapeConfig + var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig err = yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs) if err != nil { return err } - // Check if the scrape config is present - hasScrapeConfig := false - scIndex := -1 - for i, sc := range scrapeConfigs { - if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() { - hasScrapeConfig = true - scIndex = i - break + for _, sc := range scrapeConfigs { + if sc.JobName != tenant.PrometheusOperatorAddlConfigJobName() { + exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } - if hasScrapeConfig { + if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) { klog.Infof("Deleting MinIO tenant Prometheus scrape config") - // Delete the config - newScrapeConfigs := append(scrapeConfigs[:scIndex], scrapeConfigs[scIndex+1:]...) // Update the secret - scrapeCfgYaml, err := yaml.Marshal(newScrapeConfigs) + scrapeCfgYaml, err := yaml.Marshal(exceptedScrapeConfigs) if err != nil { return err } From b38e1b6f638d4c497accba5564409afc49b83dcd Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 21 Oct 2024 15:00:43 +0800 Subject: [PATCH 4/4] support muilt job name --- pkg/controller/prometheus.go | 5 +++-- pkg/resources/configmaps/prometheus.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/controller/prometheus.go b/pkg/controller/prometheus.go index 67f9d9c0afb..c5d72bceaec 100644 --- a/pkg/controller/prometheus.go +++ b/pkg/controller/prometheus.go @@ -18,6 +18,7 @@ import ( "context" "errors" "reflect" + "strings" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" @@ -169,7 +170,7 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten } // get other scrape configs for _, sc := range scrapeConfigs { - if sc.JobName != tenant.PrometheusOperatorAddlConfigJobName() { + if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) { exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } @@ -231,7 +232,7 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min return err } for _, sc := range scrapeConfigs { - if sc.JobName != tenant.PrometheusOperatorAddlConfigJobName() { + if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) { exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } diff --git a/pkg/resources/configmaps/prometheus.go b/pkg/resources/configmaps/prometheus.go index ed3f73da9ba..20aed0bba80 100644 --- a/pkg/resources/configmaps/prometheus.go +++ b/pkg/resources/configmaps/prometheus.go @@ -87,9 +87,9 @@ func GetPrometheusConfig(t *miniov2.Tenant, accessKey, secretKey string) *Promet t.Spec.PrometheusOperatorScrapeMetricsPath = []string{"/minio/v2/metrics/cluster"} } - for _, scrape := range t.Spec.PrometheusOperatorScrapeMetricsPath { + for index, scrape := range t.Spec.PrometheusOperatorScrapeMetricsPath { promConfig.ScrapeConfigs = append(promConfig.ScrapeConfigs, ScrapeConfig{ - JobName: t.PrometheusConfigJobName(), + JobName: fmt.Sprintf("%s-%d", t.PrometheusOperatorAddlConfigJobName(), index), BearerToken: bearerToken, MetricsPath: scrape, Scheme: minioScheme,