diff --git a/.github/workflows/lint_helm_chart.yaml b/.github/workflows/lint_helm_chart.yaml index c5986b89a..f088039a6 100644 --- a/.github/workflows/lint_helm_chart.yaml +++ b/.github/workflows/lint_helm_chart.yaml @@ -26,6 +26,7 @@ jobs: set -e cd deployment/helm_chart/opik helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add clickhouse-operator https://docs.altinity.com/clickhouse-operator helm dependency build helm lint --values values.yaml . cd - diff --git a/.github/workflows/publish_helm_chart.yaml b/.github/workflows/publish_helm_chart.yaml index 0e71f7ef9..028a75cbe 100644 --- a/.github/workflows/publish_helm_chart.yaml +++ b/.github/workflows/publish_helm_chart.yaml @@ -52,6 +52,7 @@ jobs: set -e cd deployment/helm_chart/opik helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add clickhouse-operator https://docs.altinity.com/clickhouse-operator helm dependency build helm lint --values values.yaml . cd - diff --git a/deployment/helm_chart/opik/Chart.lock b/deployment/helm_chart/opik/Chart.lock index 2b5f35400..ada6daf57 100644 --- a/deployment/helm_chart/opik/Chart.lock +++ b/deployment/helm_chart/opik/Chart.lock @@ -8,5 +8,8 @@ dependencies: - name: redis repository: https://charts.bitnami.com/bitnami version: 18.19.2 -digest: sha256:aff7158d368c78d7fcb68102bf04e7b5631eff0a80bddcd6404d4c2eb95e5200 -generated: "2024-08-25T14:11:27.399354+03:00" +- name: altinity-clickhouse-operator + repository: https://docs.altinity.com/clickhouse-operator/ + version: 0.23.7 +digest: sha256:1863002507fafe0a986af167356bbb3178a4870994225b728efac3cfd9db617e +generated: "2024-10-13T14:38:17.89184+03:00" diff --git a/deployment/helm_chart/opik/Chart.yaml b/deployment/helm_chart/opik/Chart.yaml index 0ff481efa..534a54d2d 100644 --- a/deployment/helm_chart/opik/Chart.yaml +++ b/deployment/helm_chart/opik/Chart.yaml @@ -39,3 +39,7 @@ dependencies: version: "18.19.2" repository: https://charts.bitnami.com/bitnami condition: redis.enabled +- name: altinity-clickhouse-operator + version: "0.23.7" + repository: https://docs.altinity.com/clickhouse-operator/ + diff --git a/deployment/helm_chart/opik/README.md b/deployment/helm_chart/opik/README.md index 06232776d..b1961e1a7 100644 --- a/deployment/helm_chart/opik/README.md +++ b/deployment/helm_chart/opik/README.md @@ -80,11 +80,14 @@ Call opik api on http://localhost:5173/api | https://charts.bitnami.com/bitnami | mysql | 11.1.9 | | https://charts.bitnami.com/bitnami | redis | 18.19.2 | | https://charts.bitnami.com/bitnami | zookeeper | 12.12.1 | +| https://docs.altinity.com/clickhouse-operator/ | altinity-clickhouse-operator | 0.23.7 | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| altinity-clickhouse-operator.metrics.enabled | bool | `false` | | +| altinity-clickhouse-operator.serviceMonitor.enabled | bool | `false` | | | basicAuth | bool | `false` | | | clickhouse.adminUser.password | string | `"opik"` | | | clickhouse.adminUser.useSecret.enabled | bool | `false` | | @@ -93,11 +96,8 @@ Call opik api on http://localhost:5173/api | clickhouse.image | string | `"altinity/clickhouse-server:24.3.5.47.altinitystable"` | | | clickhouse.logsLevel | string | `"information"` | | | clickhouse.monitoring.enabled | bool | `false` | | -| clickhouse.monitoring.monitoringReleaseName | string | `nil` | | | clickhouse.monitoring.password | string | `"opikmon"` | | | clickhouse.monitoring.username | string | `"opikmon"` | | -| clickhouse.namePrefix | string | `"opik"` | | -| clickhouse.operator.enabled | bool | `true` | | | clickhouse.replicasCount | int | `1` | | | clickhouse.service.serviceTemplate | string | `"clickhouse-cluster-svc-template"` | | | clickhouse.shardsCount | int | `1` | | diff --git a/deployment/helm_chart/opik/crds/clickhouse-operator-crds.yaml b/deployment/helm_chart/opik/crds/clickhouse-operator-crds.yaml deleted file mode 100644 index af195cfb6..000000000 --- a/deployment/helm_chart/opik/crds/clickhouse-operator-crds.yaml +++ /dev/null @@ -1,3138 +0,0 @@ -# Template Parameters:# -# -# KIND=ClickHouseInstallation -# SINGULAR=clickhouseinstallation -# PLURAL=clickhouseinstallations -# SHORT=chi -# OPERATOR_VERSION=0.23.6 -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhouseinstallations.clickhouse.altinity.com - labels: - clickhouse.altinity.com/chop: 0.23.6 -spec: - group: clickhouse.altinity.com - scope: Namespaced - names: - kind: ClickHouseInstallation - singular: clickhouseinstallation - plural: clickhouseinstallations - shortNames: - - chi - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: version - type: string - description: Operator version - priority: 1 # show in wide view - jsonPath: .status.chop-version - - name: clusters - type: integer - description: Clusters count - jsonPath: .status.clusters - - name: shards - type: integer - description: Shards count - priority: 1 # show in wide view - jsonPath: .status.shards - - name: hosts - type: integer - description: Hosts count - jsonPath: .status.hosts - - name: taskID - type: string - description: TaskID - priority: 1 # show in wide view - jsonPath: .status.taskID - - name: status - type: string - description: CHI status - jsonPath: .status.status - - name: hosts-unchanged - type: integer - description: Unchanged hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsUnchanged - - name: hosts-updated - type: integer - description: Updated hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsUpdated - - name: hosts-added - type: integer - description: Added hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsAdded - - name: hosts-completed - type: integer - description: Completed hosts count - jsonPath: .status.hostsCompleted - - name: hosts-deleted - type: integer - description: Hosts deleted count - priority: 1 # show in wide view - jsonPath: .status.hostsDeleted - - name: hosts-delete - type: integer - description: Hosts to be deleted count - priority: 1 # show in wide view - jsonPath: .status.hostsDelete - - name: endpoint - type: string - description: Client access endpoint - priority: 1 # show in wide view - jsonPath: .status.endpoint - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - schema: - openAPIV3Schema: - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" - type: object - required: - - spec - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - type: object - description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" - properties: - chop-version: - type: string - description: "ClickHouse operator version" - chop-commit: - type: string - description: "ClickHouse operator git commit SHA" - chop-date: - type: string - description: "ClickHouse operator build date" - chop-ip: - type: string - description: "IP address of the operator's pod which managed this CHI" - clusters: - type: integer - minimum: 0 - description: "Clusters count" - shards: - type: integer - minimum: 0 - description: "Shards count" - replicas: - type: integer - minimum: 0 - description: "Replicas count" - hosts: - type: integer - minimum: 0 - description: "Hosts count" - status: - type: string - description: "Status" - taskID: - type: string - description: "Current task id" - taskIDsStarted: - type: array - description: "Started task ids" - nullable: true - items: - type: string - taskIDsCompleted: - type: array - description: "Completed task ids" - nullable: true - items: - type: string - action: - type: string - description: "Action" - actions: - type: array - description: "Actions" - nullable: true - items: - type: string - error: - type: string - description: "Last error" - errors: - type: array - description: "Errors" - nullable: true - items: - type: string - hostsUnchanged: - type: integer - minimum: 0 - description: "Unchanged Hosts count" - hostsUpdated: - type: integer - minimum: 0 - description: "Updated Hosts count" - hostsAdded: - type: integer - minimum: 0 - description: "Added Hosts count" - hostsCompleted: - type: integer - minimum: 0 - description: "Completed Hosts count" - hostsDeleted: - type: integer - minimum: 0 - description: "Deleted Hosts count" - hostsDelete: - type: integer - minimum: 0 - description: "About to delete Hosts count" - pods: - type: array - description: "Pods" - nullable: true - items: - type: string - pod-ips: - type: array - description: "Pod IPs" - nullable: true - items: - type: string - fqdns: - type: array - description: "Pods FQDNs" - nullable: true - items: - type: string - endpoint: - type: string - description: "Endpoint" - generation: - type: integer - minimum: 0 - description: "Generation" - normalized: - type: object - description: "Normalized CHI requested" - x-kubernetes-preserve-unknown-fields: true - normalizedCompleted: - type: object - description: "Normalized CHI completed" - x-kubernetes-preserve-unknown-fields: true - hostsWithTablesCreated: - type: array - description: "List of hosts with tables created by the operator" - nullable: true - items: - type: string - usedTemplates: - type: array - description: "List of templates used to build this CHI" - nullable: true - x-kubernetes-preserve-unknown-fields: true - items: - type: object - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - # x-kubernetes-preserve-unknown-fields: true - description: | - Specification of the desired behavior of one or more ClickHouse clusters - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md - properties: - taskID: - type: string - description: | - Allows to define custom taskID for CHI update and watch status of this update execution. - Displayed in all .status.taskID* fields. - By default (if not filled) every update of CHI manifest will generate random taskID - stop: &TypeStringBool - type: string - description: | - Allows to stop all ClickHouse clusters defined in a CHI. - Works as the following: - - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. - - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. - enum: - # List StringBoolXXX constants from model - - "" - - "0" - - "1" - - "False" - - "false" - - "True" - - "true" - - "No" - - "no" - - "Yes" - - "yes" - - "Off" - - "off" - - "On" - - "on" - - "Disable" - - "disable" - - "Enable" - - "enable" - - "Disabled" - - "disabled" - - "Enabled" - - "enabled" - restart: - type: string - description: | - In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. - This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. - enum: - - "" - - "RollingUpdate" - troubleshoot: - <<: *TypeStringBool - description: | - Allows to troubleshoot Pods during CrashLoopBack state. - This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. - Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts - and give time to troubleshoot via CLI. - Liveness and Readiness probes are disabled as well. - namespaceDomainPattern: - type: string - description: | - Custom domain pattern which will be used for DNS names of `Service` or `Pod`. - Typical use scenario - custom cluster domain in Kubernetes cluster - Example: %s.svc.my.test - templating: - type: object - # nullable: true - description: | - Optional, applicable inside ClickHouseInstallationTemplate only. - Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." - properties: - policy: - type: string - description: | - When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate - will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. - Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. - enum: - - "" - - "auto" - - "manual" - chiSelector: - type: object - description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - reconciling: - type: object - description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" - # nullable: true - properties: - policy: - type: string - description: | - DISCUSSED TO BE DEPRECATED - Syntax sugar - Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config - Possible values: - - wait - should wait to exclude host, complete queries and include host back into the cluster - - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster - enum: - - "" - - "wait" - - "nowait" - configMapPropagationTimeout: - type: integer - description: | - Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` - More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically - minimum: 0 - maximum: 3600 - cleanup: - type: object - description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" - # nullable: true - properties: - unknownObjects: - type: object - description: | - Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, - but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. - Default behavior is `Delete`" - # nullable: true - properties: - statefulSet: &TypeObjectsCleanup - type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" - enum: - # List ObjectsCleanupXXX constants from model - - "" - - "Retain" - - "Delete" - pvc: - type: string - <<: *TypeObjectsCleanup - description: "Behavior policy for unknown PVC, `Delete` by default" - configMap: - <<: *TypeObjectsCleanup - description: "Behavior policy for unknown ConfigMap, `Delete` by default" - service: - <<: *TypeObjectsCleanup - description: "Behavior policy for unknown Service, `Delete` by default" - reconcileFailedObjects: - type: object - description: | - Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. - Default behavior is `Retain`" - # nullable: true - properties: - statefulSet: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed StatefulSet, `Retain` by default" - pvc: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed PVC, `Retain` by default" - configMap: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed ConfigMap, `Retain` by default" - service: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed Service, `Retain` by default" - defaults: - type: object - description: | - define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults - # nullable: true - properties: - replicasUseFQDN: - <<: *TypeStringBool - description: | - define should replicas be specified by FQDN in ``. - In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup - "yes" by default - distributedDDL: - type: object - description: | - allows change `` settings - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl - # nullable: true - properties: - profile: - type: string - description: "Settings from this profile will be used to execute DDL queries" - storageManagement: - type: object - description: default storage management options - properties: - provisioner: &TypePVCProvisioner - type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" - enum: - - "" - - "StatefulSet" - - "Operator" - reclaimPolicy: &TypePVCReclaimPolicy - type: string - description: | - defines behavior of `PVC` deletion. - `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet - enum: - - "" - - "Retain" - - "Delete" - templates: &TypeTemplateNames - type: object - description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" - # nullable: true - properties: - hostTemplate: - type: string - description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" - podTemplate: - type: string - description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - dataVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - logVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - serviceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" - clusterServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" - shardServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" - replicaServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" - volumeClaimTemplate: - type: string - description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" - configuration: - type: object - description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" - # nullable: true - properties: - zookeeper: &TypeZookeeperConfig - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ - currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` - More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper - # nullable: true - properties: - nodes: - type: array - description: "describe every available zookeeper cluster node for interaction" - # nullable: true - items: - type: object - #required: - # - host - properties: - host: - type: string - description: "dns name or ip address for Zookeeper node" - port: - type: integer - description: "TCP port which used to connect to Zookeeper node" - minimum: 0 - maximum: 65535 - secure: - <<: *TypeStringBool - description: "if a secure connection to Zookeeper is required" - session_timeout_ms: - type: integer - description: "session timeout during connect to Zookeeper" - operation_timeout_ms: - type: integer - description: "one operation timeout during Zookeeper transactions" - root: - type: string - description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" - identity: - type: string - description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" - users: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure password hashed, authorization restrictions, database level security row filters etc. - More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers - # nullable: true - x-kubernetes-preserve-unknown-fields: true - profiles: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of settings profile - More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles - # nullable: true - x-kubernetes-preserve-unknown-fields: true - quotas: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of resource quotas - More details: https://clickhouse.tech/docs/en/operations/quotas/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas - # nullable: true - x-kubernetes-preserve-unknown-fields: true - settings: &TypeSettings - type: object - description: | - allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings - # nullable: true - x-kubernetes-preserve-unknown-fields: true - files: &TypeFiles - type: object - description: | - allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - every key in this object is the file name - every value in this object is the file content - you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html - each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored - More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml - # nullable: true - x-kubernetes-preserve-unknown-fields: true - clusters: - type: array - description: | - describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level - every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` - all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` - Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" - minLength: 1 - # See namePartClusterMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - zookeeper: - <<: *TypeZookeeperConfig - description: | - optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.zookeeper` settings - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster - override top-level `chi.spec.configuration.templates` - schemaPolicy: - type: object - description: | - describes how schema is propagated within replicas and shards - properties: - replica: - type: string - description: "how schema is propagated within a replica" - enum: - # List SchemaPolicyReplicaXXX constants from model - - "" - - "None" - - "All" - shard: - type: string - description: "how schema is propagated between shards" - enum: - # List SchemaPolicyShardXXX constants from model - - "" - - "None" - - "All" - - "DistributedTablesOnly" - insecure: - <<: *TypeStringBool - description: optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: optional, open secure ports for cluster - secret: - type: object - description: "optional, shared secret value to secure cluster communications" - properties: - auto: - <<: *TypeStringBool - description: "Auto-generate shared secret value to secure cluster communications" - value: - description: "Cluster shared secret value in plain text" - type: string - valueFrom: - description: "Cluster shared secret source" - type: object - properties: - secretKeyRef: - description: | - Selects a key of a secret in the clickhouse installation namespace. - Should not be used if value is not empty. - type: object - properties: - name: - description: | - Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - name - - key - layout: - type: object - description: | - describe current cluster layout, how much shards in cluster, how much replica in shard - allows override settings on each shard and replica separatelly - # nullable: true - properties: - type: - type: string - description: "DEPRECATED - to be removed soon" - shardsCount: - type: integer - description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" - replicasCount: - type: integer - description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" - shards: - type: array - description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - definitionType: - type: string - description: "DEPRECATED - to be removed soon" - weight: - type: integer - description: | - optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - internalReplication: - <<: *TypeStringBool - description: | - optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise - allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard - override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` - replicasCount: - type: integer - description: | - optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, - shard contains 1 replica by default - override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` - minimum: 1 - replicas: - type: array - description: | - optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - insecure: - <<: *TypeStringBool - description: | - optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: | - optional, open secure ports - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - tlsPort: - type: integer - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpsPort: - type: integer - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` - replicas: - type: array - description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` - shardsCount: - type: integer - description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" - minimum: 1 - shards: - type: array - description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - insecure: - <<: *TypeStringBool - description: | - optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: | - optional, open secure ports - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - tlsPort: - type: integer - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpsPort: - type: integer - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` - templates: - type: object - description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" - # nullable: true - properties: - hostTemplates: - type: array - description: "hostTemplate will use during apply to generate `clickhose-server` config files" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" - type: string - portDistribution: - type: array - description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" - enum: - # List PortDistributionXXX constants - - "" - - "Unspecified" - - "ClusterScopeIndex" - spec: - # Host - type: object - properties: - name: - type: string - description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - insecure: - <<: *TypeStringBool - description: | - optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: | - optional, open secure ports - tcpPort: - type: integer - description: | - optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` - More info: https://clickhouse.tech/docs/en/interfaces/tcp/ - minimum: 1 - maximum: 65535 - tlsPort: - type: integer - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` - More info: https://clickhouse.tech/docs/en/interfaces/http/ - minimum: 1 - maximum: 65535 - httpsPort: - type: integer - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - templates: - <<: *TypeTemplateNames - description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" - - podTemplates: - type: array - description: | - podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone - More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" - generateName: - type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - zone: - type: object - description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - #required: - # - values - properties: - key: - type: string - description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" - values: - type: array - description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" - # nullable: true - items: - type: string - distribution: - type: string - description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - enum: - - "" - - "Unspecified" - - "OnePerHost" - podDistribution: - type: array - description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "you can define multiple affinity policy types" - enum: - # List PodDistributionXXX constants - - "" - - "Unspecified" - - "ClickHouseAntiAffinity" - - "ShardAntiAffinity" - - "ReplicaAntiAffinity" - - "AnotherNamespaceAntiAffinity" - - "AnotherClickHouseInstallationAntiAffinity" - - "AnotherClusterAntiAffinity" - - "MaxNumberPerNode" - - "NamespaceAffinity" - - "ClickHouseInstallationAffinity" - - "ClusterAffinity" - - "ShardAffinity" - - "ReplicaAffinity" - - "PreviousTailAffinity" - - "CircularReplication" - scope: - type: string - description: "scope for apply each podDistribution" - enum: - # list PodDistributionScopeXXX constants - - "" - - "Unspecified" - - "Shard" - - "Replica" - - "Cluster" - - "ClickHouseInstallation" - - "Namespace" - number: - type: integer - description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" - minimum: 0 - maximum: 65535 - topologyKey: - type: string - description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" - metadata: - type: object - description: | - allows pass standard object's metadata from template to Pod - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify PodSpec - type: object - description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - - volumeClaimTemplates: - type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, - cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` - provisioner: *TypePVCProvisioner - reclaimPolicy: *TypePVCReclaimPolicy - metadata: - type: object - description: | - allows to pass standard object's metadata from template to PVC - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - allows define all aspects of `PVC` resource - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims - # nullable: true - x-kubernetes-preserve-unknown-fields: true - serviceTemplates: - type: array - description: | - allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - chi-level `chi.spec.defaults.templates.serviceTemplate` - cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` - generateName: - type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - metadata: - # TODO specify ObjectMeta - type: object - description: | - allows pass standard object's metadata from template to Service - Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify ServiceSpec - type: object - description: | - describe behavior of generated Service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - useTemplates: - type: array - description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "name of `ClickHouseInstallationTemplate` (chit) resource" - namespace: - type: string - description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" - useType: - type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" - enum: - # List useTypeXXX constants from model - - "" - - "merge" ---- -# Template Parameters: -# -# KIND=ClickHouseInstallationTemplate -# SINGULAR=clickhouseinstallationtemplate -# PLURAL=clickhouseinstallationtemplates -# SHORT=chit -# OPERATOR_VERSION=0.23.6 -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhouseinstallationtemplates.clickhouse.altinity.com - labels: - clickhouse.altinity.com/chop: 0.23.6 -spec: - group: clickhouse.altinity.com - scope: Namespaced - names: - kind: ClickHouseInstallationTemplate - singular: clickhouseinstallationtemplate - plural: clickhouseinstallationtemplates - shortNames: - - chit - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: version - type: string - description: Operator version - priority: 1 # show in wide view - jsonPath: .status.chop-version - - name: clusters - type: integer - description: Clusters count - jsonPath: .status.clusters - - name: shards - type: integer - description: Shards count - priority: 1 # show in wide view - jsonPath: .status.shards - - name: hosts - type: integer - description: Hosts count - jsonPath: .status.hosts - - name: taskID - type: string - description: TaskID - priority: 1 # show in wide view - jsonPath: .status.taskID - - name: status - type: string - description: CHI status - jsonPath: .status.status - - name: hosts-unchanged - type: integer - description: Unchanged hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsUnchanged - - name: hosts-updated - type: integer - description: Updated hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsUpdated - - name: hosts-added - type: integer - description: Added hosts count - priority: 1 # show in wide view - jsonPath: .status.hostsAdded - - name: hosts-completed - type: integer - description: Completed hosts count - jsonPath: .status.hostsCompleted - - name: hosts-deleted - type: integer - description: Hosts deleted count - priority: 1 # show in wide view - jsonPath: .status.hostsDeleted - - name: hosts-delete - type: integer - description: Hosts to be deleted count - priority: 1 # show in wide view - jsonPath: .status.hostsDelete - - name: endpoint - type: string - description: Client access endpoint - priority: 1 # show in wide view - jsonPath: .status.endpoint - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - schema: - openAPIV3Schema: - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" - type: object - required: - - spec - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - type: object - description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" - properties: - chop-version: - type: string - description: "ClickHouse operator version" - chop-commit: - type: string - description: "ClickHouse operator git commit SHA" - chop-date: - type: string - description: "ClickHouse operator build date" - chop-ip: - type: string - description: "IP address of the operator's pod which managed this CHI" - clusters: - type: integer - minimum: 0 - description: "Clusters count" - shards: - type: integer - minimum: 0 - description: "Shards count" - replicas: - type: integer - minimum: 0 - description: "Replicas count" - hosts: - type: integer - minimum: 0 - description: "Hosts count" - status: - type: string - description: "Status" - taskID: - type: string - description: "Current task id" - taskIDsStarted: - type: array - description: "Started task ids" - nullable: true - items: - type: string - taskIDsCompleted: - type: array - description: "Completed task ids" - nullable: true - items: - type: string - action: - type: string - description: "Action" - actions: - type: array - description: "Actions" - nullable: true - items: - type: string - error: - type: string - description: "Last error" - errors: - type: array - description: "Errors" - nullable: true - items: - type: string - hostsUnchanged: - type: integer - minimum: 0 - description: "Unchanged Hosts count" - hostsUpdated: - type: integer - minimum: 0 - description: "Updated Hosts count" - hostsAdded: - type: integer - minimum: 0 - description: "Added Hosts count" - hostsCompleted: - type: integer - minimum: 0 - description: "Completed Hosts count" - hostsDeleted: - type: integer - minimum: 0 - description: "Deleted Hosts count" - hostsDelete: - type: integer - minimum: 0 - description: "About to delete Hosts count" - pods: - type: array - description: "Pods" - nullable: true - items: - type: string - pod-ips: - type: array - description: "Pod IPs" - nullable: true - items: - type: string - fqdns: - type: array - description: "Pods FQDNs" - nullable: true - items: - type: string - endpoint: - type: string - description: "Endpoint" - generation: - type: integer - minimum: 0 - description: "Generation" - normalized: - type: object - description: "Normalized CHI requested" - x-kubernetes-preserve-unknown-fields: true - normalizedCompleted: - type: object - description: "Normalized CHI completed" - x-kubernetes-preserve-unknown-fields: true - hostsWithTablesCreated: - type: array - description: "List of hosts with tables created by the operator" - nullable: true - items: - type: string - usedTemplates: - type: array - description: "List of templates used to build this CHI" - nullable: true - x-kubernetes-preserve-unknown-fields: true - items: - type: object - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - # x-kubernetes-preserve-unknown-fields: true - description: | - Specification of the desired behavior of one or more ClickHouse clusters - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md - properties: - taskID: - type: string - description: | - Allows to define custom taskID for CHI update and watch status of this update execution. - Displayed in all .status.taskID* fields. - By default (if not filled) every update of CHI manifest will generate random taskID - stop: &TypeStringBool - type: string - description: | - Allows to stop all ClickHouse clusters defined in a CHI. - Works as the following: - - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. - - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. - enum: - # List StringBoolXXX constants from model - - "" - - "0" - - "1" - - "False" - - "false" - - "True" - - "true" - - "No" - - "no" - - "Yes" - - "yes" - - "Off" - - "off" - - "On" - - "on" - - "Disable" - - "disable" - - "Enable" - - "enable" - - "Disabled" - - "disabled" - - "Enabled" - - "enabled" - restart: - type: string - description: | - In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. - This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. - enum: - - "" - - "RollingUpdate" - troubleshoot: - <<: *TypeStringBool - description: | - Allows to troubleshoot Pods during CrashLoopBack state. - This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. - Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts - and give time to troubleshoot via CLI. - Liveness and Readiness probes are disabled as well. - namespaceDomainPattern: - type: string - description: | - Custom domain pattern which will be used for DNS names of `Service` or `Pod`. - Typical use scenario - custom cluster domain in Kubernetes cluster - Example: %s.svc.my.test - templating: - type: object - # nullable: true - description: | - Optional, applicable inside ClickHouseInstallationTemplate only. - Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." - properties: - policy: - type: string - description: | - When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate - will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. - Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. - enum: - - "" - - "auto" - - "manual" - chiSelector: - type: object - description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - reconciling: - type: object - description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" - # nullable: true - properties: - policy: - type: string - description: | - DISCUSSED TO BE DEPRECATED - Syntax sugar - Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config - Possible values: - - wait - should wait to exclude host, complete queries and include host back into the cluster - - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster - enum: - - "" - - "wait" - - "nowait" - configMapPropagationTimeout: - type: integer - description: | - Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` - More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically - minimum: 0 - maximum: 3600 - cleanup: - type: object - description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" - # nullable: true - properties: - unknownObjects: - type: object - description: | - Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, - but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. - Default behavior is `Delete`" - # nullable: true - properties: - statefulSet: &TypeObjectsCleanup - type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" - enum: - # List ObjectsCleanupXXX constants from model - - "" - - "Retain" - - "Delete" - pvc: - type: string - <<: *TypeObjectsCleanup - description: "Behavior policy for unknown PVC, `Delete` by default" - configMap: - <<: *TypeObjectsCleanup - description: "Behavior policy for unknown ConfigMap, `Delete` by default" - service: - <<: *TypeObjectsCleanup - description: "Behavior policy for unknown Service, `Delete` by default" - reconcileFailedObjects: - type: object - description: | - Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. - Default behavior is `Retain`" - # nullable: true - properties: - statefulSet: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed StatefulSet, `Retain` by default" - pvc: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed PVC, `Retain` by default" - configMap: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed ConfigMap, `Retain` by default" - service: - <<: *TypeObjectsCleanup - description: "Behavior policy for failed Service, `Retain` by default" - defaults: - type: object - description: | - define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults - # nullable: true - properties: - replicasUseFQDN: - <<: *TypeStringBool - description: | - define should replicas be specified by FQDN in ``. - In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup - "yes" by default - distributedDDL: - type: object - description: | - allows change `` settings - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl - # nullable: true - properties: - profile: - type: string - description: "Settings from this profile will be used to execute DDL queries" - storageManagement: - type: object - description: default storage management options - properties: - provisioner: &TypePVCProvisioner - type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" - enum: - - "" - - "StatefulSet" - - "Operator" - reclaimPolicy: &TypePVCReclaimPolicy - type: string - description: | - defines behavior of `PVC` deletion. - `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet - enum: - - "" - - "Retain" - - "Delete" - templates: &TypeTemplateNames - type: object - description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" - # nullable: true - properties: - hostTemplate: - type: string - description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" - podTemplate: - type: string - description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - dataVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - logVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - serviceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" - clusterServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" - shardServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" - replicaServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" - volumeClaimTemplate: - type: string - description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" - configuration: - type: object - description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" - # nullable: true - properties: - zookeeper: &TypeZookeeperConfig - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ - currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` - More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper - # nullable: true - properties: - nodes: - type: array - description: "describe every available zookeeper cluster node for interaction" - # nullable: true - items: - type: object - #required: - # - host - properties: - host: - type: string - description: "dns name or ip address for Zookeeper node" - port: - type: integer - description: "TCP port which used to connect to Zookeeper node" - minimum: 0 - maximum: 65535 - secure: - <<: *TypeStringBool - description: "if a secure connection to Zookeeper is required" - session_timeout_ms: - type: integer - description: "session timeout during connect to Zookeeper" - operation_timeout_ms: - type: integer - description: "one operation timeout during Zookeeper transactions" - root: - type: string - description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" - identity: - type: string - description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" - users: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure password hashed, authorization restrictions, database level security row filters etc. - More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers - # nullable: true - x-kubernetes-preserve-unknown-fields: true - profiles: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of settings profile - More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles - # nullable: true - x-kubernetes-preserve-unknown-fields: true - quotas: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of resource quotas - More details: https://clickhouse.tech/docs/en/operations/quotas/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas - # nullable: true - x-kubernetes-preserve-unknown-fields: true - settings: &TypeSettings - type: object - description: | - allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings - # nullable: true - x-kubernetes-preserve-unknown-fields: true - files: &TypeFiles - type: object - description: | - allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - every key in this object is the file name - every value in this object is the file content - you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html - each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored - More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml - # nullable: true - x-kubernetes-preserve-unknown-fields: true - clusters: - type: array - description: | - describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level - every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` - all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` - Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" - minLength: 1 - # See namePartClusterMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - zookeeper: - <<: *TypeZookeeperConfig - description: | - optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.zookeeper` settings - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster - override top-level `chi.spec.configuration.templates` - schemaPolicy: - type: object - description: | - describes how schema is propagated within replicas and shards - properties: - replica: - type: string - description: "how schema is propagated within a replica" - enum: - # List SchemaPolicyReplicaXXX constants from model - - "" - - "None" - - "All" - shard: - type: string - description: "how schema is propagated between shards" - enum: - # List SchemaPolicyShardXXX constants from model - - "" - - "None" - - "All" - - "DistributedTablesOnly" - insecure: - <<: *TypeStringBool - description: optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: optional, open secure ports for cluster - secret: - type: object - description: "optional, shared secret value to secure cluster communications" - properties: - auto: - <<: *TypeStringBool - description: "Auto-generate shared secret value to secure cluster communications" - value: - description: "Cluster shared secret value in plain text" - type: string - valueFrom: - description: "Cluster shared secret source" - type: object - properties: - secretKeyRef: - description: | - Selects a key of a secret in the clickhouse installation namespace. - Should not be used if value is not empty. - type: object - properties: - name: - description: | - Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - name - - key - layout: - type: object - description: | - describe current cluster layout, how much shards in cluster, how much replica in shard - allows override settings on each shard and replica separatelly - # nullable: true - properties: - type: - type: string - description: "DEPRECATED - to be removed soon" - shardsCount: - type: integer - description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" - replicasCount: - type: integer - description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" - shards: - type: array - description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - definitionType: - type: string - description: "DEPRECATED - to be removed soon" - weight: - type: integer - description: | - optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - internalReplication: - <<: *TypeStringBool - description: | - optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise - allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard - override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` - replicasCount: - type: integer - description: | - optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, - shard contains 1 replica by default - override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` - minimum: 1 - replicas: - type: array - description: | - optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - insecure: - <<: *TypeStringBool - description: | - optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: | - optional, open secure ports - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - tlsPort: - type: integer - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpsPort: - type: integer - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` - replicas: - type: array - description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` - shardsCount: - type: integer - description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" - minimum: 1 - shards: - type: array - description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - insecure: - <<: *TypeStringBool - description: | - optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: | - optional, open secure ports - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - tlsPort: - type: integer - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpsPort: - type: integer - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` - templates: - type: object - description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" - # nullable: true - properties: - hostTemplates: - type: array - description: "hostTemplate will use during apply to generate `clickhose-server` config files" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" - type: string - portDistribution: - type: array - description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" - enum: - # List PortDistributionXXX constants - - "" - - "Unspecified" - - "ClusterScopeIndex" - spec: - # Host - type: object - properties: - name: - type: string - description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - insecure: - <<: *TypeStringBool - description: | - optional, open insecure ports for cluster, defaults to "yes" - secure: - <<: *TypeStringBool - description: | - optional, open secure ports - tcpPort: - type: integer - description: | - optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` - More info: https://clickhouse.tech/docs/en/interfaces/tcp/ - minimum: 1 - maximum: 65535 - tlsPort: - type: integer - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` - More info: https://clickhouse.tech/docs/en/interfaces/http/ - minimum: 1 - maximum: 65535 - httpsPort: - type: integer - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - templates: - <<: *TypeTemplateNames - description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" - - podTemplates: - type: array - description: | - podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone - More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" - generateName: - type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - zone: - type: object - description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - #required: - # - values - properties: - key: - type: string - description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" - values: - type: array - description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" - # nullable: true - items: - type: string - distribution: - type: string - description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - enum: - - "" - - "Unspecified" - - "OnePerHost" - podDistribution: - type: array - description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "you can define multiple affinity policy types" - enum: - # List PodDistributionXXX constants - - "" - - "Unspecified" - - "ClickHouseAntiAffinity" - - "ShardAntiAffinity" - - "ReplicaAntiAffinity" - - "AnotherNamespaceAntiAffinity" - - "AnotherClickHouseInstallationAntiAffinity" - - "AnotherClusterAntiAffinity" - - "MaxNumberPerNode" - - "NamespaceAffinity" - - "ClickHouseInstallationAffinity" - - "ClusterAffinity" - - "ShardAffinity" - - "ReplicaAffinity" - - "PreviousTailAffinity" - - "CircularReplication" - scope: - type: string - description: "scope for apply each podDistribution" - enum: - # list PodDistributionScopeXXX constants - - "" - - "Unspecified" - - "Shard" - - "Replica" - - "Cluster" - - "ClickHouseInstallation" - - "Namespace" - number: - type: integer - description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" - minimum: 0 - maximum: 65535 - topologyKey: - type: string - description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" - metadata: - type: object - description: | - allows pass standard object's metadata from template to Pod - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify PodSpec - type: object - description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - - volumeClaimTemplates: - type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, - cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` - provisioner: *TypePVCProvisioner - reclaimPolicy: *TypePVCReclaimPolicy - metadata: - type: object - description: | - allows to pass standard object's metadata from template to PVC - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - allows define all aspects of `PVC` resource - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims - # nullable: true - x-kubernetes-preserve-unknown-fields: true - serviceTemplates: - type: array - description: | - allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - chi-level `chi.spec.defaults.templates.serviceTemplate` - cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` - generateName: - type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - metadata: - # TODO specify ObjectMeta - type: object - description: | - allows pass standard object's metadata from template to Service - Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify ServiceSpec - type: object - description: | - describe behavior of generated Service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - useTemplates: - type: array - description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "name of `ClickHouseInstallationTemplate` (chit) resource" - namespace: - type: string - description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" - useType: - type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" - enum: - # List useTypeXXX constants from model - - "" - - "merge" ---- -# Template Parameters: -# -# NONE -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhouseoperatorconfigurations.clickhouse.altinity.com - labels: - clickhouse.altinity.com/chop: 0.23.6 -spec: - group: clickhouse.altinity.com - scope: Namespaced - names: - kind: ClickHouseOperatorConfiguration - singular: clickhouseoperatorconfiguration - plural: clickhouseoperatorconfigurations - shortNames: - - chopconf - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: namespaces - type: string - description: Watch namespaces - jsonPath: .status - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - schema: - openAPIV3Schema: - type: object - description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" - x-kubernetes-preserve-unknown-fields: true - properties: - status: - type: object - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - Allows to define settings of the clickhouse-operator. - More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml - Check into etc-clickhouse-operator* ConfigMaps if you need more control - x-kubernetes-preserve-unknown-fields: true - properties: - watch: - type: object - description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" - properties: - namespaces: - type: array - description: "List of namespaces where clickhouse-operator watches for events." - items: - type: string - clickhouse: - type: object - description: "Clickhouse related parameters used by clickhouse-operator" - properties: - configuration: - type: object - properties: - file: - type: object - properties: - path: - type: object - description: | - Each 'path' can be either absolute or relative. - In case path is absolute - it is used as is. - In case path is relative - it is relative to the folder where configuration file you are reading right now is located. - properties: - common: - type: string - description: | - Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. - Default value - config.d - host: - type: string - description: | - Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. - Default value - conf.d - user: - type: string - description: | - Path to the folder where ClickHouse configuration files with users settings are located. - Files are common for all instances within a CHI. - Default value - users.d - user: - type: object - description: "Default parameters for any user which will create" - properties: - default: - type: object - properties: - profile: - type: string - description: "ClickHouse server configuration `...` for any " - quota: - type: string - description: "ClickHouse server configuration `...` for any " - networksIP: - type: array - description: "ClickHouse server configuration `...` for any " - items: - type: string - password: - type: string - description: "ClickHouse server configuration `...` for any " - network: - type: object - description: "Default network parameters for any user which will create" - properties: - hostRegexpTemplate: - type: string - description: "ClickHouse server configuration `...` for any " - configurationRestartPolicy: - type: object - description: "Configuration restart policy describes what configuration changes require ClickHouse restart" - properties: - rules: - type: array - description: "Array of set of rules per specified ClickHouse versions" - items: - type: object - properties: - version: - type: string - description: "ClickHouse version expression" - rules: - type: array - description: "Set of configuration rules for specified ClickHouse version" - items: - type: object - description: "setting: value pairs for configuration restart policy" - access: - type: object - description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" - properties: - scheme: - type: string - description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" - username: - type: string - description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" - password: - type: string - description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" - rootCA: - type: string - description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" - secret: - type: object - properties: - namespace: - type: string - description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" - name: - type: string - description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" - port: - type: integer - minimum: 1 - maximum: 65535 - description: "Port to be used by operator to connect to ClickHouse instances" - timeouts: - type: object - description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" - properties: - connect: - type: integer - minimum: 1 - maximum: 10 - description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." - query: - type: integer - minimum: 1 - maximum: 600 - description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." - metrics: - type: object - description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" - properties: - timeouts: - type: object - description: | - Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances - Specified in seconds. - properties: - collect: - type: integer - minimum: 1 - maximum: 600 - description: | - Timeout used to limit metrics collection request. In seconds. - Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. - All collected metrics are returned. - template: - type: object - description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" - properties: - chi: - type: object - properties: - policy: - type: string - description: | - CHI template updates handling policy - Possible policy values: - - ReadOnStart. Accept CHIT updates on the operators start only. - - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI - enum: - - "" - - "ReadOnStart" - - "ApplyOnNextReconcile" - path: - type: string - description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." - reconcile: - type: object - description: "allow tuning reconciling process" - properties: - runtime: - type: object - description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" - properties: - reconcileCHIsThreadsNumber: - type: integer - minimum: 1 - maximum: 65535 - description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" - reconcileShardsThreadsNumber: - type: integer - minimum: 1 - maximum: 65535 - description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" - reconcileShardsMaxConcurrencyPercent: - type: integer - minimum: 0 - maximum: 100 - description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." - statefulSet: - type: object - description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" - properties: - create: - type: object - description: "Behavior during create StatefulSet" - properties: - onFailure: - type: string - description: | - What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds - Possible options: - 1. abort - do nothing, just break the process and wait for admin. - 2. delete - delete newly created problematic StatefulSet. - 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. - update: - type: object - description: "Behavior during update StatefulSet" - properties: - timeout: - type: integer - description: "How many seconds to wait for created/updated StatefulSet to be Ready" - pollInterval: - type: integer - description: "How many seconds to wait between checks for created/updated StatefulSet status" - onFailure: - type: string - description: | - What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds - Possible options: - 1. abort - do nothing, just break the process and wait for admin. - 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. - 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. - host: - type: object - description: | - Whether the operator during reconcile procedure should wait for a ClickHouse host: - - to be excluded from a ClickHouse cluster - - to complete all running queries - - to be included into a ClickHouse cluster - respectfully before moving forward - - properties: - wait: - type: object - properties: - exclude: &TypeStringBool - type: string - description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" - enum: - # List StringBoolXXX constants from model - - "" - - "0" - - "1" - - "False" - - "false" - - "True" - - "true" - - "No" - - "no" - - "Yes" - - "yes" - - "Off" - - "off" - - "On" - - "on" - - "Disable" - - "disable" - - "Enable" - - "enable" - - "Disabled" - - "disabled" - - "Enabled" - - "enabled" - queries: - <<: *TypeStringBool - description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" - include: - <<: *TypeStringBool - description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" - annotation: - type: object - description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" - properties: - include: - type: array - description: | - When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, - include annotations with names from the following list - items: - type: string - exclude: - type: array - description: | - When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, - exclude annotations with names from the following list - items: - type: string - label: - type: object - description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" - properties: - include: - type: array - description: | - When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, - include labels from the following list - items: - type: string - exclude: - type: array - items: - type: string - description: | - When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, - exclude labels from the following list - appendScope: - <<: *TypeStringBool - description: | - Whether to append *Scope* labels to StatefulSet and Pod - - "LabelShardScopeIndex" - - "LabelReplicaScopeIndex" - - "LabelCHIScopeIndex" - - "LabelCHIScopeCycleSize" - - "LabelCHIScopeCycleIndex" - - "LabelCHIScopeCycleOffset" - - "LabelClusterScopeIndex" - - "LabelClusterScopeCycleSize" - - "LabelClusterScopeCycleIndex" - - "LabelClusterScopeCycleOffset" - statefulSet: - type: object - description: "define StatefulSet-specific parameters" - properties: - revisionHistoryLimit: - type: integer - description: | - revisionHistoryLimit is the maximum number of revisions that will be - maintained in the StatefulSet's revision history. - Look details in `statefulset.spec.revisionHistoryLimit` - pod: - type: object - description: "define pod specific parameters" - properties: - terminationGracePeriod: - type: integer - description: | - Optional duration in seconds the pod needs to terminate gracefully. - Look details in `pod.spec.terminationGracePeriodSeconds` - logger: - type: object - description: "allow setup clickhouse-operator logger behavior" - properties: - logtostderr: - type: string - description: "boolean, allows logs to stderr" - alsologtostderr: - type: string - description: "boolean allows logs to stderr and files both" - v: - type: string - description: "verbosity level of clickhouse-operator log, default - 1 max - 9" - stderrthreshold: - type: string - vmodule: - type: string - description: | - Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. - Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. - log_backtrace_at: - type: string - description: | - It can be set to a file and line number with a logging line. - Ex.: file.go:123 - Each time when this line is being executed, a stack trace will be written to the Info log. ---- -# Template Parameters: -# -# OPERATOR_VERSION=0.23.6 -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com - labels: - clickhouse-keeper.altinity.com/chop: 0.23.6 -spec: - group: clickhouse-keeper.altinity.com - scope: Namespaced - names: - kind: ClickHouseKeeperInstallation - singular: clickhousekeeperinstallation - plural: clickhousekeeperinstallations - shortNames: - - chk - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: status - type: string - description: CHK status - jsonPath: .status.status - - name: replicas - type: integer - description: Replica count - priority: 1 # show in wide view - jsonPath: .status.replicas - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - required: - - spec - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster" - properties: - apiVersion: - type: string - description: | - APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - kind: - type: string - description: | - Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - metadata: - type: object - status: - type: object - description: | - Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints - properties: - chop-version: - type: string - description: "ClickHouse operator version" - chop-commit: - type: string - description: "ClickHouse operator git commit SHA" - chop-date: - type: string - description: "ClickHouse operator build date" - chop-ip: - type: string - description: "IP address of the operator's pod which managed this CHI" - status: - type: string - description: "Status" - replicas: - type: integer - format: int32 - description: Replicas is the number of number of desired replicas in the cluster - readyReplicas: - type: array - description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster - items: - type: object - properties: - host: - type: string - description: dns name or ip address for Keeper node - port: - type: integer - minimum: 0 - maximum: 65535 - description: TCP port which used to connect to Keeper node - secure: - type: string - description: if a secure connection to Keeper is required - normalized: - type: object - description: "Normalized CHK requested" - x-kubernetes-preserve-unknown-fields: true - normalizedCompleted: - type: object - description: "Normalized CHK completed" - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: KeeperSpec defines the desired state of a Keeper cluster - properties: - namespaceDomainPattern: - type: string - description: | - Custom domain pattern which will be used for DNS names of `Service` or `Pod`. - Typical use scenario - custom cluster domain in Kubernetes cluster - Example: %s.svc.my.test - replicas: - type: integer - format: int32 - description: | - Replicas is the expected size of the keeper cluster. - The valid range of size is from 1 to 7. - minimum: 1 - maximum: 7 - configuration: - type: object - description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" - # nullable: true - properties: - settings: - type: object - description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance" - x-kubernetes-preserve-unknown-fields: true - clusters: - type: array - description: | - describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources" - minLength: 1 - # See namePartClusterMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - layout: - type: object - description: | - describe current cluster layout, how many replicas - # nullable: true - properties: - replicasCount: - type: integer - description: "how many replicas in ClickHouseKeeper cluster" - templates: - type: object - description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" - # nullable: true - properties: - podTemplates: - type: array - description: | - podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone - More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" - metadata: - type: object - description: | - allows pass standard object's metadata from template to Pod - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify PodSpec - type: object - description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - - volumeClaimTemplates: - type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, - cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` - metadata: - type: object - description: | - allows to pass standard object's metadata from template to PVC - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - allows define all aspects of `PVC` resource - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims - # nullable: true - x-kubernetes-preserve-unknown-fields: true - serviceTemplates: - type: array - description: | - allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - chi-level `chi.spec.defaults.templates.serviceTemplate` - cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` - metadata: - # TODO specify ObjectMeta - type: object - description: | - allows pass standard object's metadata from template to Service - Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify ServiceSpec - type: object - description: | - describe behavior of generated Service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true diff --git a/deployment/helm_chart/opik/templates/clickhouse-operator-install-bundle.yaml b/deployment/helm_chart/opik/templates/clickhouse-operator-install-bundle.yaml deleted file mode 100644 index e8ba75fa2..000000000 --- a/deployment/helm_chart/opik/templates/clickhouse-operator-install-bundle.yaml +++ /dev/null @@ -1,1201 +0,0 @@ ---- -# Setup ServiceAccount -apiVersion: v1 -kind: ServiceAccount -metadata: - name: clickhouse-operator - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 ---- -# Template Parameters: -# -# NAMESPACE={{ .Release.namespace }} -# COMMENT=# -# ROLE_KIND=Role -# ROLE_NAME=clickhouse-operator -# ROLE_BINDING_KIND=RoleBinding -# ROLE_BINDING_NAME=clickhouse-operator -# - - -# Specifies either -# ClusterRole -# or -# Role -# to be bound to ServiceAccount. -# ClusterRole is namespace-less and must have unique name -# Role is namespace-bound -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: clickhouse-operator - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 -rules: - - # - # Core API group - # - - - apiGroups: - - "" - resources: - - configmaps - - services - - persistentvolumeclaims - - secrets - verbs: - - get - - list - - patch - - update - - watch - - create - - delete - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - patch - - update - - watch - - delete - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - # - # apps.* resources - # - - - apiGroups: - - apps - resources: - - statefulsets - verbs: - - get - - list - - patch - - update - - watch - - create - - delete - - apiGroups: - - apps - resources: - - replicasets - verbs: - - get - - patch - - update - - delete - # The operator deployment personally, identified by name - - apiGroups: - - apps - resources: - - deployments - resourceNames: - - clickhouse-operator - verbs: - - get - - patch - - update - - delete - - # - # policy.* resources - # - - - apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - get - - list - - patch - - update - - watch - - create - - delete - - # - # apiextensions - # - - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - # - # The operator's specific Custom Resources - # - - # clickhouse - related resources - - apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations - verbs: - - get - - list - - watch - - patch - - update - - delete - - apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallationtemplates - - clickhouseoperatorconfigurations - verbs: - - get - - list - - watch - - apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations/finalizers - - clickhouseinstallationtemplates/finalizers - - clickhouseoperatorconfigurations/finalizers - verbs: - - update - - apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations/status - - clickhouseinstallationtemplates/status - - clickhouseoperatorconfigurations/status - verbs: - - get - - update - - patch - - create - - delete - - # clickhouse-keeper - related resources - - apiGroups: - - clickhouse-keeper.altinity.com - resources: - - clickhousekeeperinstallations - verbs: - - get - - list - - watch - - patch - - update - - delete - - apiGroups: - - clickhouse-keeper.altinity.com - resources: - - clickhousekeeperinstallations/finalizers - verbs: - - update - - apiGroups: - - clickhouse-keeper.altinity.com - resources: - - clickhousekeeperinstallations/status - verbs: - - get - - update - - patch - - create - - delete ---- -# Specifies either -# ClusterRoleBinding between ClusterRole and ServiceAccount. -# or -# RoleBinding between Role and ServiceAccount. -# ClusterRoleBinding is namespace-less and must have unique name -# RoleBinding is namespace-bound -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: clickhouse-operator - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: clickhouse-operator -subjects: - - kind: ServiceAccount - name: clickhouse-operator - namespace: {{ .Release.namespace }} ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-files -# NAMESPACE=namespace: {{ .Release.namespace }} -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-files - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -data: - config.yaml: | - # IMPORTANT - # This file is auto-generated - # Do not edit this file - all changes would be lost - # Edit appropriate template in the following folder: - # deploy/builder/templates-config - # IMPORTANT - # - # Template parameters available: - # WATCH_NAMESPACES= - # CH_USERNAME_PLAIN= - # CH_PASSWORD_PLAIN= - # CH_CREDENTIALS_SECRET_NAMESPACE= - # CH_CREDENTIALS_SECRET_NAME=clickhouse-operator - # VERBOSITY=1 - - ################################################ - ## - ## Watch section - ## - ################################################ - watch: - # List of namespaces where clickhouse-operator watches for events. - # Concurrently running operators should watch on different namespaces. - # IMPORTANT - # Regexp is applicable. - #namespaces: ["dev", "test"] - namespaces: [] - - clickhouse: - configuration: - ################################################ - ## - ## Configuration files section - ## - ################################################ - file: - # Each 'path' can be either absolute or relative. - # In case path is absolute - it is used as is - # In case path is relative - it is relative to the folder where configuration file you are reading right now is located. - path: - # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. - common: config.d - # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. - host: conf.d - # Path to the folder where ClickHouse configuration files with users' settings are located. - # Files are common for all instances within a CHI. - user: users.d - ################################################ - ## - ## Configuration users section - ## - ################################################ - user: - # Default settings for user accounts, created by the operator. - # IMPORTANT. These are not access credentials or settings for 'default' user account, - # it is a template for filling out missing fields for all user accounts to be created by the operator, - # with the following EXCEPTIONS: - # 1. 'default' user account DOES NOT use provided password, but uses all the rest of the fields. - # Password for 'default' user account has to be provided explicitly, if to be used. - # 2. CHOP user account DOES NOT use: - # - profile setting. It uses predefined profile called 'clickhouse_operator' - # - quota setting. It uses empty quota name. - # - networks IP setting. Operator specifies 'networks/ip' user setting to match operators' pod IP only. - # - password setting. Password for CHOP account is used from 'clickhouse.access.*' section - default: - # Default values for ClickHouse user account(s) created by the operator - # 1. user/profile - string - # 2. user/quota - string - # 3. user/networks/ip - multiple strings - # 4. user/password - string - # These values can be overwritten on per-user basis. - profile: "default" - quota: "default" - networksIP: - - "::1" - - "127.0.0.1" - password: "default" - ################################################ - ## - ## Configuration network section - ## - ################################################ - network: - # Default host_regexp to limit network connectivity from outside - hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" - - ################################################ - ## - ## Configuration restart policy section - ## Configuration restart policy describes what configuration changes require ClickHouse restart - ## - ################################################ - configurationRestartPolicy: - rules: - # IMPORTANT! - # Special version of "*" - default version - has to satisfy all ClickHouse versions. - # Default version will also be used in case ClickHouse version is unknown. - # ClickHouse version may be unknown due to host being down - for example, because of incorrect "settings" section. - # ClickHouse is not willing to start in case incorrect/unknown settings are provided in config file. - - version: "*" - rules: - # see https://kb.altinity.com/altinity-kb-setup-and-maintenance/altinity-kb-server-config-files/#server-config-configxml-sections-which-dont-require-restart - # to be replaced with "select * from system.server_settings where changeable_without_restart = 'No'" - - - settings/*: "yes" - - # single values - - settings/access_control_path: "no" - - settings/dictionaries_config: "no" - - settings/max_server_memory_*: "no" - - settings/max_*_to_drop: "no" - - settings/max_concurrent_queries: "no" - - settings/models_config: "no" - - settings/user_defined_executable_functions_config: "no" - - # structured XML - - settings/logger/*: "no" - - settings/macros/*: "no" - - settings/remote_servers/*: "no" - - settings/user_directories/*: "no" - - - zookeeper/*: "yes" - - - files/*.xml: "yes" - - files/config.d/*.xml: "yes" - - files/config.d/*dict*.xml: "no" - - # exceptions in default profile - - profiles/default/background_*_pool_size: "yes" - - profiles/default/max_*_for_server: "yes" - - version: "21.*" - rules: - - settings/logger: "yes" - - ################################################# - ## - ## Access to ClickHouse instances - ## - ################################################ - access: - # Possible values for 'scheme' are: - # 1. http - force http to be used to connect to ClickHouse instances - # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" - # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. - # These credentials are used for: - # 1. Metrics requests - # 2. Schema maintenance - # 3. DROP DNS CACHE - # User with these credentials can be specified in additional ClickHouse .xml config files, - # located in 'clickhouse.configuration.file.path.user' folder - username: "" - password: "" - rootCA: "" - - # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. - # Can be used instead of explicitly specified username and password available in sections: - # - clickhouse.access.username - # - clickhouse.access.password - # Secret should have two keys: - # 1. username - # 2. password - secret: - # Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running. - namespace: "" - # Empty `name` means no k8s Secret would be looked for - name: "clickhouse-operator" - # Port where to connect to ClickHouse instances to - port: 8123 - - # Timeouts used to limit connection and queries from the operator to ClickHouse instances - # Specified in seconds. - timeouts: - # Timout to setup connection from the operator to ClickHouse instances. In seconds. - connect: 1 - # Timout to perform SQL query from the operator to ClickHouse instances. In seconds. - query: 4 - - ################################################# - ## - ## Metrics collection - ## - ################################################ - - metrics: - # Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances - # Specified in seconds. - timeouts: - # Timeout used to limit metrics collection request. In seconds. - # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. - # All collected metrics are returned. - collect: 9 - - ################################################ - ## - ## Template(s) management section - ## - ################################################ - template: - chi: - # CHI template updates handling policy - # Possible policy values: - # - ReadOnStart. Accept CHIT updates on the operators start only. - # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI - policy: ApplyOnNextReconcile - - # Path to the folder where ClickHouseInstallation templates .yaml manifests are located. - # Templates are added to the list of all templates and used when CHI is reconciled. - # Templates are applied in sorted alpha-numeric order. - path: templates.d - - ################################################ - ## - ## Reconcile section - ## - ################################################ - reconcile: - # Reconcile runtime settings - runtime: - # Max number of concurrent CHI reconciles in progress - reconcileCHIsThreadsNumber: 10 - - # The operator reconciles shards concurrently in each CHI with the following limitations: - # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently - # can not be greater than 'reconcileShardsThreadsNumber'. - # 2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently - # can not be greater than 'reconcileShardsMaxConcurrencyPercent'. - # 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward. - # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage - - # Max number of concurrent shard reconciles within one CHI in progress - reconcileShardsThreadsNumber: 5 - # Max percentage of concurrent shard reconciles within one CHI in progress - reconcileShardsMaxConcurrencyPercent: 50 - - # Reconcile StatefulSet scenario - statefulSet: - # Create StatefulSet scenario - create: - # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds - # Possible options: - # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is, - # do not try to fix or delete or update it, just abort reconcile cycle. - # Do not proceed to the next StatefulSet(s) and wait for an admin to assist. - # 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards. - # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. - onFailure: ignore - - # Update StatefulSet scenario - update: - # How many seconds to wait for created/updated StatefulSet to be 'Ready' - timeout: 300 - # How many seconds to wait between checks/polls for created/updated StatefulSet status - pollInterval: 5 - # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds - # Possible options: - # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is, - # do not try to fix or delete or update it, just abort reconcile cycle. - # Do not proceed to the next StatefulSet(s) and wait for an admin to assist. - # 2. rollback - delete Pod and rollback StatefulSet to previous Generation. - # Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration. - # Follow 'abort' path afterwards. - # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. - onFailure: abort - - # Reconcile Host scenario - host: - # Whether the operator during reconcile procedure should wait for a ClickHouse host: - # - to be excluded from a ClickHouse cluster - # - to complete all running queries - # - to be included into a ClickHouse cluster - # respectfully before moving forward - wait: - exclude: true - queries: true - include: false - - ################################################ - ## - ## Annotations management section - ## - ################################################ - annotation: - # Applied when: - # 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`, - # 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`, - # Include annotations from the following list: - # Applied only when not empty. Empty list means "include all, no selection" - include: [] - # Exclude annotations from the following list: - exclude: [] - - ################################################ - ## - ## Labels management section - ## - ################################################ - label: - # Applied when: - # 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`, - # 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`, - # Include labels from the following list: - # Applied only when not empty. Empty list means "include all, no selection" - include: [] - # Exclude labels from the following list: - # Applied only when not empty. Empty list means "nothing to exclude, no selection" - exclude: [] - # Whether to append *Scope* labels to StatefulSet and Pod. - # Full list of available *scope* labels check in 'labeler.go' - # LabelShardScopeIndex - # LabelReplicaScopeIndex - # LabelCHIScopeIndex - # LabelCHIScopeCycleSize - # LabelCHIScopeCycleIndex - # LabelCHIScopeCycleOffset - # LabelClusterScopeIndex - # LabelClusterScopeCycleSize - # LabelClusterScopeCycleIndex - # LabelClusterScopeCycleOffset - appendScope: "no" - - ################################################ - ## - ## StatefulSet management section - ## - ################################################ - statefulSet: - revisionHistoryLimit: 0 - - ################################################ - ## - ## Pod management section - ## - ################################################ - pod: - # Grace period for Pod termination. - # How many seconds to wait between sending - # SIGTERM and SIGKILL during Pod termination process. - # Increase this number is case of slow shutdown. - terminationGracePeriod: 30 - - ################################################ - ## - ## Log parameters section - ## - ################################################ - logger: - logtostderr: "true" - alsologtostderr: "false" - v: "1" - stderrthreshold: "" - vmodule: "" - log_backtrace_at: "" - ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-confd-files -# NAMESPACE=namespace: {{ .Release.namespace }} -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-confd-files - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -data: ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-configd-files -# NAMESPACE={{ .Release.namespace }} -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-configd-files - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -data: - 01-clickhouse-01-listen.xml: | - - - - - - - - - :: - 0.0.0.0 - 1 - - - 01-clickhouse-02-logger.xml: | - - - - - - - - - - {{.Values.clickhouse.logsLevel}} - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - 1 - - - - 01-clickhouse-03-query_log.xml: | - - - - - - - - - system - query_log
- Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day - 7500 - - - - - 01-clickhouse-04-part_log.xml: | - - - - - - - - - system - part_log
- Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day - 7500 - - - - 01-clickhouse-05-trace_log.xml: | - - - - - - - - - system - trace_log
- Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day - 7500 - - - ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-templatesd-files -# NAMESPACE={{ .Release.namespace }} -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-templatesd-files - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -data: - 001-templates.json.example: | - { - "apiVersion": "clickhouse.altinity.com/v1", - "kind": "ClickHouseInstallationTemplate", - "metadata": { - "name": "01-default-volumeclaimtemplate" - }, - "spec": { - "templates": { - "volumeClaimTemplates": [ - { - "name": "chi-default-volume-claim-template", - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "2Gi" - } - } - } - } - ], - "podTemplates": [ - { - "name": "chi-default-oneperhost-pod-template", - "distribution": "OnePerHost", - "spec": { - "containers" : [ - { - "name": "clickhouse", - "image": "clickhouse/clickhouse-server:23.8", - "ports": [ - { - "name": "http", - "containerPort": 8123 - }, - { - "name": "client", - "containerPort": 9000 - }, - { - "name": "interserver", - "containerPort": 9009 - } - ] - } - ] - } - } - ] - } - } - } - - default-pod-template.yaml.example: | - apiVersion: "clickhouse.altinity.com/v1" - kind: "ClickHouseInstallationTemplate" - metadata: - name: "default-oneperhost-pod-template" - spec: - templates: - podTemplates: - - name: default-oneperhost-pod-template - distribution: "OnePerHost" - default-storage-template.yaml.example: | - apiVersion: "clickhouse.altinity.com/v1" - kind: "ClickHouseInstallationTemplate" - metadata: - name: "default-storage-template-2Gi" - spec: - templates: - volumeClaimTemplates: - - name: default-storage-template-2Gi - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi - - readme: | - Templates in this folder are packaged with an operator and available via 'useTemplate' ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-usersd-files -# NAMESPACE={{ .Release.namespace }} -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-usersd-files - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -data: - 01-clickhouse-operator-profile.xml: | - - - - - - - - - - - - 0 - 1 - 10 - 0 - 0 - - - - - 02-clickhouse-default-profile.xml: | - - - - - - - - - - 2 - 1 - 1000 - 1 - 1 - 1 - nearest_hostname - 0 - - - - ---- -# -# Template parameters available: -# NAMESPACE={{ .Release.namespace }} -# COMMENT= -# OPERATOR_VERSION=0.23.6 -# CH_USERNAME_SECRET_PLAIN=clickhouse_operator -# CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password -# -apiVersion: v1 -kind: Secret -metadata: - name: clickhouse-operator - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -type: Opaque -stringData: - username: clickhouse_operator - password: clickhouse_operator_password ---- -# Template Parameters: -# -# NAMESPACE={{ .Release.namespace }} -# COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.6 -# OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.6 -# METRICS_EXPORTER_IMAGE_PULL_POLICY=Always -# -# Setup Deployment for clickhouse-operator -# Deployment would be created in kubectl-specified namespace -kind: Deployment -apiVersion: apps/v1 -metadata: - name: clickhouse-operator - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -spec: - replicas: 1 - selector: - matchLabels: - app: clickhouse-operator - template: - metadata: - labels: - app: clickhouse-operator - annotations: - prometheus.io/port: '8888' - prometheus.io/scrape: 'true' - clickhouse-operator-metrics/port: '9999' - clickhouse-operator-metrics/scrape: 'true' - spec: - serviceAccountName: clickhouse-operator - volumes: - - name: etc-clickhouse-operator-folder - configMap: - name: etc-clickhouse-operator-files - - name: etc-clickhouse-operator-confd-folder - configMap: - name: etc-clickhouse-operator-confd-files - - name: etc-clickhouse-operator-configd-folder - configMap: - name: etc-clickhouse-operator-configd-files - - name: etc-clickhouse-operator-templatesd-folder - configMap: - name: etc-clickhouse-operator-templatesd-files - - name: etc-clickhouse-operator-usersd-folder - configMap: - name: etc-clickhouse-operator-usersd-files - containers: - - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.6 - imagePullPolicy: Always - volumeMounts: - - name: etc-clickhouse-operator-folder - mountPath: /etc/clickhouse-operator - - name: etc-clickhouse-operator-confd-folder - mountPath: /etc/clickhouse-operator/conf.d - - name: etc-clickhouse-operator-configd-folder - mountPath: /etc/clickhouse-operator/config.d - - name: etc-clickhouse-operator-templatesd-folder - mountPath: /etc/clickhouse-operator/templates.d - - name: etc-clickhouse-operator-usersd-folder - mountPath: /etc/clickhouse-operator/users.d - env: - # Pod-specific - # spec.nodeName: ip-172-20-52-62.ec2.internal - - name: OPERATOR_POD_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # metadata.name: clickhouse-operator-6f87589dbb-ftcsf - - name: OPERATOR_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # metadata.namespace: {{ .Release.namespace }} - - name: OPERATOR_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # status.podIP: 100.96.3.2 - - name: OPERATOR_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - # spec.serviceAccount: clickhouse-operator - # spec.serviceAccountName: clickhouse-operator - - name: OPERATOR_POD_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - # Container-specific - - name: OPERATOR_CONTAINER_CPU_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.cpu - - name: OPERATOR_CONTAINER_CPU_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.cpu - - name: OPERATOR_CONTAINER_MEM_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.memory - - name: OPERATOR_CONTAINER_MEM_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.memory - ports: - - containerPort: 9999 - name: metrics - {{- if .Values.clickhouse.operator.operator }} - {{- with .Values.clickhouse.operator.operator.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - - - name: metrics-exporter - image: altinity/metrics-exporter:0.23.6 - imagePullPolicy: Always - volumeMounts: - - name: etc-clickhouse-operator-folder - mountPath: /etc/clickhouse-operator - - name: etc-clickhouse-operator-confd-folder - mountPath: /etc/clickhouse-operator/conf.d - - name: etc-clickhouse-operator-configd-folder - mountPath: /etc/clickhouse-operator/config.d - - name: etc-clickhouse-operator-templatesd-folder - mountPath: /etc/clickhouse-operator/templates.d - - name: etc-clickhouse-operator-usersd-folder - mountPath: /etc/clickhouse-operator/users.d - env: - # Pod-specific - # spec.nodeName: ip-172-20-52-62.ec2.internal - - name: OPERATOR_POD_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # metadata.name: clickhouse-operator-6f87589dbb-ftcsf - - name: OPERATOR_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # metadata.namespace: {{ .Release.namespace }} - - name: OPERATOR_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # status.podIP: 100.96.3.2 - - name: OPERATOR_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - # spec.serviceAccount: clickhouse-operator - # spec.serviceAccountName: clickhouse-operator - - name: OPERATOR_POD_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - # Container-specific - - name: OPERATOR_CONTAINER_CPU_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.cpu - - name: OPERATOR_CONTAINER_CPU_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.cpu - - name: OPERATOR_CONTAINER_MEM_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.memory - - name: OPERATOR_CONTAINER_MEM_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.memory - ports: - - containerPort: 8888 - name: metrics - {{- if .Values.clickhouse.operator.metrics }} - {{- with .Values.clickhouse.operator.metrics.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - - {{- with .Values.clickhouse.operator.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.clickhouse.operator.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - - {{- with .Values.clickhouse.operator.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -# Template Parameters: -# -# NAMESPACE={{ .Release.namespace }} -# COMMENT= -# -# Setup ClusterIP Service to provide monitoring metrics for Prometheus -# Service would be created in kubectl-specified namespace -# In order to get access outside of k8s it should be exposed as: -# kubectl --namespace prometheus port-forward service/prometheus 9090 -# and point browser to localhost:9090 -{{- if .Values.clickhouse.monitoring.enables }} -kind: Service -apiVersion: v1 -metadata: - name: clickhouse-operator-metrics - namespace: {{ .Release.namespace }} - labels: - clickhouse.altinity.com/chop: 0.23.6 - app: clickhouse-operator -spec: - ports: - - port: 8888 - name: clickhouse-metrics - - port: 9999 - name: operator-metrics - selector: - app: clickhouse-operator -{{- end }} ---- -## ServiceMonitor -{{- if .Values.clickhouse.monitoring.enables }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: clickhouse - namespace: {{ .Release.namespace }} - labels: - release: {{ .Values.clickhouse.monitoring.monitoringReleaseName }} -spec: - endpoints: - - interval: 30s - port: clickhouse-metrics - - interval: 30s - port: operator-metrics - namespaceSelector: - matchNames: - - {{ .Release.namespace }} - selector: - matchLabels: - app: clickhouse-operator - argocd.argoproj.io/instance: {{ .Release.namespace }} - clickhouse.altinity.com/chop: 0.23.6 -{{- end }} \ No newline at end of file diff --git a/deployment/helm_chart/opik/templates/clickhouseinstallation.yaml b/deployment/helm_chart/opik/templates/clickhouseinstallation.yaml index 57e7116a9..b8ce79883 100644 --- a/deployment/helm_chart/opik/templates/clickhouseinstallation.yaml +++ b/deployment/helm_chart/opik/templates/clickhouseinstallation.yaml @@ -23,8 +23,8 @@ spec: {{ .Values.clickhouse.adminUser.username }}/networks/ip: - '127.0.0.1/32' - '0.0.0.0/0' - {{- if .Values.clickhouse.monitoring.enables }} - {{- if .Values.clickhouse.monitoring.useSecret.enables }} + {{- if .Values.clickhouse.monitoring.enabled }} + {{- if .Values.clickhouse.monitoring.useSecret.enabled }} {{ .Values.clickhouse.monitoring.username }}/k8s_secret_password: {{ include "opik.name" .}}-{{ .Values.clickhouse.monitoring.secretname }}/{{ .Values.clickhouse.monitoring.password_key }} {{- else }} {{ .Values.clickhouse.monitoring.username }}/password: {{ .Values.clickhouse.monitoring.password }} diff --git a/deployment/helm_chart/opik/values.yaml b/deployment/helm_chart/opik/values.yaml index 701ba6757..c9dec4f0a 100644 --- a/deployment/helm_chart/opik/values.yaml +++ b/deployment/helm_chart/opik/values.yaml @@ -193,8 +193,13 @@ redis: \n--loadmodule ${BASEDIR}/lib/rejson.so ${REDISJSON_ARGS} \\\n--loadmodule\ \ ${BASEDIR}/lib/redisbloom.so ${REDISBLOOM_ARGS}\n" +altinity-clickhouse-operator: + metrics: + enabled: false + serviceMonitor: + enabled: false + clickhouse: - namePrefix: opik shardsCount: 1 replicasCount: 1 image: altinity/clickhouse-server:24.3.5.47.altinitystable @@ -214,14 +219,10 @@ clickhouse: # secretName: "" # schedule: "" # baseBackup: "" - operator: - enabled: true monitoring: enabled: false - monitoringReleaseName: # if monitoring is enables, this must be set username: opikmon # readonly user password: opikmon - # resources: # requests: # cpu: 300m