From 387b5178c9cb2c7c9bb1be70adb47741f4570070 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 Oct 2023 14:51:44 +0200 Subject: [PATCH 01/12] bump hbase to 2.4.17 and zk to 3.8.2 --- .../modules/hbase/examples/getting_started/getting_started.sh | 2 +- .../hbase/examples/getting_started/getting_started.sh.j2 | 2 +- docs/modules/hbase/examples/getting_started/hbase.yaml | 2 +- docs/modules/hbase/examples/getting_started/hbase.yaml.j2 | 2 +- docs/modules/hbase/examples/getting_started/zk.yaml | 2 +- docs/modules/hbase/examples/getting_started/zk.yaml.j2 | 2 +- docs/modules/hbase/pages/getting_started/first_steps.adoc | 2 +- rust/crd/src/affinity.rs | 4 ++-- rust/crd/src/lib.rs | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/modules/hbase/examples/getting_started/getting_started.sh b/docs/modules/hbase/examples/getting_started/getting_started.sh index 85fb3e37..d9bc04bb 100755 --- a/docs/modules/hbase/examples/getting_started/getting_started.sh +++ b/docs/modules/hbase/examples/getting_started/getting_started.sh @@ -129,7 +129,7 @@ version() { echo "Check cluster version..." cluster_version=$(version | jq -r '.Version') -if [ "$cluster_version" == "2.4.12" ]; then +if [ "$cluster_version" == "2.4.17" ]; then echo "Cluster version: $cluster_version" else echo "Unexpected version: $cluster_version" diff --git a/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 b/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 index d5d40782..670c9143 100755 --- a/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 @@ -129,7 +129,7 @@ version() { echo "Check cluster version..." cluster_version=$(version | jq -r '.Version') -if [ "$cluster_version" == "2.4.12" ]; then +if [ "$cluster_version" == "2.4.17" ]; then echo "Cluster version: $cluster_version" else echo "Unexpected version: $cluster_version" diff --git a/docs/modules/hbase/examples/getting_started/hbase.yaml b/docs/modules/hbase/examples/getting_started/hbase.yaml index 66a6fe1a..f98776a9 100644 --- a/docs/modules/hbase/examples/getting_started/hbase.yaml +++ b/docs/modules/hbase/examples/getting_started/hbase.yaml @@ -5,7 +5,7 @@ metadata: name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-hbase-znode diff --git a/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 b/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 index 6268f498..472f3c83 100644 --- a/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 +++ b/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode diff --git a/docs/modules/hbase/examples/getting_started/zk.yaml b/docs/modules/hbase/examples/getting_started/zk.yaml index 3684c21a..65ad270c 100644 --- a/docs/modules/hbase/examples/getting_started/zk.yaml +++ b/docs/modules/hbase/examples/getting_started/zk.yaml @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.8.0 + productVersion: 3.8.2 servers: roleGroups: default: diff --git a/docs/modules/hbase/examples/getting_started/zk.yaml.j2 b/docs/modules/hbase/examples/getting_started/zk.yaml.j2 index 3684c21a..65ad270c 100644 --- a/docs/modules/hbase/examples/getting_started/zk.yaml.j2 +++ b/docs/modules/hbase/examples/getting_started/zk.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.8.0 + productVersion: 3.8.2 servers: roleGroups: default: diff --git a/docs/modules/hbase/pages/getting_started/first_steps.adoc b/docs/modules/hbase/pages/getting_started/first_steps.adoc index fdbdc504..57e86bcd 100644 --- a/docs/modules/hbase/pages/getting_started/first_steps.adoc +++ b/docs/modules/hbase/pages/getting_started/first_steps.adoc @@ -76,7 +76,7 @@ include::example$getting_started/getting_started.sh[tag=cluster-version] This will return the version that was specified in the HBase cluster definition: [source,json] -{"Version":"2.4.12"} +{"Version":"2.4.17"} The cluster status can be checked and formatted like this: diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 6f6f20cd..95fbaff8 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -106,7 +106,7 @@ mod tests { name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode @@ -229,7 +229,7 @@ mod tests { name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 0680946a..612b990a 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -338,7 +338,7 @@ impl Configuration for HbaseConfigFragment { match file { HBASE_ENV_SH => { result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string())); - let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={METRICS_PORT}:/stackable/jmx/region-server.yaml"); + let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/region-server.yaml"); if let Some(hbase_opts) = &self.hbase_opts { all_hbase_opts += " "; all_hbase_opts += hbase_opts; From b493aaffcefba28cf43da8eeccc8250f222f4fa5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 Oct 2023 14:51:54 +0200 Subject: [PATCH 02/12] adapt to supported versions --- .../hbase/partials/supported-versions.adoc | 7 ++----- tests/test-definition.yaml | 18 +++++++----------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/modules/hbase/partials/supported-versions.adoc b/docs/modules/hbase/partials/supported-versions.adoc index 91756f34..4e279f56 100644 --- a/docs/modules/hbase/partials/supported-versions.adoc +++ b/docs/modules/hbase/partials/supported-versions.adoc @@ -2,8 +2,5 @@ // This is a separate file, since it is used by both the direct HBase-Operator documentation, and the overarching // Stackable Platform documentation. -- 2.4.6 -- 2.4.8 -- 2.4.9 -- 2.4.11 -- 2.4.12 +- 2.4.17 +- 2.4.12 (deprecated) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5ed87de2..6ef37283 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,28 +2,24 @@ dimensions: - name: hbase values: - - 2.4.8 - - 2.4.9 - - 2.4.11 - 2.4.12 + - 2.4.17 - name: hbase-latest values: - - 2.4.12 + - 2.4.17 - name: hdfs values: - - 3.2.2 - - 3.3.3 + - 3.2.4 + - 3.3.6 - name: hdfs-latest values: - - 3.3.4 + - 3.3.6 - name: zookeeper values: - - 3.6.3 - - 3.7.0 - - 3.8.0 + - 3.8.2 - name: zookeeper-latest values: - - 3.8.0 + - 3.8.2 # Used for zookeeper, hdfs and hbase - name: listener-class values: From b7697a382af93ad1ca9b07b12a03b266b3fb916c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 Oct 2023 15:01:18 +0200 Subject: [PATCH 03/12] adapted changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b362cf1b..39dad306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,21 +7,28 @@ - Default stackableVersion to operator version ([#385]). - Configuration overrides for the JVM security properties, such as DNS caching ([#389]). - Support PodDisruptionBudgets ([#399]). +- Added support for version 2.4.17 ([#403]). ### Changed - `vector` `0.26.0` -> `0.31.0` ([#382]). - `operator-rs` `0.44.0` -> `0.52.1` ([#385], [#399]). +- Use jmx_exporter soft link instead of hardcoded version ([#403]). ### Fixed - Fix Zookeeper hbase.rootdir when users point to discovery ConfigMap of ZookeeperCluster rather than ZNode. Print a warning in that case ([#394]). +### Removed + +- Removed support for 2.4.6, 2.4.8, 2.4.9, 2.4.11 ([#403]). + [#382]: https://github.com/stackabletech/hbase-operator/pull/382 [#385]: https://github.com/stackabletech/hbase-operator/pull/385 [#389]: https://github.com/stackabletech/hbase-operator/pull/389 [#394]: https://github.com/stackabletech/hbase-operator/pull/394 [#399]: https://github.com/stackabletech/hbase-operator/pull/399 +[#403]: https://github.com/stackabletech/hbase-operator/pull/403 ## [23.7.0] - 2023-07-14 From 10fcb0f3b083364b7f35a88668d759a25972eba1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 13 Oct 2023 18:19:52 +0200 Subject: [PATCH 04/12] let jmx_exporter use the config file specified for each role --- rust/crd/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 612b990a..3cd61d29 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -338,7 +338,7 @@ impl Configuration for HbaseConfigFragment { match file { HBASE_ENV_SH => { result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string())); - let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/region-server.yaml"); + let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/{role_name}.yaml"); if let Some(hbase_opts) = &self.hbase_opts { all_hbase_opts += " "; all_hbase_opts += hbase_opts; From 3fadb250f848694ec2a7c510005d9f7dd3927712 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 13 Oct 2023 18:20:11 +0200 Subject: [PATCH 05/12] bump vector aggregator --- .../kuttl/logging/01-install-hbase-vector-aggregator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml index fd9f2cd7..6124a411 100644 --- a/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml @@ -5,7 +5,7 @@ commands: - script: >- helm install hbase-vector-aggregator vector --namespace $NAMESPACE - --version 0.23.0 + --version 0.26.0 --repo https://helm.vector.dev --values hbase-vector-aggregator-values.yaml --- From 6ebefc41175ebb1d3e0986af0e64fb138f3bc9d2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 13 Oct 2023 18:28:53 +0200 Subject: [PATCH 06/12] fix zookeeper versions --- docs/modules/hbase/examples/getting_started/zk.yaml | 2 +- docs/modules/hbase/examples/getting_started/zk.yaml.j2 | 2 +- tests/test-definition.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/modules/hbase/examples/getting_started/zk.yaml b/docs/modules/hbase/examples/getting_started/zk.yaml index 65ad270c..76e9c28e 100644 --- a/docs/modules/hbase/examples/getting_started/zk.yaml +++ b/docs/modules/hbase/examples/getting_started/zk.yaml @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.8.2 + productVersion: 3.8.3 servers: roleGroups: default: diff --git a/docs/modules/hbase/examples/getting_started/zk.yaml.j2 b/docs/modules/hbase/examples/getting_started/zk.yaml.j2 index 65ad270c..76e9c28e 100644 --- a/docs/modules/hbase/examples/getting_started/zk.yaml.j2 +++ b/docs/modules/hbase/examples/getting_started/zk.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.8.2 + productVersion: 3.8.3 servers: roleGroups: default: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 6ef37283..28ac0d90 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -16,10 +16,10 @@ dimensions: - 3.3.6 - name: zookeeper values: - - 3.8.2 + - 3.8.3 - name: zookeeper-latest values: - - 3.8.2 + - 3.8.3 # Used for zookeeper, hdfs and hbase - name: listener-class values: From 6696140aed057b7a183b21d80a6f7c9360c22e1a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 17 Oct 2023 14:43:49 +0200 Subject: [PATCH 07/12] bump vector to 0.33.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39dad306..c5d625a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ ### Changed -- `vector` `0.26.0` -> `0.31.0` ([#382]). +- `vector` `0.26.0` -> `0.33.0` ([#382]). - `operator-rs` `0.44.0` -> `0.52.1` ([#385], [#399]). - Use jmx_exporter soft link instead of hardcoded version ([#403]). From ddb289ad87cb209730a606c4c6c0aa96b51d2f81 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 17 Oct 2023 14:45:29 +0200 Subject: [PATCH 08/12] bump operator-rs to 0.55.0 --- Cargo.lock | 8 +- Cargo.toml | 2 +- deploy/helm/hbase-operator/crds/crds.yaml | 96 +++++++++++++++----- rust/operator-binary/src/hbase_controller.rs | 2 +- 4 files changed, 78 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d29e5c69..9591dc4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1864,8 +1864,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.52.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.52.1#18af9be0473cd6c30d7426e9ade74c90e4abce22" +version = "0.55.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" dependencies = [ "chrono", "clap", @@ -1898,8 +1898,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.52.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.52.1#18af9be0473cd6c30d7426e9ade74c90e4abce22" +version = "0.55.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" dependencies = [ "darling 0.20.3", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 4c48e016..ce29f4ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.52.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/deploy/helm/hbase-operator/crds/crds.yaml b/deploy/helm/hbase-operator/crds/crds.yaml index 032082a3..ccf43189 100644 --- a/deploy/helm/hbase-operator/crds/crds.yaml +++ b/deploy/helm/hbase-operator/crds/crds.yaml @@ -606,6 +606,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -613,13 +614,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -635,14 +637,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -656,9 +660,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -671,10 +676,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -4042,6 +4050,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -4049,13 +4058,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4071,14 +4081,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4092,9 +4104,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4107,10 +4120,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -7495,6 +7511,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -7502,13 +7519,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7524,14 +7542,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7545,9 +7565,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7560,10 +7581,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -10931,6 +10955,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -10938,13 +10963,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -10960,14 +10986,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -10981,9 +11009,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -10996,10 +11025,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -14384,6 +14416,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -14391,13 +14424,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -14413,14 +14447,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -14434,9 +14470,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -14449,10 +14486,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -17820,6 +17860,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -17827,13 +17868,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -17849,14 +17891,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -17870,9 +17914,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -17885,10 +17930,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 3c55bf1c..dd96879f 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -27,7 +27,6 @@ use stackable_operator::{ product_image_selection::ResolvedProductImage, rbac::{build_rbac_resources, service_account_name}, }, - duration::Duration, k8s_openapi::{api::core::v1::Volume, DeepMerge}, k8s_openapi::{ api::{ @@ -61,6 +60,7 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + time::Duration, }; use std::{ collections::{BTreeMap, HashMap}, From f472a8a0f297d4ce67f383392abea88c70a448d4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 18 Oct 2023 09:42:03 +0200 Subject: [PATCH 09/12] adapted changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5d625a3..7081174e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ ### Changed - `vector` `0.26.0` -> `0.33.0` ([#382]). -- `operator-rs` `0.44.0` -> `0.52.1` ([#385], [#399]). +- `operator-rs` `0.44.0` -> `0.55.0` ([#385], [#399], [#403]). - Use jmx_exporter soft link instead of hardcoded version ([#403]). ### Fixed From 1b52b46e5d17c361bf6d7c2cba336e51c9d2cfb3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 2 Nov 2023 17:27:35 +0100 Subject: [PATCH 10/12] test snappy compression --- tests/templates/kuttl/smoke/test-hbase.py | 106 +++++++++++----------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/tests/templates/kuttl/smoke/test-hbase.py b/tests/templates/kuttl/smoke/test-hbase.py index a9a80e66..af520d1b 100644 --- a/tests/templates/kuttl/smoke/test-hbase.py +++ b/tests/templates/kuttl/smoke/test-hbase.py @@ -22,11 +22,11 @@ def encode_value(plain_string): def decode_value(base64_string): return base64.b64decode(base64_string).decode('utf-8') - def create_table(self, rest_url, name, column_family): + def create_table(self, rest_url, name, column_family, compression): response = self.session.put( f'{rest_url}/{name}/schema', data=f''' - + ''' @@ -78,52 +78,56 @@ def delete_scanner(self, scanner_location): hbase = HbaseClient() -print(''' -Create a table -==============''') -column_family = 'cf' -(table_location, table_schema_location) = hbase.create_table( - rest_url=hbase_rest_url, - name='companies', - column_family=column_family -) - -print(''' -Write a row to the table -========================''') -cell_value = 'Stackable GmbH' -hbase.put_row( - table_location=table_location, - row_key='stackable', - column_family=column_family, - column='name', - cell_value=cell_value -) - -print(''' -Get a scanner object -====================''') -scanner_location = hbase.put_scanner(table_location) - -print(''' -Get the next batch from the scanner -===================================''') -scan = hbase.get_scanner(scanner_location) - -print(''' -Verify table content -====================''') -parser = ET.fromstring(scan) -actual_cell_value = hbase.decode_value(parser.findtext('./Row/Cell')) -print(f'assert "{actual_cell_value}" == "{cell_value}"') -assert actual_cell_value == cell_value - -print(''' -Delete the scanner -==================''') -hbase.delete_scanner(scanner_location) - -print(''' -Delete the table -================''') -hbase.delete_table(table_schema_location) +compression_opts = ['none', 'snappy'] + +for compression in compression_opts: + print(f''' + Create a table with compression={compression} + ==============''') + column_family = 'cf' + (table_location, table_schema_location) = hbase.create_table( + rest_url=hbase_rest_url, + name='companies', + column_family=column_family, + compression=compression + ) + + print(''' + Write a row to the table + ========================''') + cell_value = 'Stackable GmbH' + hbase.put_row( + table_location=table_location, + row_key='stackable', + column_family=column_family, + column='name', + cell_value=cell_value + ) + + print(''' + Get a scanner object + ====================''') + scanner_location = hbase.put_scanner(table_location) + + print(''' + Get the next batch from the scanner + ===================================''') + scan = hbase.get_scanner(scanner_location) + + print(''' + Verify table content + ====================''') + parser = ET.fromstring(scan) + actual_cell_value = hbase.decode_value(parser.findtext('./Row/Cell')) + print(f'assert "{actual_cell_value}" == "{cell_value}"') + assert actual_cell_value == cell_value + + print(''' + Delete the scanner + ==================''') + hbase.delete_scanner(scanner_location) + + print(''' + Delete the table + ================''') + hbase.delete_table(table_schema_location) From 079f48f74a60818c7620a72d7adc5d87f39bdd56 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 3 Nov 2023 10:41:03 +0100 Subject: [PATCH 11/12] add more compression algorithms --- tests/templates/kuttl/smoke/test-hbase.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/templates/kuttl/smoke/test-hbase.py b/tests/templates/kuttl/smoke/test-hbase.py index af520d1b..231355da 100644 --- a/tests/templates/kuttl/smoke/test-hbase.py +++ b/tests/templates/kuttl/smoke/test-hbase.py @@ -78,7 +78,8 @@ def delete_scanner(self, scanner_location): hbase = HbaseClient() -compression_opts = ['none', 'snappy'] +# Valid compression types are snappy, lzo, gz, bzip2, lz4 or zstd +compression_opts = ['none', 'snappy', 'lzo', 'gz', 'bzip2', 'lz4', 'zstd'] for compression in compression_opts: print(f''' @@ -127,7 +128,7 @@ def delete_scanner(self, scanner_location): ==================''') hbase.delete_scanner(scanner_location) - print(''' - Delete the table - ================''') - hbase.delete_table(table_schema_location) + #print(''' + #Delete the table + #================''') + #hbase.delete_table(table_schema_location) From f76d9cacbd4558754fff5308a4cdda58d42c75d6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 3 Nov 2023 12:31:31 +0100 Subject: [PATCH 12/12] enable delete table --- tests/templates/kuttl/smoke/test-hbase.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/smoke/test-hbase.py b/tests/templates/kuttl/smoke/test-hbase.py index 231355da..03217eda 100644 --- a/tests/templates/kuttl/smoke/test-hbase.py +++ b/tests/templates/kuttl/smoke/test-hbase.py @@ -128,7 +128,7 @@ def delete_scanner(self, scanner_location): ==================''') hbase.delete_scanner(scanner_location) - #print(''' - #Delete the table - #================''') - #hbase.delete_table(table_schema_location) + print(''' + Delete the table + ================''') + hbase.delete_table(table_schema_location)