From aafd57963b9ba7e667f2471046b36ced48f4db97 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 7 Nov 2023 16:19:02 +0100 Subject: [PATCH] New hbase version 2.4.17 (#403) * bump hbase to 2.4.17 and zk to 3.8.2 * adapt to supported versions * adapted changelog * let jmx_exporter use the config file specified for each role * bump vector aggregator * fix zookeeper versions * bump vector to 0.33.0 * bump operator-rs to 0.55.0 * adapted changelog * test snappy compression * add more compression algorithms * enable delete table --- CHANGELOG.md | 7 ++ .../getting_started/getting_started.sh | 2 +- .../getting_started/getting_started.sh.j2 | 2 +- .../hbase/examples/getting_started/hbase.yaml | 2 +- .../examples/getting_started/hbase.yaml.j2 | 2 +- .../hbase/examples/getting_started/zk.yaml | 2 +- .../hbase/examples/getting_started/zk.yaml.j2 | 2 +- .../pages/getting_started/first_steps.adoc | 2 +- .../hbase/partials/supported-versions.adoc | 7 +- rust/crd/src/affinity.rs | 4 +- rust/crd/src/lib.rs | 2 +- .../01-install-hbase-vector-aggregator.yaml | 2 +- tests/templates/kuttl/smoke/test-hbase.py | 107 +++++++++--------- tests/test-definition.yaml | 18 ++- 14 files changed, 83 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65a979d6..9ceeaf9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,21 +8,28 @@ - Configuration overrides for the JVM security properties, such as DNS caching ([#389]). - Support PodDisruptionBudgets ([#399]). - Support graceful shutdown ([#402]). +- Added support for version 2.4.17 ([#403]). ### Changed - `vector` `0.26.0` -> `0.31.0` ([#382]). +- Use jmx_exporter soft link instead of hardcoded version ([#403]). ### Fixed - Fix Zookeeper hbase.rootdir when users point to discovery ConfigMap of ZookeeperCluster rather than ZNode. Print a warning in that case ([#394]). +### Removed + +- Removed support for 2.4.6, 2.4.8, 2.4.9, 2.4.11 ([#403]). + [#382]: https://github.com/stackabletech/hbase-operator/pull/382 [#385]: https://github.com/stackabletech/hbase-operator/pull/385 [#389]: https://github.com/stackabletech/hbase-operator/pull/389 [#394]: https://github.com/stackabletech/hbase-operator/pull/394 [#399]: https://github.com/stackabletech/hbase-operator/pull/399 [#402]: https://github.com/stackabletech/hbase-operator/pull/402 +[#403]: https://github.com/stackabletech/hbase-operator/pull/403 ## [23.7.0] - 2023-07-14 diff --git a/docs/modules/hbase/examples/getting_started/getting_started.sh b/docs/modules/hbase/examples/getting_started/getting_started.sh index 85fb3e37..d9bc04bb 100755 --- a/docs/modules/hbase/examples/getting_started/getting_started.sh +++ b/docs/modules/hbase/examples/getting_started/getting_started.sh @@ -129,7 +129,7 @@ version() { echo "Check cluster version..." cluster_version=$(version | jq -r '.Version') -if [ "$cluster_version" == "2.4.12" ]; then +if [ "$cluster_version" == "2.4.17" ]; then echo "Cluster version: $cluster_version" else echo "Unexpected version: $cluster_version" diff --git a/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 b/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 index d5d40782..670c9143 100755 --- a/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 @@ -129,7 +129,7 @@ version() { echo "Check cluster version..." cluster_version=$(version | jq -r '.Version') -if [ "$cluster_version" == "2.4.12" ]; then +if [ "$cluster_version" == "2.4.17" ]; then echo "Cluster version: $cluster_version" else echo "Unexpected version: $cluster_version" diff --git a/docs/modules/hbase/examples/getting_started/hbase.yaml b/docs/modules/hbase/examples/getting_started/hbase.yaml index 66a6fe1a..f98776a9 100644 --- a/docs/modules/hbase/examples/getting_started/hbase.yaml +++ b/docs/modules/hbase/examples/getting_started/hbase.yaml @@ -5,7 +5,7 @@ metadata: name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-hbase-znode diff --git a/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 b/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 index 6268f498..472f3c83 100644 --- a/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 +++ b/docs/modules/hbase/examples/getting_started/hbase.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode diff --git a/docs/modules/hbase/examples/getting_started/zk.yaml b/docs/modules/hbase/examples/getting_started/zk.yaml index 3684c21a..76e9c28e 100644 --- a/docs/modules/hbase/examples/getting_started/zk.yaml +++ b/docs/modules/hbase/examples/getting_started/zk.yaml @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.8.0 + productVersion: 3.8.3 servers: roleGroups: default: diff --git a/docs/modules/hbase/examples/getting_started/zk.yaml.j2 b/docs/modules/hbase/examples/getting_started/zk.yaml.j2 index 3684c21a..76e9c28e 100644 --- a/docs/modules/hbase/examples/getting_started/zk.yaml.j2 +++ b/docs/modules/hbase/examples/getting_started/zk.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.8.0 + productVersion: 3.8.3 servers: roleGroups: default: diff --git a/docs/modules/hbase/pages/getting_started/first_steps.adoc b/docs/modules/hbase/pages/getting_started/first_steps.adoc index fdbdc504..57e86bcd 100644 --- a/docs/modules/hbase/pages/getting_started/first_steps.adoc +++ b/docs/modules/hbase/pages/getting_started/first_steps.adoc @@ -76,7 +76,7 @@ include::example$getting_started/getting_started.sh[tag=cluster-version] This will return the version that was specified in the HBase cluster definition: [source,json] -{"Version":"2.4.12"} +{"Version":"2.4.17"} The cluster status can be checked and formatted like this: diff --git a/docs/modules/hbase/partials/supported-versions.adoc b/docs/modules/hbase/partials/supported-versions.adoc index 91756f34..4e279f56 100644 --- a/docs/modules/hbase/partials/supported-versions.adoc +++ b/docs/modules/hbase/partials/supported-versions.adoc @@ -2,8 +2,5 @@ // This is a separate file, since it is used by both the direct HBase-Operator documentation, and the overarching // Stackable Platform documentation. -- 2.4.6 -- 2.4.8 -- 2.4.9 -- 2.4.11 -- 2.4.12 +- 2.4.17 +- 2.4.12 (deprecated) diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 6f6f20cd..95fbaff8 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -106,7 +106,7 @@ mod tests { name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode @@ -229,7 +229,7 @@ mod tests { name: simple-hbase spec: image: - productVersion: 2.4.12 + productVersion: 2.4.17 clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 43c387c1..683d0873 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -357,7 +357,7 @@ impl Configuration for HbaseConfigFragment { match file { HBASE_ENV_SH => { result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string())); - let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={METRICS_PORT}:/stackable/jmx/region-server.yaml"); + let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/{role_name}.yaml"); if let Some(hbase_opts) = &self.hbase_opts { all_hbase_opts += " "; all_hbase_opts += hbase_opts; diff --git a/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml index fd9f2cd7..6124a411 100644 --- a/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml @@ -5,7 +5,7 @@ commands: - script: >- helm install hbase-vector-aggregator vector --namespace $NAMESPACE - --version 0.23.0 + --version 0.26.0 --repo https://helm.vector.dev --values hbase-vector-aggregator-values.yaml --- diff --git a/tests/templates/kuttl/smoke/test-hbase.py b/tests/templates/kuttl/smoke/test-hbase.py index a9a80e66..03217eda 100644 --- a/tests/templates/kuttl/smoke/test-hbase.py +++ b/tests/templates/kuttl/smoke/test-hbase.py @@ -22,11 +22,11 @@ def encode_value(plain_string): def decode_value(base64_string): return base64.b64decode(base64_string).decode('utf-8') - def create_table(self, rest_url, name, column_family): + def create_table(self, rest_url, name, column_family, compression): response = self.session.put( f'{rest_url}/{name}/schema', data=f''' - + ''' @@ -78,52 +78,57 @@ def delete_scanner(self, scanner_location): hbase = HbaseClient() -print(''' -Create a table -==============''') -column_family = 'cf' -(table_location, table_schema_location) = hbase.create_table( - rest_url=hbase_rest_url, - name='companies', - column_family=column_family -) - -print(''' -Write a row to the table -========================''') -cell_value = 'Stackable GmbH' -hbase.put_row( - table_location=table_location, - row_key='stackable', - column_family=column_family, - column='name', - cell_value=cell_value -) - -print(''' -Get a scanner object -====================''') -scanner_location = hbase.put_scanner(table_location) - -print(''' -Get the next batch from the scanner -===================================''') -scan = hbase.get_scanner(scanner_location) - -print(''' -Verify table content -====================''') -parser = ET.fromstring(scan) -actual_cell_value = hbase.decode_value(parser.findtext('./Row/Cell')) -print(f'assert "{actual_cell_value}" == "{cell_value}"') -assert actual_cell_value == cell_value - -print(''' -Delete the scanner -==================''') -hbase.delete_scanner(scanner_location) - -print(''' -Delete the table -================''') -hbase.delete_table(table_schema_location) +# Valid compression types are snappy, lzo, gz, bzip2, lz4 or zstd +compression_opts = ['none', 'snappy', 'lzo', 'gz', 'bzip2', 'lz4', 'zstd'] + +for compression in compression_opts: + print(f''' + Create a table with compression={compression} + ==============''') + column_family = 'cf' + (table_location, table_schema_location) = hbase.create_table( + rest_url=hbase_rest_url, + name='companies', + column_family=column_family, + compression=compression + ) + + print(''' + Write a row to the table + ========================''') + cell_value = 'Stackable GmbH' + hbase.put_row( + table_location=table_location, + row_key='stackable', + column_family=column_family, + column='name', + cell_value=cell_value + ) + + print(''' + Get a scanner object + ====================''') + scanner_location = hbase.put_scanner(table_location) + + print(''' + Get the next batch from the scanner + ===================================''') + scan = hbase.get_scanner(scanner_location) + + print(''' + Verify table content + ====================''') + parser = ET.fromstring(scan) + actual_cell_value = hbase.decode_value(parser.findtext('./Row/Cell')) + print(f'assert "{actual_cell_value}" == "{cell_value}"') + assert actual_cell_value == cell_value + + print(''' + Delete the scanner + ==================''') + hbase.delete_scanner(scanner_location) + + print(''' + Delete the table + ================''') + hbase.delete_table(table_schema_location) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5ed87de2..28ac0d90 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,28 +2,24 @@ dimensions: - name: hbase values: - - 2.4.8 - - 2.4.9 - - 2.4.11 - 2.4.12 + - 2.4.17 - name: hbase-latest values: - - 2.4.12 + - 2.4.17 - name: hdfs values: - - 3.2.2 - - 3.3.3 + - 3.2.4 + - 3.3.6 - name: hdfs-latest values: - - 3.3.4 + - 3.3.6 - name: zookeeper values: - - 3.6.3 - - 3.7.0 - - 3.8.0 + - 3.8.3 - name: zookeeper-latest values: - - 3.8.0 + - 3.8.3 # Used for zookeeper, hdfs and hbase - name: listener-class values: