feat: run containerdebug in the background (#605)

* feat: run containerdebug in the background

* cargo update

* remove unused env var

* reintroduce the CONTAINERDEBUG_LOG_DIRECTORY env var

* factor out ctnrdebug test

* up test memory from @nightkr

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 - The lifetime of auto generated TLS certificates is now configurable with the role and roleGroup
   config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#598]).
+- Run a `containerdebug` process in the background of each HBase container to collect debugging information ([#605]).
 
 ### Fixed
 
@@ -15,6 +16,7 @@
 
 [#594]: https://github.com/stackabletech/hbase-operator/pull/594
 [#598]: https://github.com/stackabletech/hbase-operator/pull/598
+[#605]: https://github.com/stackabletech/hbase-operator/pull/605
 
 ## [24.11.0] - 2024-11-18
 

rust/operator-binary/src/hbase_controller.rs

@@ -881,6 +881,7 @@ fn build_rolegroup_statefulset(
             {COMMON_BASH_TRAP_FUNCTIONS}
             {remove_vector_shutdown_file_command}
             prepare_signal_handlers
+            containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
             bin/hbase {hbase_role_name_in_command} start &
             wait_for_termination $!
             {create_vector_shutdown_file_command}
@@ -893,6 +894,11 @@ fn build_rolegroup_statefulset(
                 create_vector_shutdown_file_command(STACKABLE_LOG_DIR),
         }])
         .add_env_vars(merged_env)
+        // Needed for the `containerdebug` process to log it's tracing information to.
+        .add_env_var(
+            "CONTAINERDEBUG_LOG_DIRECTORY",
+            format!("{STACKABLE_LOG_DIR}/containerdebug"),
+        )
         .add_volume_mount("hbase-config", HBASE_CONFIG_TMP_DIR)
         .context(AddVolumeMountSnafu)?
         .add_volume_mount("hdfs-discovery", HDFS_DISCOVERY_TMP_DIR)

tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2

@@ -39,6 +39,9 @@ commands:
           config:
             logging:
               enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+            resources:
+              memory:
+                limit: 1536Mi
           roleGroups:
             default:
               replicas: 2

tests/templates/kuttl/smoke/30-install-hbase.yaml.j2

@@ -45,6 +45,9 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+      resources:
+        memory:
+          limit: 1Gi
     roleGroups:
       default:
         configOverrides:

tests/templates/kuttl/smoke/31-assert.yaml

@@ -0,0 +1,9 @@
+---
+# This test checks if the containerdebug-state.json file is present and valid
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-master-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"'
+  - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-regionserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status  '"valif JSON"'
+  - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-restserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"'