[DPE-5053] Add jmx exporter (#110)

canonical · Sep 26, 2024 · 3a877ab · 3a877ab
1 parent b9dfbd6
commit 3a877ab
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 1 deletion.
diff --git a/.github/workflows/integration-gpu.yaml b/.github/workflows/integration-gpu.yaml
@@ -45,7 +45,7 @@ jobs:
  run: |
  sudo -E bash ./tests/integration/setup-environment.sh
  env:
- MICROK8S_CHANNEL: 1.29/stable
+ MICROK8S_CHANNEL: 1.28/stable
 
  - name: Setup microk8s
  timeout-minutes: 30

diff --git a/images/charmed-spark/conf/jmx_prometheus.yaml b/images/charmed-spark/conf/jmx_prometheus.yaml
@@ -0,0 +1,2 @@
+rules:
+- pattern: ".*"
diff --git a/images/charmed-spark/rockcraft.yaml b/images/charmed-spark/rockcraft.yaml
@@ -100,14 +100,17 @@ parts:
  ICEBERG_VERSION='1.4.3'
  SPARK_METRICS_VERSION='3.4-1.0.2'
  SERVLET_FILTERS_VERSION='0.0.1'
+ PROMETHEUS_JMX_EXPORTER_VERSION='0.20.0'
  SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
  SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='9be728c3bda6a8e9db77452f416bc23245271a5db2da64557429352917c0772801ead19f3b1a33f955ec2eced3cb952c6c3a7c617cdeb4389cd17284f3c711f7'
  SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096'
+ SHA1SUM_PROMETHEUS_JMX_EXPORTER='7b8a98e3482cee8889698ef391b85c47a3c4ce5b'
 
  JARS=(
  "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/LIB_VERSION/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-LIB_VERSION.jar $ICEBERG_VERSION sha1sum $SHA1SUM_ICEBERG_JAR"
  "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-LIB_VERSION/spark-metrics-assembly-LIB_VERSION.jar $SPARK_METRICS_VERSION sha512sum $SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR"
  "https://github.com/canonical/central-uploader/releases/download/servlet-filters-LIB_VERSION/servlet-filters-LIB_VERSION.jar $SERVLET_FILTERS_VERSION sha512sum $SHA512SUM_SPARK_SERVLET_FILTER_JAR"
+ "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/LIB_VERSION/jmx_prometheus_javaagent-LIB_VERSION.jar $PROMETHEUS_JMX_EXPORTER_VERSION sha1sum $SHA1SUM_PROMETHEUS_JMX_EXPORTER"
  )
  for ENTRY in "${JARS[@]}"; do
  echo "$ENTRY"
@@ -171,6 +174,7 @@ parts:
  source: .
  organize:
  conf/spark-defaults.conf: etc/spark8t/conf/spark-defaults.conf
+ conf/jmx_prometheus.yaml: etc/spark/conf/jmx_prometheus.yaml
  bin/sparkd.sh: opt/pebble/sparkd.sh
  bin/history-server.sh: opt/pebble/history-server.sh
  bin/spark-client.pyspark: opt/spark-client/python/bin/spark-client.pyspark
@@ -180,6 +184,7 @@ parts:
  bin/spark-client.spark-submit: opt/spark-client/python/bin/spark-client.spark-submit
  stage:
  - etc/spark8t/conf/
+ - etc/spark/conf/
  - opt/pebble/sparkd.sh
  - opt/pebble/history-server.sh
  - opt/spark-client/python/bin/spark-client.pyspark