Addon: expose /metrics endpoints for Prometheus

10broker-config.yml

@@ -256,3 +256,22 @@ data:
     # Change to DEBUG to enable audit log for the authorizer
     log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender
     log4j.additivity.kafka.authorizer.logger=false
+
+  jmx-kafka-prometheus.yml: |+
+    lowercaseOutputName: true
+    jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi
+    ssl: false
+    whitelistObjectNames: ["kafka.server:*","java.lang:*"]
+    rules:
+      - pattern : kafka.server<type=ReplicaFetcherManager, name=MaxLag, clientId=(.+)><>Value
+      - pattern : kafka.server<type=BrokerTopicMetrics, name=(BytesInPerSec|BytesOutPerSec|MessagesInPerSec), topic=(.+)><>OneMinuteRate
+      - pattern : kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
+      - pattern : kafka.server<type=Produce><>queue-size
+      - pattern : kafka.server<type=ReplicaManager, name=(PartitionCount|UnderReplicatedPartitions)><>(Value|OneMinuteRate)
+      - pattern : kafka.server<type=controller-channel-metrics, broker-id=(.+)><>(.*)
+      - pattern : kafka.server<type=socket-server-metrics, networkProcessor=(.+)><>(.*)
+      - pattern : kafka.server<type=Fetch><>queue-size
+      - pattern : kafka.server<type=SessionExpireListener, name=(.+)><>OneMinuteRate
+      - pattern : java.lang<type=OperatingSystem><>SystemCpuLoad
+      - pattern : java.lang<type=Memory><HeapMemoryUsage>used
+      - pattern : java.lang<type=OperatingSystem><>FreePhysicalMemorySize

50kafka.yml

@@ -11,6 +11,8 @@ spec:
       labels:
         app: kafka
       annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5556"
     spec:
       terminationGracePeriodSeconds: 30
       initContainers:
@@ -31,6 +33,8 @@ spec:
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
+        - name: JMX_PORT
+          value: "5555"
         ports:
         - containerPort: 9092
         command:
@@ -59,6 +63,27 @@ spec:
           mountPath: /etc/kafka
         - name: data
           mountPath: /var/lib/kafka/data
+      - name: metrics
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53
+        command:
+        - java
+        - -Xmx64M
+        - -XX:MaxMetaspaceSize=32m
+        - -jar
+        - jmx_prometheus_httpserver.jar
+        - "5556"
+        - /etc/kafka/jmx-kafka-prometheus.yml
+        ports:
+        - containerPort: 5556
+        resources:
+          requests:
+            cpu: 0m
+            memory: 100Mi
+          limits:
+            memory: 150Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/kafka
       volumes:
       - name: config
         configMap:

test/jmx-selftest.yml

@@ -0,0 +1,44 @@
+# Sets up a pod that monitors itself, to test resource usage etc.
+# testpod=$(kubectl -n test-kafka get pods -l test-target=jmx-exporter -o=jsonpath={.items[*].metadata.name})
+# kubectl exec -n test-kafka $testpod -- apt-get update
+# kubectl exec -n test-kafka $testpod -- apt-get install -y --no-install-recommends curl
+# kubectl exec -n test-kafka $testpod -- curl http://localhost:5556/metrics
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: jmx-selftest
+  namespace: test-kafka
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        test-target: jmx-exporter
+        test-type: readiness
+      # Uncomment to test with Prometheus
+      #annotations:
+      #  prometheus.io/scrape: "true"
+      #  prometheus.io/port: "5556"
+    spec:
+      containers:
+      - name: monitor
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53
+        command:
+        - java
+        - -Dcom.sun.management.jmxremote.ssl=false
+        - -Dcom.sun.management.jmxremote.authenticate=false
+        - -Dcom.sun.management.jmxremote.port=5555
+        - -jar
+        - jmx_prometheus_httpserver.jar
+        - "5556"
+        - example_configs/httpserver_sample_config.yml
+        ports:
+        - name: jmx
+          containerPort: 5555
+        - name: slashmetrics
+          containerPort: 5556
+        # Test run, again and again
+        readinessProbe:
+          httpGet:
+            path: /metrics
+            port: 5556

test/metrics.yml

@@ -0,0 +1,107 @@
+# kubectl apply -f test/metrics.yml && kubectl scale --replicas=0 deploy/metrics && kubectl scale --replicas=1 deploy/metrics
+# kubectl exec metrics-... -- tail -f /tmp/loglast | egrep 'time_total|^jmx_scrape_duration_seconds|^java_lang_memory_heapmemoryusage_used|^java_lang_memory_nonheapmemoryusage_used'
+---
+kind: ConfigMap
+metadata:
+  name: metrics
+  namespace: test-kafka
+apiVersion: v1
+data:
+
+  curl-format.txt: |-
+    \n
+    # ------ curl stats ------\n
+    time_namelookup %{time_namelookup}\n
+    time_connect %{time_connect}\n
+    time_appconnect %{time_appconnect}\n
+    time_pretransfer %{time_pretransfer}\n
+    time_redirect %{time_redirect}\n
+    time_starttransfer %{time_starttransfer}\n
+    \n
+    time_total{url="%{url_effective}"} %{time_total}\n
+    \n
+    http_code{url="%{url_effective}"} %{http_code}\n
+    size_download{url="%{url_effective}"} %{size_download}\n
+    content_type %{content_type}\n
+    # ----- curl complete -----\n
+    \n
+
+  setup.sh: |-
+    touch /tmp/testlog
+    tail -f /tmp/testlog
+
+  continue.sh: |-
+    exit 0
+
+  run.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    date -u -Iseconds | tee /tmp/loglast
+
+    curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \
+      http://kafka-0.broker.kafka.svc.cluster.local:5556/metrics \
+      | tee -a /tmp/loglast \
+      | grep http_code \
+      | grep 200
+
+    curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \
+      http://zoo-0.zoo.kafka.svc.cluster.local:5556/metrics \
+      | tee -a /tmp/loglast \
+      | grep http_code \
+      | grep 200
+
+    curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \
+      http://pzoo-0.pzoo.kafka.svc.cluster.local:5556/metrics \
+      | tee -a /tmp/loglast \
+      | grep http_code \
+      | grep 200
+
+    exit 0
+
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: metrics
+  namespace: test-kafka
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        test-target: kafka
+        test-type: readiness
+    spec:
+      containers:
+      - name: testcase
+        image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf
+        env:
+        - name: MAX_RESPONSE_TIME
+          value: "3"
+        # Test set up
+        command:
+        - /bin/bash
+        - -e
+        - /test/setup.sh
+        # Test run, again and again
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/run.sh
+        # Test quit on nonzero exit
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/continue.sh
+        volumeMounts:
+        - name: config
+          mountPath: /test
+      volumes:
+      - name: config
+        configMap:
+          name: metrics

zookeeper/10zookeeper-config.yml

@@ -35,3 +35,29 @@ data:
     # Suppress connection log messages, three lines per livenessProbe execution
     log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN
     log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
+
+  jmx-zookeeper-prometheus.yaml: |+
+    lowercaseOutputName: true
+    jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
+    ssl: false
+    whitelistObjectNames: ["org.apache.ZooKeeperService:*","java.lang:*"]
+    rules:
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d)><>(\\w+)"
+      name: "zookeeper_$2"
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d)><>(\\w+)"
+      name: "zookeeper_$3"
+      labels:
+        replicaId: "$2"
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d), name2=(\\w+)><>(\\w+)"
+      name: "zookeeper_$4"
+      labels:
+        replicaId: "$2"
+        memberType: "$3"
+    - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d), name2=(\\w+), name3=(\\w+)><>(\\w+)"
+      name: "zookeeper_$4_$5"
+      labels:
+        replicaId: "$2"
+        memberType: "$3"
+    - pattern : java.lang<type=OperatingSystem><>SystemCpuLoad
+    - pattern : java.lang<type=Memory><HeapMemoryUsage>used
+    - pattern : java.lang<type=OperatingSystem><>FreePhysicalMemorySize

zookeeper/50pzoo.yml

@@ -12,6 +12,8 @@ spec:
         app: zookeeper
         storage: persistent
       annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5556"
     spec:
       terminationGracePeriodSeconds: 10
       initContainers:
@@ -29,6 +31,8 @@ spec:
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
+        - name: JMX_PORT
+          value: "5555"
         command:
         - ./bin/zookeeper-server-start.sh
         - /etc/kafka/zookeeper.properties
@@ -54,6 +58,27 @@ spec:
           mountPath: /etc/kafka
         - name: data
           mountPath: /var/lib/zookeeper/data
+      - name: metrics
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53
+        command:
+        - java
+        - -Xmx64M
+        - -XX:MaxMetaspaceSize=32m
+        - -jar
+        - jmx_prometheus_httpserver.jar
+        - "5556"
+        - /etc/kafka/jmx-zookeeper-prometheus.yaml
+        ports:
+        - containerPort: 5556
+        resources:
+          requests:
+            cpu: 0m
+            memory: 100Mi
+          limits:
+            memory: 150Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/kafka
       volumes:
       - name: config
         configMap:

zookeeper/51zoo.yml

@@ -12,6 +12,8 @@ spec:
         app: zookeeper
         storage: ephemeral
       annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5556"
     spec:
       terminationGracePeriodSeconds: 10
       initContainers:
@@ -32,6 +34,8 @@ spec:
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
+        - name: JMX_PORT
+          value: "5555"
         command:
         - ./bin/zookeeper-server-start.sh
         - /etc/kafka/zookeeper.properties
@@ -57,6 +61,27 @@ spec:
           mountPath: /etc/kafka
         - name: data
           mountPath: /var/lib/zookeeper/data
+      - name: metrics
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53
+        command:
+        - java
+        - -Xmx64M
+        - -XX:MaxMetaspaceSize=32m
+        - -jar
+        - jmx_prometheus_httpserver.jar
+        - "5556"
+        - /etc/kafka/jmx-zookeeper-prometheus.yaml
+        ports:
+        - containerPort: 5556
+        resources:
+          requests:
+            cpu: 0m
+            memory: 100Mi
+          limits:
+            memory: 150Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/kafka
       volumes:
       - name: config
         configMap: