diff --git a/10broker-config.yml b/10broker-config.yml index 2025bbfd..78a1a8c1 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -256,3 +256,22 @@ data: # Change to DEBUG to enable audit log for the authorizer log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender log4j.additivity.kafka.authorizer.logger=false + + jmx-kafka-prometheus.yml: |+ + lowercaseOutputName: true + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi + ssl: false + whitelistObjectNames: ["kafka.server:*","java.lang:*"] + rules: + - pattern : kafka.server<>Value + - pattern : kafka.server<>OneMinuteRate + - pattern : kafka.server<>OneMinuteRate + - pattern : kafka.server<>queue-size + - pattern : kafka.server<>(Value|OneMinuteRate) + - pattern : kafka.server<>(.*) + - pattern : kafka.server<>(.*) + - pattern : kafka.server<>queue-size + - pattern : kafka.server<>OneMinuteRate + - pattern : java.lang<>SystemCpuLoad + - pattern : java.langused + - pattern : java.lang<>FreePhysicalMemorySize diff --git a/50kafka.yml b/50kafka.yml index 73a64850..57933f62 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -11,6 +11,8 @@ spec: labels: app: kafka annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "5556" spec: terminationGracePeriodSeconds: 30 initContainers: @@ -31,6 +33,8 @@ spec: env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties + - name: JMX_PORT + value: "5555" ports: - containerPort: 9092 command: @@ -59,6 +63,27 @@ spec: mountPath: /etc/kafka - name: data mountPath: /var/lib/kafka/data + - name: metrics + image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53 + command: + - java + - -Xmx64M + - -XX:MaxMetaspaceSize=32m + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/kafka/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 100Mi + limits: + memory: 150Mi + volumeMounts: + - name: config + mountPath: /etc/kafka volumes: - name: config configMap: diff --git a/test/jmx-selftest.yml b/test/jmx-selftest.yml new file mode 100644 index 00000000..b73b8071 --- /dev/null +++ b/test/jmx-selftest.yml @@ -0,0 +1,44 @@ +# Sets up a pod that monitors itself, to test resource usage etc. +# testpod=$(kubectl -n test-kafka get pods -l test-target=jmx-exporter -o=jsonpath={.items[*].metadata.name}) +# kubectl exec -n test-kafka $testpod -- apt-get update +# kubectl exec -n test-kafka $testpod -- apt-get install -y --no-install-recommends curl +# kubectl exec -n test-kafka $testpod -- curl http://localhost:5556/metrics +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: jmx-selftest + namespace: test-kafka +spec: + replicas: 1 + template: + metadata: + labels: + test-target: jmx-exporter + test-type: readiness + # Uncomment to test with Prometheus + #annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "5556" + spec: + containers: + - name: monitor + image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53 + command: + - java + - -Dcom.sun.management.jmxremote.ssl=false + - -Dcom.sun.management.jmxremote.authenticate=false + - -Dcom.sun.management.jmxremote.port=5555 + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - example_configs/httpserver_sample_config.yml + ports: + - name: jmx + containerPort: 5555 + - name: slashmetrics + containerPort: 5556 + # Test run, again and again + readinessProbe: + httpGet: + path: /metrics + port: 5556 diff --git a/test/metrics.yml b/test/metrics.yml new file mode 100644 index 00000000..b28cae8c --- /dev/null +++ b/test/metrics.yml @@ -0,0 +1,107 @@ +# kubectl apply -f test/metrics.yml && kubectl scale --replicas=0 deploy/metrics && kubectl scale --replicas=1 deploy/metrics +# kubectl exec metrics-... -- tail -f /tmp/loglast | egrep 'time_total|^jmx_scrape_duration_seconds|^java_lang_memory_heapmemoryusage_used|^java_lang_memory_nonheapmemoryusage_used' +--- +kind: ConfigMap +metadata: + name: metrics + namespace: test-kafka +apiVersion: v1 +data: + + curl-format.txt: |- + \n + # ------ curl stats ------\n + time_namelookup %{time_namelookup}\n + time_connect %{time_connect}\n + time_appconnect %{time_appconnect}\n + time_pretransfer %{time_pretransfer}\n + time_redirect %{time_redirect}\n + time_starttransfer %{time_starttransfer}\n + \n + time_total{url="%{url_effective}"} %{time_total}\n + \n + http_code{url="%{url_effective}"} %{http_code}\n + size_download{url="%{url_effective}"} %{size_download}\n + content_type %{content_type}\n + # ----- curl complete -----\n + \n + + setup.sh: |- + touch /tmp/testlog + tail -f /tmp/testlog + + continue.sh: |- + exit 0 + + run.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + date -u -Iseconds | tee /tmp/loglast + + curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \ + http://kafka-0.broker.kafka.svc.cluster.local:5556/metrics \ + | tee -a /tmp/loglast \ + | grep http_code \ + | grep 200 + + curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \ + http://zoo-0.zoo.kafka.svc.cluster.local:5556/metrics \ + | tee -a /tmp/loglast \ + | grep http_code \ + | grep 200 + + curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \ + http://pzoo-0.pzoo.kafka.svc.cluster.local:5556/metrics \ + | tee -a /tmp/loglast \ + | grep http_code \ + | grep 200 + + exit 0 + +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: metrics + namespace: test-kafka +spec: + replicas: 1 + template: + metadata: + labels: + test-target: kafka + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf + env: + - name: MAX_RESPONSE_TIME + value: "3" + # Test set up + command: + - /bin/bash + - -e + - /test/setup.sh + # Test run, again and again + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/run.sh + # Test quit on nonzero exit + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/continue.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: metrics diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index e796b4ba..eeba1e00 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -35,3 +35,29 @@ data: # Suppress connection log messages, three lines per livenessProbe execution log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN + + jmx-zookeeper-prometheus.yaml: |+ + lowercaseOutputName: true + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + ssl: false + whitelistObjectNames: ["org.apache.ZooKeeperService:*","java.lang:*"] + rules: + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$3" + labels: + replicaId: "$2" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$4" + labels: + replicaId: "$2" + memberType: "$3" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$4_$5" + labels: + replicaId: "$2" + memberType: "$3" + - pattern : java.lang<>SystemCpuLoad + - pattern : java.langused + - pattern : java.lang<>FreePhysicalMemorySize diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 446748a7..593c7a11 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -12,6 +12,8 @@ spec: app: zookeeper storage: persistent annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "5556" spec: terminationGracePeriodSeconds: 10 initContainers: @@ -29,6 +31,8 @@ spec: env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties + - name: JMX_PORT + value: "5555" command: - ./bin/zookeeper-server-start.sh - /etc/kafka/zookeeper.properties @@ -54,6 +58,27 @@ spec: mountPath: /etc/kafka - name: data mountPath: /var/lib/zookeeper/data + - name: metrics + image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53 + command: + - java + - -Xmx64M + - -XX:MaxMetaspaceSize=32m + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/kafka/jmx-zookeeper-prometheus.yaml + ports: + - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 100Mi + limits: + memory: 150Mi + volumeMounts: + - name: config + mountPath: /etc/kafka volumes: - name: config configMap: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index a2922ef2..c88659a5 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -12,6 +12,8 @@ spec: app: zookeeper storage: ephemeral annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "5556" spec: terminationGracePeriodSeconds: 10 initContainers: @@ -32,6 +34,8 @@ spec: env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties + - name: JMX_PORT + value: "5555" command: - ./bin/zookeeper-server-start.sh - /etc/kafka/zookeeper.properties @@ -57,6 +61,27 @@ spec: mountPath: /etc/kafka - name: data mountPath: /var/lib/zookeeper/data + - name: metrics + image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53 + command: + - java + - -Xmx64M + - -XX:MaxMetaspaceSize=32m + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/kafka/jmx-zookeeper-prometheus.yaml + ports: + - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 100Mi + limits: + memory: 150Mi + volumeMounts: + - name: config + mountPath: /etc/kafka volumes: - name: config configMap: