diff --git a/50kafka.yml b/50kafka.yml index 372eba8b..2a3faf6a 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -20,6 +20,13 @@ spec: image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 80Mi - name: broker image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -36,6 +43,10 @@ spec: --override log.retention.hours=-1 --override log.dirs=/var/lib/kafka/data/topics --override broker.id=${HOSTNAME##*-} + resources: + requests: + cpu: 100m + memory: 512Mi volumeMounts: - name: data mountPath: /var/lib/kafka/data diff --git a/README.md b/README.md index ef5db0c8..960de581 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,6 @@ To get consistent service DNS names `kafka-N.broker.kafka`(`.svc.cluster.local`) kubectl create -f 00namespace.yml ``` -## Set up volume claims - -You may add [storage class](http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses) -to the kafka StatefulSet declaration to enable automatic volume provisioning. - -Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistent-volumes)s and [PVC](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims)s manually. For example in Minikube. - -``` -./bootstrap/pv.sh -kubectl create -f ./10pvc.yml -# check that claims are bound -kubectl -n kafka get pvc -``` - ## Set up Zookeeper The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. diff --git a/test/monitoring-test.yml b/test/monitoring-test.yml new file mode 100644 index 00000000..4b5f230a --- /dev/null +++ b/test/monitoring-test.yml @@ -0,0 +1,35 @@ +# Sets up a pod that monitors itself, to test resource usage etc. +# kubectl exec test-metrics-... -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics' +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: monitoring-test + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: monitoring-test + # Uncomment to test with prometheus + #annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "5556" + spec: + containers: + - name: monitor + image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d + command: + - java + - -Dcom.sun.management.jmxremote.ssl=false + - -Dcom.sun.management.jmxremote.authenticate=false + - -Dcom.sun.management.jmxremote.port=5555 + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - example_configs/httpserver_sample_config.yml + ports: + - name: jmx + containerPort: 5555 + - name: slashmetrics + containerPort: 5556 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 25e2ceb4..da1ba818 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -20,13 +20,20 @@ spec: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d command: - - "java" - - "-jar" - - "jmx_prometheus_httpserver.jar" + - java + - -jar + - jmx_prometheus_httpserver.jar - "5556" - example_configs/zookeeper.yaml ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 80Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -49,6 +56,10 @@ spec: name: peer - containerPort: 3888 name: leader-election + resources: + requests: + cpu: 10m + memory: 100Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index f041a9a9..f4f6298b 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -20,13 +20,20 @@ spec: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d command: - - "java" - - "-jar" - - "jmx_prometheus_httpserver.jar" + - java + - -jar + - jmx_prometheus_httpserver.jar - "5556" - example_configs/zookeeper.yaml ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 80Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -49,6 +56,10 @@ spec: name: peer - containerPort: 3888 name: leader-election + resources: + requests: + cpu: 10m + memory: 100Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config