From 3bc821b085281e5563be20221583870746e07ad8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson <staffan@repos.se> Date: Tue, 27 Jun 2017 08:31:04 +0200 Subject: [PATCH 1/5] Adds tentative resource requests, based on what idle pods use (though this includes monitoring) --- 50kafka.yml | 4 ++++ zookeeper/50pzoo.yml | 4 ++++ zookeeper/51zoo.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/50kafka.yml b/50kafka.yml index 372eba8b..56460491 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -36,6 +36,10 @@ spec: --override log.retention.hours=-1 --override log.dirs=/var/lib/kafka/data/topics --override broker.id=${HOSTNAME##*-} + resources: + requests: + cpu: 100m + memory: 600Mi volumeMounts: - name: data mountPath: /var/lib/kafka/data diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 25e2ceb4..db992071 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -49,6 +49,10 @@ spec: name: peer - containerPort: 3888 name: leader-election + resources: + requests: + cpu: 10m + memory: 120Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index f041a9a9..caa13d6e 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -49,6 +49,10 @@ spec: name: peer - containerPort: 3888 name: leader-election + resources: + requests: + cpu: 10m + memory: 120Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config From 620c4e21134ccdb20ba2e00d25f16bee5a1be6e9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson <staffan@repos.se> Date: Tue, 27 Jun 2017 10:27:31 +0200 Subject: [PATCH 2/5] Removes volume claims documentation, as we've gone completely dynamic --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index ef5db0c8..960de581 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,6 @@ To get consistent service DNS names `kafka-N.broker.kafka`(`.svc.cluster.local`) kubectl create -f 00namespace.yml ``` -## Set up volume claims - -You may add [storage class](http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses) -to the kafka StatefulSet declaration to enable automatic volume provisioning. - -Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistent-volumes)s and [PVC](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims)s manually. For example in Minikube. - -``` -./bootstrap/pv.sh -kubectl create -f ./10pvc.yml -# check that claims are bound -kubectl -n kafka get pvc -``` - ## Set up Zookeeper The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. From f4ac28897dadadf976f862cb525726c30e4f49bc Mon Sep 17 00:00:00 2001 From: Staffan Olsson <staffan@repos.se> Date: Tue, 27 Jun 2017 10:42:19 +0200 Subject: [PATCH 3/5] A monitoring-only pod uses 0m / ~32Mi resources --- 50kafka.yml | 9 ++++++++- test/monitoring-test.yml | 35 +++++++++++++++++++++++++++++++++++ zookeeper/50pzoo.yml | 9 ++++++++- zookeeper/51zoo.yml | 15 +++++++++++---- 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 test/monitoring-test.yml diff --git a/50kafka.yml b/50kafka.yml index 56460491..806f6500 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -20,6 +20,13 @@ spec: image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 40Mi - name: broker image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -39,7 +46,7 @@ spec: resources: requests: cpu: 100m - memory: 600Mi + memory: 512Mi volumeMounts: - name: data mountPath: /var/lib/kafka/data diff --git a/test/monitoring-test.yml b/test/monitoring-test.yml new file mode 100644 index 00000000..4b5f230a --- /dev/null +++ b/test/monitoring-test.yml @@ -0,0 +1,35 @@ +# Sets up a pod that monitors itself, to test resource usage etc. +# kubectl exec test-metrics-... -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics' +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: monitoring-test + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: monitoring-test + # Uncomment to test with prometheus + #annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "5556" + spec: + containers: + - name: monitor + image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d + command: + - java + - -Dcom.sun.management.jmxremote.ssl=false + - -Dcom.sun.management.jmxremote.authenticate=false + - -Dcom.sun.management.jmxremote.port=5555 + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - example_configs/httpserver_sample_config.yml + ports: + - name: jmx + containerPort: 5555 + - name: slashmetrics + containerPort: 5556 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index db992071..12890958 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -27,6 +27,13 @@ spec: - example_configs/zookeeper.yaml ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 40Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -52,7 +59,7 @@ spec: resources: requests: cpu: 10m - memory: 120Mi + memory: 100Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index caa13d6e..82cf7ee2 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -20,13 +20,20 @@ spec: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d command: - - "java" - - "-jar" - - "jmx_prometheus_httpserver.jar" + - java + - -jar + - jmx_prometheus_httpserver.jar - "5556" - example_configs/zookeeper.yaml ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 40Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -52,7 +59,7 @@ spec: resources: requests: cpu: 10m - memory: 120Mi + memory: 100Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config From 1a8f2d95be8d56cdd57bb3dccb366a5c4fc2e17b Mon Sep 17 00:00:00 2001 From: Staffan Olsson <staffan@repos.se> Date: Tue, 27 Jun 2017 10:56:49 +0200 Subject: [PATCH 4/5] s --- zookeeper/50pzoo.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 12890958..7dfc1e64 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -20,9 +20,9 @@ spec: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d command: - - "java" - - "-jar" - - "jmx_prometheus_httpserver.jar" + - java + - -jar + - jmx_prometheus_httpserver.jar - "5556" - example_configs/zookeeper.yaml ports: From 13442382d2be57fc9e368ceb67f392c5494fdfa9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson <staffan@repos.se> Date: Tue, 27 Jun 2017 12:23:21 +0200 Subject: [PATCH 5/5] Got quite repeatable OOMKilled on pzoo pods, so I figured it must be... in metrics becuase nither zoo nor kafka has limits --- 50kafka.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/50kafka.yml b/50kafka.yml index 806f6500..2a3faf6a 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -26,7 +26,7 @@ spec: memory: 40Mi limits: cpu: 10m - memory: 40Mi + memory: 80Mi - name: broker image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 7dfc1e64..da1ba818 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: memory: 40Mi limits: cpu: 10m - memory: 40Mi + memory: 80Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 82cf7ee2..f4f6298b 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -33,7 +33,7 @@ spec: memory: 40Mi limits: cpu: 10m - memory: 40Mi + memory: 80Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: