Merge pull request #95 from Yolean/ops-jobs

Add Jobs and tests for common maintenance operations

maintenance/README.md

@@ -0,0 +1,32 @@
+
+## Re-assign Leadership
+
+This is one of the cases where this repo begs to differ from traditional Kafka setups.
+In Kubernetes the restart of a pod, and subsequent start on a different node, should be a non-event.
+
+> ”when a broker is stopped and restarted, it does not resume leadership of any partitions automatically”
+
+_-- Neha Narkhede, Gwen Shapira, and Todd Palino. ”Kafka: The Definitive Guide”_
+
+Create the `preferred-replica-election-job.yml` resource, after deleting any previous one.
+
+## Change a Partition's Replicas
+
+> ”From time to time, it may be necessary to change the replica assignments for a partition. Some examples of when this might be needed are:
+>  * If a topic’s partitions are not balanced across the cluster, causing uneven load on brokers
+>  * If a broker is taken offline and the partition is under-replicated
+>  * If a new broker is added and needs to receive a share of the cluster load”
+
+_-- Neha Narkhede, Gwen Shapira, and Todd Palino. ”Kafka: The Definitive Guide”_
+
+Use the `reassign-paritions-job.yml`, after editing `TOPICS` and `BROKERS`.
+
+## Increase a topic's replication factor
+
+See https://github.com/Yolean/kubernetes-kafka/pull/140
+
+Use the `replication-factor-increase-job.yml`, after editing `TOPICS` and `BROKERS`.
+
+The affected topics may end up without a preferred replica. See above to fix that,
+or to affect only your selected topics use [Kafka Manager](https://github.com/Yolean/kubernetes-kafka/pull/83)'s topic screen,
+Generate Partition Assignments followed by Reassign Partitions.

maintenance/preferred-replica-election-job.yml

@@ -0,0 +1,19 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: preferred-replica-election
+  namespace: kafka
+spec:
+  template:
+    metadata:
+      name: preferred-replica-election
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        command:
+        - ./bin/kafka-preferred-replica-election.sh
+        - --zookeeper
+        - zookeeper:2181
+      restartPolicy: Never
+  backoffLimit: 3

maintenance/reassign-paritions-job.yml

@@ -0,0 +1,51 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: reassign-partitions
+  namespace: kafka
+spec:
+  template:
+    metadata:
+      name: reassign-partitions
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        env:
+        - name: ZOOKEEPER
+          value: zookeeper.kafka:2181
+        # the following must be edited per job
+        - name: TOPICS
+          value: test-produce-consume,test-kafkacat
+        - name: BROKERS
+          value: 0,2
+        command:
+        - /bin/bash
+        - -ce
+        - >
+          echo '{"topics":[' > /tmp/reassign-topics.json;
+          echo -n ' {"topic":"' >> /tmp/reassign-topics.json;
+          echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json;
+          echo '"}' >> /tmp/reassign-topics.json;
+          echo ']}' >> /tmp/reassign-topics.json;
+
+          echo "# reassign-topics.json";
+          cat /tmp/reassign-topics.json;
+
+          ./bin/kafka-reassign-partitions.sh 
+          --zookeeper=$ZOOKEEPER
+          --generate
+          --topics-to-move-json-file=/tmp/reassign-topics.json
+          --broker-list=$BROKERS > /tmp/generated.txt;
+
+          tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json;
+
+          echo "# proposed-reassignment.json";
+          cat /tmp/proposed-reassignment.json;
+
+          ./bin/kafka-reassign-partitions.sh
+          --zookeeper=$ZOOKEEPER
+          --execute
+          --reassignment-json-file=/tmp/proposed-reassignment.json;
+      restartPolicy: Never
+  backoffLimit: 3

maintenance/replication-factor-increase-job.yml

@@ -0,0 +1,65 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: replication-factor-increase
+  namespace: kafka
+spec:
+  template:
+    metadata:
+      name: replication-factor-increase
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        env:
+        - name: ZOOKEEPER
+          value: zookeeper.kafka:2181
+        # the following must be edited per job
+        - name: TOPICS
+          value: ""
+        - name: BROKERS
+          value: 0,1,2
+        command:
+        - /bin/bash
+        - -ce
+        - >
+          if [ -z "$TOPICS" ];  then
+            echo "Please set the TOPICS env (comma-separated) and re-create the job"
+            tail -f /dev/null
+          fi
+
+          echo '{"topics":[' > /tmp/reassign-topics.json;
+          echo -n ' {"topic":"' >> /tmp/reassign-topics.json;
+          echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json;
+          echo '"}' >> /tmp/reassign-topics.json;
+          echo ']}' >> /tmp/reassign-topics.json;
+
+          echo "# reassign-topics.json";
+          cat /tmp/reassign-topics.json;
+
+          ./bin/kafka-reassign-partitions.sh
+          --zookeeper=$ZOOKEEPER
+          --generate
+          --topics-to-move-json-file=/tmp/reassign-topics.json
+          --broker-list=$BROKERS > /tmp/generated.txt;
+
+          tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json;
+
+          sleep 1;
+          echo "# proposed-reassignment.json";
+          cat /tmp/proposed-reassignment.json;
+
+          sed -i 's/"replicas":\[.\]/"replicas":[0,1,2]/g' /tmp/proposed-reassignment.json;
+          sed -i 's/,"log_dirs":\["any"\]//g' /tmp/proposed-reassignment.json;
+          echo "# proposed-reassignment.json modified to affect replication factor";
+          cat /tmp/proposed-reassignment.json;
+
+          echo "# Triggering kafka-reassign-partitions.sh"
+          ./bin/kafka-reassign-partitions.sh
+          --zookeeper=$ZOOKEEPER
+          --execute
+          --reassignment-json-file=/tmp/proposed-reassignment.json;
+
+          echo "# Reassignment exited. Upon success you may want to run preferred-replica-election."
+      restartPolicy: Never
+  backoffLimit: 3

maintenance/test/replicated-partitions.yml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: replicated-partitions
+  namespace: test-kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      test-type: readiness
+      test-target: under-replicated-partitions
+  template:
+    metadata:
+      labels:
+        test-type: readiness
+        test-target: under-replicated-partitions
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        command:
+        - /bin/bash
+        - -ec
+        - >
+          touch /tmp/testlog;
+          tail -f /tmp/testlog
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -c
+            - >
+              echo "### $(date -Ins -u) ###" >> /tmp/testlog
+              &&
+              [
+              $(
+              ./bin/kafka-topics.sh
+              --zookeeper zookeeper.kafka:2181
+              --describe
+              --under-replicated-partitions
+              |
+              tee -a /tmp/testlog
+              |
+              wc -l
+              )
+              -eq
+              0
+              ]
+          periodSeconds: 30
+          timeoutSeconds: 29