Merge pull request #103 from ImperialCollegeLondon/kafka-in-a-box

Add a Kafka service to docker compose

README.md

@@ -59,11 +59,10 @@ docker compose exec app python scripts/talk_to_process_manager.py
 
 Take the servers down with `docker compose down`
 
-### Working with Kafka
+### Development without Docker Compose
 
-Due to the complexities of containerising Kafka it is not possible to use the standard
-Docker Compose setup. Instead when working with functionality that requires Kafka it is
-necessary to run the individual components manually.
+In the event that you want to develop without using Docker Compose you must start the
+required components manually.
 
 1. Start Kafka - See [Running drunc with pocket kafka].
 

docker-compose.yml

@@ -6,6 +6,7 @@ services:
  - -c
  - |
  python manage.py migrate
+ python scripts/kafka_consumer.py &
  python manage.py runserver 0:8000
  ports:
  - 127.0.0.1:8000:8000
@@ -14,15 +15,38 @@ services:
  - db:/usr/src/app/db
  environment:
  - PROCESS_MANAGER_URL=drunc:10054
+ - KAFKA_URL=kafka:9092
+ depends_on:
+ kafka:
+ condition: service_healthy
  drunc:
  build: ./drunc_docker_service/
  command:
  - sh
  - -c
  - |
  service ssh start &&
- drunc-process-manager --log-level debug file:///process-manager-no-kafka.json
+ drunc-process-manager --log-level debug file:///process-manager-kafka.json
  expose:
  - 10054
+ depends_on:
+ kafka:
+ condition: service_healthy
+ kafka:
+ image: bitnami/kafka:latest
+ environment:
+ - KAFKA_CFG_NODE_ID=0
+ - KAFKA_CFG_PROCESS_ROLES=controller,broker
+ - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+ - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+ - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+ - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+ expose:
+ - 9092
+ healthcheck:
+ test: timeout 5s kafka-cluster.sh cluster-id --bootstrap-server localhost:9092
+ interval: 1s
+ timeout: 6s
+ retries: 20
 volumes:
  db:

drunc_docker_service/Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get update \
 COPY requirements.txt /
 RUN pip install --no-cache-dir -r /requirements.txt
 
-COPY process-manager-no-kafka.json /
+COPY process-manager-no-kafka.json process-manager-kafka.json /
 
 EXPOSE 22
 RUN mkdir -p /root/.ssh && \

drunc_docker_service/process-manager-kafka.json

@@ -0,0 +1,13 @@
+{
+ "type": "ssh",
+ "name": "SSHProcessManager",
+ "command_address": "0.0.0.0:10054",
+ "authoriser": {
+ "type": "dummy"
+ },
+ "broadcaster": {
+ "type": "kafka",
+ "kafka_address": "kafka:9092",
+ "publish_timeout": 2
+ }
+}