diff --git a/README.md b/README.md index f548566..3e6a56c 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,10 @@ docker compose exec app python scripts/talk_to_process_manager.py Take the servers down with `docker compose down` -### Working with Kafka +### Development without Docker Compose -Due to the complexities of containerising Kafka it is not possible to use the standard -Docker Compose setup. Instead when working with functionality that requires Kafka it is -necessary to run the individual components manually. +In the event that you want to develop without using Docker Compose you must start the +required components manually. 1. Start Kafka - See [Running drunc with pocket kafka]. diff --git a/docker-compose.yml b/docker-compose.yml index 9e1fc82..b5cfb24 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: - -c - | python manage.py migrate + python scripts/kafka_consumer.py & python manage.py runserver 0:8000 ports: - 127.0.0.1:8000:8000 @@ -14,6 +15,10 @@ services: - db:/usr/src/app/db environment: - PROCESS_MANAGER_URL=drunc:10054 + - KAFKA_URL=kafka:9092 + depends_on: + kafka: + condition: service_healthy drunc: build: ./drunc_docker_service/ command: @@ -21,8 +26,27 @@ services: - -c - | service ssh start && - drunc-process-manager --log-level debug file:///process-manager-no-kafka.json + drunc-process-manager --log-level debug file:///process-manager-kafka.json expose: - 10054 + depends_on: + kafka: + condition: service_healthy + kafka: + image: bitnami/kafka:latest + environment: + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + expose: + - 9092 + healthcheck: + test: timeout 5s kafka-cluster.sh cluster-id --bootstrap-server localhost:9092 + interval: 1s + timeout: 6s + retries: 20 volumes: db: diff --git a/drunc_docker_service/Dockerfile b/drunc_docker_service/Dockerfile index 98c9867..b318e23 100644 --- a/drunc_docker_service/Dockerfile +++ b/drunc_docker_service/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update \ COPY requirements.txt / RUN pip install --no-cache-dir -r /requirements.txt -COPY process-manager-no-kafka.json / +COPY process-manager-no-kafka.json process-manager-kafka.json / EXPOSE 22 RUN mkdir -p /root/.ssh && \ diff --git a/drunc_docker_service/process-manager-kafka.json b/drunc_docker_service/process-manager-kafka.json new file mode 100644 index 0000000..bfff8d6 --- /dev/null +++ b/drunc_docker_service/process-manager-kafka.json @@ -0,0 +1,13 @@ +{ + "type": "ssh", + "name": "SSHProcessManager", + "command_address": "0.0.0.0:10054", + "authoriser": { + "type": "dummy" + }, + "broadcaster": { + "type": "kafka", + "kafka_address": "kafka:9092", + "publish_timeout": 2 + } +}