From 63d128d335b85113281a240413893a2603d7f7a9 Mon Sep 17 00:00:00 2001 From: Charles Dufour Date: Sun, 10 Jul 2022 19:28:52 +0200 Subject: [PATCH] Update CP to version 7.2.0 (M1 support) and enable docker container to kafka connexion --- README.md | 48 +++++++++---------- full-stack.yml | 19 ++++---- ...ultiple-kafka-multiple-schema-registry.yml | 35 +++++++------- zk-multiple-kafka-multiple.yml | 33 +++++++------ zk-multiple-kafka-single.yml | 15 +++--- zk-single-kafka-multiple.yml | 29 ++++++----- zk-single-kafka-single.yml | 11 +++-- 7 files changed, 100 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index ca99eb7..1772126 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This project is sponsored by [Conduktor.io](https://www.conduktor.io/), a graphical desktop user interface for Apache Kafka. Once you have started your cluster, you can use Conduktor to easily manage it. -Just connect against `localhost:9092` if using Docker, or `192.168.99.100` if using Docker Toolbox +Just connect against `localhost:9092`. If you are on Mac or Windows and want to connect from another container, use `host.docker.internal:29092` # kafka-stack-docker-compose @@ -15,38 +15,36 @@ This replicates as well as possible real deployment configurations, where you ha ## Stack version - - Zookeeper version: 3.6.3 (Confluent 7.1.1) - - Kafka version: 3.1.0 (Confluent 7.1.1) - - Kafka Schema Registry: Confluent 7.1.1 - - Kafka Rest Proxy: Confluent 7.1.1 - - Kafka Connect: Confluent 7.1.1 - - ksqlDB Server: Confluent 7.1.1 + - Zookeeper version: 3.6.3 (Confluent 7.2.0) + - Kafka version: 3.2.0 (Confluent 7.2.0) + - Kafka Schema Registry: Confluent 7.2.0 + - Kafka Rest Proxy: Confluent 7.2.0 + - Kafka Connect: Confluent 7.2.0 + - ksqlDB Server: Confluent 7.2.0 - Zoonavigator: 1.1.1 For a UI tool to access your local Kafka cluster, use the free version of [Conduktor](https://www.conduktor.io/download) # Requirements -## Docker - -Please export your environment before starting the stack: -``` -export DOCKER_HOST_IP=127.0.0.1 -``` -(that's the default value and you actually don't need to do a thing) +Kafka will be exposed on `127.0.0.1` or `DOCKER_HOST_IP` if set in the environment. +(You probably don't need to set it if you're not using Docker-Toolbox) ## Docker-Toolbox -If you are using Docker for Mac <= 1.11, or Docker Toolbox for Windows -(your docker machine IP is usually `192.168.99.100`) - -Please export your environment before starting the stack: +Docker toolbox is [deprecated](https://github.com/docker-archive/toolbox) and not maintained anymore for several years. +We can't guarantee this stack will work with Docker Toolbox, but if you want to try anyway, please export your environment before starting the stack: ``` export DOCKER_HOST_IP=192.168.99.100 ``` +(your docker machine IP is usually `192.168.99.100`) + +## Apple M1 support +Confluent platform supports Apple M1 (ARM64) since version `7.2.0`! Basically, this stack will work out of the box. -## Mac M1 issues +If you want to downgrade confluent platform version, there are two ways: -Currently, the Docker Images are not working with M1 Mac. This is because they haven't been built by Confluent for that platform. See [confluentinc/common-docker/#117](https://github.com/confluentinc/common-docker/issues/117) for more details +1. Add `platform: linux/amd64`. It will work as docker is able to emulate AMD64 instructions. +2. Previous versions have been [built](https://github.com/arm64-compat/confluent-platform) for ARM64 by the community. If you want to use it, just change the image in the corresponding yml. Since it is a not an official image, use it at your own risks. ## Single Zookeeper / Single Kafka @@ -155,12 +153,12 @@ A: yes. Say you want to change `zoo1` port to `12181` (only relevant lines are s A: yes. Say you want to change `kafka1` port to `12345` (only relevant lines are shown). Note only `LISTENER_DOCKER_EXTERNAL` changes: ``` kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 ports: - "12345:12345" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:12345 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:12345,DOCKER://host.docker.internal:29092 ``` **Q: Kafka is using a lot of disk space for testing. Can I reduce it?** @@ -169,7 +167,7 @@ A: yes. This is for testing only!!! Reduce the KAFKA_LOG_SEGMENT_BYTES to 16MB a ``` kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 ... environment: ... @@ -185,11 +183,11 @@ For example, if the IP of your machine is `50.10.2.3`, follow the sample mapping ``` kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 ... environment: ... - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://50.10.2.3:9093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://50.10.2.3:9093,DOCKER://host.docker.internal:29093 ``` **Q: How do I add connectors to kafka connect?** diff --git a/full-stack.yml b/full-stack.yml index 28b24ad..7879b28 100644 --- a/full-stack.yml +++ b/full-stack.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,16 +13,17 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -37,7 +38,7 @@ services: - zoo1 kafka-schema-registry: - image: confluentinc/cp-schema-registry:7.1.1 + image: confluentinc/cp-schema-registry:7.2.0 hostname: kafka-schema-registry container_name: kafka-schema-registry ports: @@ -52,7 +53,7 @@ services: kafka-rest-proxy: - image: confluentinc/cp-kafka-rest:7.1.1 + image: confluentinc/cp-kafka-rest:7.2.0 hostname: kafka-rest-proxy container_name: kafka-rest-proxy ports: @@ -70,7 +71,7 @@ services: kafka-connect: - image: confluentinc/cp-kafka-connect:7.1.1 + image: confluentinc/cp-kafka-connect:7.2.0 hostname: kafka-connect container_name: kafka-connect ports: @@ -112,7 +113,7 @@ services: ksqldb-server: - image: confluentinc/cp-ksqldb-server:7.1.1 + image: confluentinc/cp-ksqldb-server:7.2.0 hostname: ksqldb-server container_name: ksqldb-server ports: diff --git a/zk-multiple-kafka-multiple-schema-registry.yml b/zk-multiple-kafka-multiple-schema-registry.yml index df62a77..7b6caa5 100644 --- a/zk-multiple-kafka-multiple-schema-registry.yml +++ b/zk-multiple-kafka-multiple-schema-registry.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +13,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +24,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo3 container_name: zoo3 ports: @@ -36,15 +36,16 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -56,15 +57,16 @@ services: - zoo3 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka2 container_name: kafka2 ports: - "9093:9093" + - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -76,15 +78,16 @@ services: - zoo3 kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka3 container_name: kafka3 ports: - "9094:9094" + - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -96,7 +99,7 @@ services: - zoo3 kafka-schema-registry: - image: confluentinc/cp-schema-registry:7.1.1 + image: confluentinc/cp-schema-registry:7.2.0 hostname: kafka-schema-registry container_name: kafka-schema-registry depends_on: diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml index af993dc..a000789 100644 --- a/zk-multiple-kafka-multiple.yml +++ b/zk-multiple-kafka-multiple.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +13,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +24,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo3 container_name: zoo3 ports: @@ -37,15 +37,16 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -57,15 +58,16 @@ services: - zoo3 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka2 container_name: kafka2 ports: - "9093:9093" + - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -77,15 +79,16 @@ services: - zoo3 kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka3 container_name: kafka3 ports: - "9094:9094" + - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" diff --git a/zk-multiple-kafka-single.yml b/zk-multiple-kafka-single.yml index f5d492e..7f5b73d 100644 --- a/zk-multiple-kafka-single.yml +++ b/zk-multiple-kafka-single.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +13,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +24,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo3 container_name: zoo3 ports: @@ -36,16 +36,17 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml index 9e49667..fd15096 100644 --- a/zk-single-kafka-multiple.yml +++ b/zk-single-kafka-multiple.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo1 container_name: zoo1 ports: @@ -14,15 +14,16 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -32,15 +33,16 @@ services: - zoo1 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka2 container_name: kafka2 ports: - "9093:9093" + - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -51,15 +53,16 @@ services: kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka3 container_name: kafka3 ports: - "9094:9094" + - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" diff --git a/zk-single-kafka-single.yml b/zk-single-kafka-single.yml index ab67625..bda17b5 100644 --- a/zk-single-kafka-single.yml +++ b/zk-single-kafka-single.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.2.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,16 +13,17 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"