From bf8b59175aa3226c4ae747a6587f3ed74ec1405b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 2 Oct 2017 21:31:20 +0200 Subject: [PATCH 1/5] Runs KSQL using the kind of deployment that REST Proxy has --- ksql/ksql-config.yml | 30 ++++++++++++++++++++++++++++++ ksql/ksql-service.yml | 10 ++++++++++ ksql/ksql.yml | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 ksql/ksql-config.yml create mode 100644 ksql/ksql-service.yml create mode 100644 ksql/ksql.yml diff --git a/ksql/ksql-config.yml b/ksql/ksql-config.yml new file mode 100644 index 00000000..2e1a3725 --- /dev/null +++ b/ksql/ksql-config.yml @@ -0,0 +1,30 @@ +kind: ConfigMap +metadata: + name: ksql-config + namespace: kafka +apiVersion: v1 +data: + ksqlserver.properties: |- + bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092 + application.id=ksql_server_quickstart + ksql.command.topic.suffix=commands + + listeners=http://0.0.0.0:80 + + log4j.properties: |- + log4j.rootLogger=INFO, stdout + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.appender.streams=org.apache.log4j.ConsoleAppender + log4j.appender.streams.layout=org.apache.log4j.PatternLayout + log4j.appender.streams.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.logger.kafka=ERROR, stdout + log4j.logger.org.apache.kafka.streams=INFO, streams + log4j.additivity.org.apache.kafka.streams=false + log4j.logger.org.apache.zookeeper=ERROR, stdout + log4j.logger.org.apache.kafka=ERROR, stdout + log4j.logger.org.I0Itec.zkclient=ERROR, stdout diff --git a/ksql/ksql-service.yml b/ksql/ksql-service.yml new file mode 100644 index 00000000..2668592e --- /dev/null +++ b/ksql/ksql-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: ksql + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: ksql diff --git a/ksql/ksql.yml b/ksql/ksql.yml new file mode 100644 index 00000000..b0429551 --- /dev/null +++ b/ksql/ksql.yml @@ -0,0 +1,37 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: ksql + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: ksql + spec: + containers: + - name: cp + image: solsson/kafka-ksql@sha256:5b8555bcc047ca26c28f3932a1f3cb84baa7bb2f0f4bdc4856b9881053ae4465 + readinessProbe: + httpGet: + path: / + port: 80 + livenessProbe: + httpGet: + path: / + port: 80 + ports: + - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/ksql-rest-app + volumes: + - name: config + configMap: + name: ksql-config From 64a4075107f5932940ee3784749896c908a7ceec Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 3 Oct 2017 06:19:47 +0200 Subject: [PATCH 2/5] New KSQL image build fixes the ClassNotFoundException --- ksql/ksql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ksql/ksql.yml b/ksql/ksql.yml index b0429551..7de6d239 100644 --- a/ksql/ksql.yml +++ b/ksql/ksql.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-ksql@sha256:5b8555bcc047ca26c28f3932a1f3cb84baa7bb2f0f4bdc4856b9881053ae4465 + image: solsson/kafka-ksql@sha256:cea326dd042c4451d0c503d1ee2666d1fa893008acf20b9fb2bc0b2327fc23fb readinessProbe: httpGet: path: / From 2c5e9e47fe70e9db838eeb5561d36bc3dc1a3594 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 3 Oct 2017 08:26:17 +0200 Subject: [PATCH 3/5] Adds a test case stub to test CLI from locally built image --- ksql/test/ksql-basic-tail.yml | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 ksql/test/ksql-basic-tail.yml diff --git a/ksql/test/ksql-basic-tail.yml b/ksql/test/ksql-basic-tail.yml new file mode 100644 index 00000000..bce4f180 --- /dev/null +++ b/ksql/test/ksql-basic-tail.yml @@ -0,0 +1,73 @@ +--- +kind: ConfigMap +metadata: + name: ksql-basic-tail + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + # do nothing for now + tail -f /dev/null + + tail -f /tmp/testlog + + continue.sh: |- + exit 0 + + run.sh: |- + exit 0 + +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: ksql-basic-tail + namespace: test-kafka +spec: + replicas: 1 + template: + metadata: + labels: + test-target: kafka + test-type: readiness + spec: + containers: + - name: testcase + image: ksql-build + imagePullPolicy: Never + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092 + - name: ZOOKEEPER + value: zookeeper.kafka.svc.cluster.local:2181 + # Test set up + command: + - /bin/bash + - -e + - /test/setup.sh + # Test run, again and again + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/run.sh + # JVM start is slow, can we keep producer started and restore the default preriod 10s? + periodSeconds: 30 + # Test quit on nonzero exit + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/continue.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: ksql-basic-tail From f35ddff17a455542168bbd40f9766b3c29aff4f3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 3 Oct 2017 08:49:12 +0200 Subject: [PATCH 4/5] A prompt that connects to the rest-app cluster --- ksql/test/ksql-basic-tail.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ksql/test/ksql-basic-tail.yml b/ksql/test/ksql-basic-tail.yml index bce4f180..862a0238 100644 --- a/ksql/test/ksql-basic-tail.yml +++ b/ksql/test/ksql-basic-tail.yml @@ -9,8 +9,8 @@ data: setup.sh: |- touch /tmp/testlog - # do nothing for now - tail -f /dev/null + echo "Nothing happens in this test yet. To get a KSQL prompt run something like:" >> /tmp/testlog + echo "kubectl -n test-kafka exec -ti $(hostname) -- ksql-cli remote http://ksql.kafka" >> /tmp/testlog tail -f /tmp/testlog From 7b4f7b58f0258ff0774909507e0e1ad8d6fe7e60 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 3 Oct 2017 09:59:25 +0200 Subject: [PATCH 5/5] Image with ksql-cli, but without ... the examples standalone jar that can generate quickstart data --- ksql/ksql.yml | 4 ++-- ksql/test/ksql-basic-tail.yml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ksql/ksql.yml b/ksql/ksql.yml index 7de6d239..f4fd3a0c 100644 --- a/ksql/ksql.yml +++ b/ksql/ksql.yml @@ -16,8 +16,8 @@ spec: app: ksql spec: containers: - - name: cp - image: solsson/kafka-ksql@sha256:cea326dd042c4451d0c503d1ee2666d1fa893008acf20b9fb2bc0b2327fc23fb + - name: ksql + image: solsson/kafka-ksql@sha256:0ecd9e9042f422ac5856a306c7ada0c52046cbe9e0612cba6ed75b6260f2c101 readinessProbe: httpGet: path: / diff --git a/ksql/test/ksql-basic-tail.yml b/ksql/test/ksql-basic-tail.yml index 862a0238..ea33b44b 100644 --- a/ksql/test/ksql-basic-tail.yml +++ b/ksql/test/ksql-basic-tail.yml @@ -36,8 +36,7 @@ spec: spec: containers: - name: testcase - image: ksql-build - imagePullPolicy: Never + image: solsson/kafka-ksql@sha256:0ecd9e9042f422ac5856a306c7ada0c52046cbe9e0612cba6ed75b6260f2c101 env: - name: BOOTSTRAP value: kafka-0.broker.kafka.svc.cluster.local:9092