From fba19e09e0b9ba1fd790b1ff0f3b1d672e3bc5ea Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 10:47:38 +0200 Subject: [PATCH 1/6] Default conf from 1.0.0-rc1, except our Server Basics and logs kafkaAppender not enabled by default --- 10broker-config.yml | 42 ++++-------------------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/10broker-config.yml b/10broker-config.yml index 7d296bf3..d02960d6 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -25,23 +25,6 @@ data: } server.properties: |- - # Licensed to the Apache Software Foundation (ASF) under one or more - # contributor license agreements. See the NOTICE file distributed with - # this work for additional information regarding copyright ownership. - # The ASF licenses this file to You under the Apache License, Version 2.0 - # (the "License"); you may not use this file except in compliance with - # the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # see kafka.server.KafkaConfig for additional details and defaults - ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. @@ -134,8 +117,8 @@ data: # The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 - # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining - # segments don't drop below log.retention.bytes. Functions independently of log.retention.hours. + # A size-based retention policy for logs. Segments are pruned from the log unless the remaining + # segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. @@ -168,23 +151,6 @@ data: group.initial.rebalance.delay.ms=0 log4j.properties: |- - # Licensed to the Apache Software Foundation (ASF) under one or more - # contributor license agreements. See the NOTICE file distributed with - # this work for additional information regarding copyright ownership. - # The ASF licenses this file to You under the Apache License, Version 2.0 - # (the "License"); you may not use this file except in compliance with - # the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # Unspecified loggers and loggers with additivity=true output to server.log and stdout - # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender @@ -256,6 +222,6 @@ data: log4j.logger.state.change.logger=TRACE, stateChangeAppender log4j.additivity.state.change.logger=false - # Change to DEBUG to enable audit log for the authorizer - log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender + # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses + log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender log4j.additivity.kafka.authorizer.logger=false From ed9fe4e90c7dab4164686e3105fe6de30901f871 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 10:54:59 +0200 Subject: [PATCH 2/6] Continues the transition to property files fixes #72 --- 10broker-config.yml | 3 +++ 50kafka.yml | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/10broker-config.yml b/10broker-config.yml index d02960d6..aef7a897 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -35,6 +35,9 @@ data: # Switch to enable topic deletion or not, default value is false delete.topic.enable=true + # Yolean/kubernetes-kafka + auto.create.topics.enable=false + ############################# Socket Server Settings ############################# # The address the socket server listens on. It will get the value returned from diff --git a/50kafka.yml b/50kafka.yml index 0889d658..3909db07 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -42,8 +42,6 @@ spec: - log.retention.hours=-1 - --override - log.dirs=/var/lib/kafka/data/topics - - --override - - auto.create.topics.enable=false resources: requests: cpu: 100m From 2b446b54f76183622db902bcdcbefa1e84360d24 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 10:56:31 +0200 Subject: [PATCH 3/6] Makes indefinite retention the default --- 10broker-config.yml | 2 +- 50kafka.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/10broker-config.yml b/10broker-config.yml index aef7a897..b1d4ef11 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -118,7 +118,7 @@ data: # from the end of the log. # The minimum age of a log file to be eligible for deletion due to age - log.retention.hours=168 + log.retention.hours=-1 # A size-based retention policy for logs. Segments are pruned from the log unless the remaining # segments drop below log.retention.bytes. Functions independently of log.retention.hours. diff --git a/50kafka.yml b/50kafka.yml index 3909db07..e28ae45d 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -39,8 +39,6 @@ spec: - --override - zookeeper.connect=zookeeper:2181 - --override - - log.retention.hours=-1 - - --override - log.dirs=/var/lib/kafka/data/topics resources: requests: From 8a01ab69c121496af811ac7e4667fe71d7829dd4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 10:57:51 +0200 Subject: [PATCH 4/6] localhost for zookeeper isn't a sensible default in k8s --- 10broker-config.yml | 2 +- 50kafka.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/10broker-config.yml b/10broker-config.yml index b1d4ef11..110aea34 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -138,7 +138,7 @@ data: # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. - zookeeper.connect=localhost:2181 + zookeeper.connect=zookeeper:2181 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=6000 diff --git a/50kafka.yml b/50kafka.yml index e28ae45d..706d6865 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -37,8 +37,6 @@ spec: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties - --override - - zookeeper.connect=zookeeper:2181 - - --override - log.dirs=/var/lib/kafka/data/topics resources: requests: From 96045b8f19c0e7c11ef2a31079198f7e102f2180 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 10:59:12 +0200 Subject: [PATCH 5/6] I'm hesitant because the manifest must duplicate this storage path --- 10broker-config.yml | 2 +- 50kafka.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/10broker-config.yml b/10broker-config.yml index 110aea34..0b785cd8 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -75,7 +75,7 @@ data: ############################# Log Basics ############################# # A comma seperated list of directories under which to store log files - log.dirs=/tmp/kafka-logs + log.dirs=/var/lib/kafka/data/topics # The default number of log partitions per topic. More partitions allow greater # parallelism for consumption, but this will also result in more files across diff --git a/50kafka.yml b/50kafka.yml index 706d6865..30275d88 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -36,8 +36,6 @@ spec: command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties - - --override - - log.dirs=/var/lib/kafka/data/topics resources: requests: cpu: 100m From cc0334c98d7668660f4a7ebdf096ee3a46cff366 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 11:02:03 +0200 Subject: [PATCH 6/6] We don't reflect the default config anymore, so use git blame --- 10broker-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/10broker-config.yml b/10broker-config.yml index 0b785cd8..b4249c87 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -35,7 +35,6 @@ data: # Switch to enable topic deletion or not, default value is false delete.topic.enable=true - # Yolean/kubernetes-kafka auto.create.topics.enable=false ############################# Socket Server Settings #############################