forked from newrelic/nri-kafka
-
Notifications
You must be signed in to change notification settings - Fork 0
/
kafka-config.yml.sample
130 lines (116 loc) · 8.61 KB
/
kafka-config.yml.sample
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
integration_name: com.newrelic.kafka
instances:
- name: kafka-metrics
command: metrics
arguments:
# A cluster name is required to uniquely identify this collection result in Insights
cluster_name: "testcluster1"
# In order to collect broker and topic metrics a Zookeeper connection needs to be specified.
# The "zookeeper_hosts" field is a JSON array, each entry in the array connection information for a Zookeeper
# node.
# Each entry should have the following fields:
# - host: The IP or Hostname of a Zookeeper node, if the New Relic agent is installed on a Zookeeper node "localhost" is an acceptable value
# - port: The port Zookeeper is listening on for incoming requests. If omitted, a default port of 2181 will be used.
# Example: '[{"host": "zookeeper.my.localnet", "port": 2181}]'
#
# Note: If only collecting producers/consumers and "topic_mode" is set to "List" all Zookeeper fields can be omitted.
zookeeper_hosts: <JSON Array of Zookeeper Hosts of the form '[{"host": "localhost", "port": 2181}]'>
# Authentication type for zookeeper. Currently only supports auth scheme "user". Defaults to no authentication.
zookeeper_auth_scheme: <Auth scheme for Zookeeper>
# If using "user" authentication, the credentials must be specified as a string of the form "<user>:<password>"
# Example: 'zookeeperuser:zookeeperpass'
zookeeper_auth_secret: <Auth string for Zookeeper>
# If the Kafka configuration files are not in the root node of Zookeeper, an alternative root can be specified.
# The alternative root must have a leading slash.
# Example: '/kafka-root'
zookeeper_path: <Root path of Kafka nodes in Zookeeper>
# It is common to use the same JMX configuration across a Kafka cluster
# The default username and password are the credentials that will be used to make
# a JMX connection to each broker found by Zookeeper. Theses values will also
# be used when connecting to a consumer and/or producer if the "username" or "password"
# field are omitted.
default_jmx_user: <Default User name for JMX Connection>
default_jmx_password: <Default Password for JMX Connection>
# In order to collect Java producer and consumer metrics the "producers" and "consumers" fields should be filled out.
# Both fields are JSON arrays with each entry being a separate JAVA producer or consumer, in it's respective field.
# Each entry should have the following fields:
# - name: This is the actual name of the producer/consumer as it appears in Kafka
# - host: The IP or Hostname of the producer/consumser. If omitted, will use the value of the "default_jmx_host" field
# - port: The port in which JMX is setup for on the producer/consumer. If omitted will, use the value of the "default_jmx_port" field
# - username: The username used to connect to JMX. If omitted, will use the value of the "default_jmx_user" field
# - password: The password used to connect to JMX. If omitted, will use the value of the "default_jmx_password" field
# Example: {"name": "myProducer", "host": "localhost", "port": 24, "username": "me', "password": "secret"}
producers: <JSON Array of Producers of the structure [{"name": "myProducer", "host": "localhost", "port": 24, "username": "me', "password": "secret"}]
consumers: <JSON Array of Consumers of the structure [{"name": "myConsumer", "host": "localhost", "port": 24, "username": "me', "password": "secret"}]
# If several producers/consumers are on the same host an agent can be installed on that host and the
# "default_jmx_host" and "default_jmx_port" field can be set once and used for all producers/consumers that
# do not have the "host" or "port" field repsectively.
# These fields can be removed if each producer/consumer has it's own "host" and/or "port" field filled out.
default_jmx_host: <Default Host to collect JMX Metrics>
default_jmx_port: <Default Port to collect JMX Metrics>
# This field is used to toggle the collection of broker and topic metrics. This is on by default and should only be set to "false"
# for the specific case where only producers/consumers are being monitored and "topic_mode" is set to "All".
# Any other case this field can be omitted.
collect_broker_topic_data: <true or false>
# Below are the fields used to fine tune/toggle topic metric collection.
# In order to collect topics the "topic_mode" field must be set to "all" or "list". If the field is set to "all"
# a Zookeeper connection is required, at least the "zookeeper_hosts" field is required, as topics are looked up via Zookeeper.
#
# It is recommended to use the "List" option to monitor a specific set of topics. If using "List" mode the "topic_list"
# field should be filled out. The "topic_list" is a JSON array of topic names to be monitored.
# Example of topic_list: '["topic1", "topic2"]'
#
# If monitoring topics via the "all" or "list" option for "topic_mode", the topic size can be collected from zookeeper by setting
# "collect_topic_size" to true. This operation is intensive and can take a while to collect for a larger number of topics.
# It is recommended to only enable this feature if using a small "topic_list".
# If the field is omitted it will default to false.
topic_mode: <all, none, list or regex. All mode requires a zookeeper_host to be specified>
topic_list: <JSON Array of Topics to monitor. Ignored if topic_mode is not list>
topic_regex: <Regex pattern that matches the topics to be collected. Ignored if topic_mode is not regex>
collect_topic_size: <true or false. Indicate if topic size should be collected as it is a very resource intensive metric to collect>
labels:
env: production
role: kafka
- name: kafka-inventory
command: inventory
arguments:
cluster_name: "testcluster2"
# More information about the zookeeper connection can be found above
zookeeper_hosts: <JSON Array of Zookeeper Hosts of the form '[{"host": "localhost", "port": 2181}]'>
zookeeper_auth_scheme: <Auth scheme for Zookeeper>
zookeeper_auth_secret: <Auth string for Zookeeper>
zookeeper_path: <Root path of Kafka nodes in Zookeeper>
# Below are the fields used to fine tune/toggle topic inventory collection.
# In order to collect topics the "topic_mode" field must be set to "All" or "List". If the field is set to "All"
# a Zookeeper connection is required, at least the "zookeeper_hosts" field is required, as topics are looked up via Zookeeper.
#
# It is recommended to use the "List" option to monitor a specific set of topics. If using "List" mode the "topic_list"
# field should be filled out. The "topic_list" is a JSON array of topic names to be monitored.
topic_mode: <All, None, or List. All mode requires a zookeeper_host to be specified>
topic_list: <JSON Array of Topics to monitor. Ignored if topic_mode is not List>
labels:
env: production
role: kafka
- name: kafka-consumer-offsets
command: consumer_offset
arguments:
cluster_name: "testcluster3"
# More information about the zookeeper connection can be found above
zookeeper_hosts: <JSON Array of Zookeeper Hosts of the form '[{"host": "localhost", "port": 2181}]'>
zookeeper_auth_scheme: <Auth scheme for Zookeeper>
zookeeper_auth_secret: <Auth string for Zookeeper>
zookeeper_path: <Root path of Kafka nodes in Zookeeper>
# Whitelist of consumer groups to collect offset data for. This feature is only avaiable in Kafka 0.8.2.2 and above.
# The value should be a JSON object with the follow structure:
# Consumer Group Name mapped to a map of topic names.
# Those topics then map to a JSON array of partition numbers
#
# Say we wanted to monitor two consumer groups "consumer_group_1" and "consumer_group_2".
# "consumer_group_1" we want to monitor "topic_1" with partitions 1, 2, and 3. We also want to monitor
# "topic_2" within that same group, but want to monitor all partitions within that topic, so we'll give it a blank
# set of partitions. Below is an example of the value for "consumer_group" field to achieve our desired configuration.
# '{"consumer_group_1": {"topic_1": [1,2,3], "topic_2":[]}}'
consumer_groups: <JSON Object whitelist of consumer groups to their topics and topics to their partitions, in which to collect consumer offsets for. Example form {"group_1":{"topic_1":[1,2]}}>
labels:
env: production
role: kafka