-
-
Notifications
You must be signed in to change notification settings - Fork 9
/
logs_pipeline.tf
404 lines (377 loc) · 14.6 KB
/
logs_pipeline.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
################################################################################################################################################
# Datadog Logs Pipeline
# - Reminder there is only ONE 'datadog_logs_pipeline_order' resource for the whole tenant
# - The 'datadog_logs_integration_pipeline' resource are sparse: it's only imported empty shells
# - For custom logs pipelines parsers, it is recommended to use the UI to comose it (with samples, assistants, etc.) and add the pipeline here
# as code once finalized to go through review and audit
################################################################################################################################################
resource "datadog_logs_pipeline_order" "custom_order" {
name = "custom_order"
# TODO: define each of these pipeline as code here
pipelines = [
datadog_logs_integration_pipeline.glog.id,
datadog_logs_integration_pipeline.core_dns.id,
datadog_logs_integration_pipeline.datadog_agent.id,
datadog_logs_integration_pipeline.nginx.id,
datadog_logs_integration_pipeline.httpd.id,
datadog_logs_integration_pipeline.mysql.id,
datadog_logs_integration_pipeline.kube_cluster_autoscaler.id,
datadog_logs_integration_pipeline.docker.id,
datadog_logs_integration_pipeline.apache.id, # Yes, there is "Apache" and "Apache HTTPD" in the default Datadog integrations.
datadog_logs_integration_pipeline.redis.id,
datadog_logs_integration_pipeline.ruby.id,
datadog_logs_custom_pipeline.nginx_artifact_caching_proxy.id,
datadog_logs_custom_pipeline.all_jenkins_infra_logs.id,
datadog_logs_custom_pipeline.mirrorbits_logs.id,
]
}
resource "datadog_logs_integration_pipeline" "glog" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "core_dns" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "datadog_agent" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "nginx" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "httpd" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "mysql" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "kube_cluster_autoscaler" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "docker" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "apache" {
is_enabled = true
}
resource "datadog_logs_integration_pipeline" "redis" {
is_enabled = true
}
import {
to = datadog_logs_integration_pipeline.ruby
id = "Nx_rG0R4Sx6o97dFkOHMsQ"
}
resource "datadog_logs_integration_pipeline" "ruby" {
is_enabled = true
}
## TODO: describe the intent of this pipeline (remap status? Check request caching status? Other?)
resource "datadog_logs_custom_pipeline" "nginx_artifact_caching_proxy" {
filter {
query = "source:nginx service:artifact-caching-proxy"
}
name = "Nginx - artifact-caching-proxy-view"
is_enabled = true
processor {
grok_parser {
is_enabled = true
name = "Parsing Nginx logs + ACP cache (hlm)"
samples = [
"10.0.30.36 - - [16/May/2023:14:34:24 +0000] \"GET /public/org/jenkins-ci/main/jenkins-war/2.404/jenkins-war-2.404.war HTTP/1.1\" 200 94777217 20.587 \"-\" \"Apache-Maven/3.9.1 (Java 17.0.7; Linux 5.10.176-157.645.amzn2.x86_64)\" \"10.0.0.20\" \"-\" \"-\" \"-\"",
"2017/09/26 14:36:50 [error] 8409#8409: *317058 \"/usr/share/nginx/html/sql/sql-admin/index.html\" is not found (2: No such file or directory), client: 217.92.148.44, server: localhost, request: \"HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1\", host: \"174.138.82.103\"",
"10.244.2.36 - - [22/May/2023:12:55:57 +0000] \"GET /public/org/apache/maven/maven-compat/3.0/maven-compat-3.0.jar.sha1 HTTP/1.1\" 200 40 1.200 \"-\" \"Apache-Maven/3.9.1 (Java 17.0.7; Windows Server 2019 10.0)\" \"20.242.86.255\" \"-\" \"-\" \"-\"",
"10.244.0.124 - - [22/May/2023:13:09:40 +0000] \"GET /incrementals/org/jenkins-ci/main/jenkins-bom/2.361/jenkins-bom-2.361.pom HTTP/1.1\" 404 91 0.858 \"-\" \"Apache-Maven/3.9.1 (Java 11.0.19; Linux 5.10.0-0.deb10.17-amd64)\" \"164.90.211.131\" \"34.235.175.210:443\" \"404\" \"0.401\"",
"10.244.2.24 - - [15/May/2023:23:17:18 +0000] \"GET /public/org/apache/maven/plugins/maven-install-plugin/maven-metadata.xml HTTP/1.1\" 200 781 31.540 \"-\" \"Apache-Maven/3.9.1 (Java 11.0.19; Linux 5.15.0-1037-azure)\" \"172.177.134.158\" \"3.93.92.223:443\" \"200\" \"31.541\"",
]
source = "message"
grok {
match_rules = <<-EOT
access.common %%{_client_ip} %%{_ident} %%{_auth} \[%%{_date_access}\] "(?>%%{_method} |)%%{_url}(?> %%{_version}|)" %%{_status_code} (?>%%{_bytes_written}|-)
access.combined %%{access.common} (%%{_request_time} )?"%%{_referer}" "%%{_user_agent}" "%%{_x_forwarded_for}" (%%{_upstream_cached}|%%{_upstream_not_cached}).*
error.format %%{date("yyyy/MM/dd HH:mm:ss"):date_access} \[%%{word:level}\] %%{data:error.message}(, %%{data::keyvalue(": ",",")})?
EOT
support_rules = <<-EOT
_auth %%{notSpace:http.auth:nullIf("-")}
_bytes_written %%{integer:network.bytes_written}
_bytes_written_root %%{integer:network_bytes_written}
_client_ip %%{ipOrHost:network.client.ip}
_version HTTP\/%%{regex("\\d+\\.\\d+"):http.version}
_url %%{notSpace:http.url}
_ident %%{notSpace:http.ident:nullIf("-")}
_user_agent %%{regex("[^\\\"]*"):http.useragent}
_referer %%{notSpace:http.referer}
_status_code %%{integer:http.status_code}
_method %%{word:http.method}
_date_access %%{date("dd/MMM/yyyy:HH:mm:ss Z"):date_access}
_x_forwarded_for %%{regex("[^\\\"]*"):http._x_forwarded_for:nullIf("-")}
_request_time %%{number:duration:scale(1000000000)}
_upstream_addr %%{notSpace:cache.upstream_addr}
_upstream_addr_empty %%{regex("-"):cache.upstream_addr_empty}
_upstream_status %%{notSpace:cache.upstream_status}
_upstream_status_empty %%{regex("-"):cache.upstream_status_empty}
_upstream_response_time %%{notSpace:cache.upstream_response_time}
_upstream_cached "%%{regex("-")}" "%%{regex("-")}" "%%{boolean("-",""):cache.upstream_cached}
_upstream_not_cached "%%{notSpace:cache.upstream_addr}" "%%{number:cache.upstream_status}" "%%{number:cache.upstream_response_time}"
EOT
}
}
}
processor {
attribute_remapper {
is_enabled = true
name = "Map `client` to `network.client.ip`"
override_on_conflict = false
preserve_source = false
source_type = "attribute"
sources = [
"client",
]
target = "network.client.ip"
target_format = null
target_type = "attribute"
}
}
processor {
grok_parser {
is_enabled = true
name = "Parsing Nginx Error log requests"
samples = [
"HEAD http://174.138.82.103:80/sql/sql-admin/ HTTP/1.1",
]
source = "request"
grok {
match_rules = <<-EOT
request_parsing (?>%%{_method} |)%%{_url}(?> %%{_version}|)
EOT
support_rules = <<-EOT
_method %%{word:http.method}
_url %%{notSpace:http.url}
_version HTTP\/%%{regex("\\d+\\.\\d+"):http.version}
EOT
}
}
}
processor {
url_parser {
is_enabled = true
name = null
normalize_ending_slashes = false
sources = [
"http.url",
]
target = "http.url_details"
}
}
processor {
user_agent_parser {
is_enabled = true
is_encoded = false
name = null
sources = [
"http.useragent",
]
target = "http.useragent_details"
}
}
processor {
date_remapper {
is_enabled = true
name = "Define `date_access` as the official date of the log"
sources = [
"date_access",
]
}
}
processor {
category_processor {
is_enabled = true
name = "Categorise status code"
target = "http.status_category"
category {
name = "OK"
filter {
query = "@http.status_code:[200 TO 299]"
}
}
category {
name = "notice"
filter {
query = "@http.status_code:[300 TO 399]"
}
}
category {
name = "warning"
filter {
query = "@http.status_code:[400 TO 499]"
}
}
category {
name = "error"
filter {
query = "@http.status_code:[500 TO 599]"
}
}
}
}
processor {
status_remapper {
is_enabled = true
name = "Define `http.status_category`, `level` as the official status of the log"
sources = [
"http.status_category",
"level",
]
}
}
processor {
string_builder_processor {
is_enabled = false
is_replace_missing = true
name = "cached_if_empty: %%{cache.upstream_cached} - in attribute cache.resultCached"
target = "cache.resultCached"
template = "cached_if_empty: %%{cache.upstream_cached}"
}
}
processor {
attribute_remapper {
is_enabled = true
name = "cachedProcessor"
override_on_conflict = false
preserve_source = true
source_type = "attribute"
sources = [
"cache.resultCached",
]
target = "cached"
target_format = null
target_type = "attribute"
}
}
processor {
attribute_remapper {
is_enabled = true
name = "writtenBytesProcessor"
override_on_conflict = false
preserve_source = true
source_type = "attribute"
sources = [
"network.bytes_written",
]
target = "written_bytes"
target_format = "integer"
target_type = "attribute"
}
}
processor {
category_processor {
is_enabled = true
name = "category cached cat processor"
target = "cache.cat_chached"
category {
name = "cached"
filter {
query = "@cache.upstream_cached:true"
}
}
category {
name = "notcached"
filter {
query = "[email protected]_cached:true"
}
}
}
}
}
## Remap statuses of all logs from jenkins-infra
# TODO: Could be worth remapping per application: accountapp, Jenkins controllers, etc.
resource "datadog_logs_custom_pipeline" "all_jenkins_infra_logs" {
filter {
query = "source:*"
}
name = "All logs"
is_enabled = true
processor {
grok_parser {
is_enabled = true
name = "Status Level Remapper"
samples = [
"WARNING: Looking up parameter names for public org.kohsuke.stapler.HttpResponse org.jenkinsci.account.AdminUI.doSearch(java.lang.String) throws javax.naming.NamingException; update plugin to a version created with a newer harness",
"Aug 17, 2024 2:36:18 PM org.kohsuke.stapler.BytecodeReadingParanamer lookupParameterNames",
"2024-08-16 23:14:56.467:WARN :oejshC.ROOT_war:Handling GET /doSignup : qtp74885833-24: Error while serving http://accounts.jenkins.io/doSignup",
"INFO: Both error and output logs will be printed to /home/jenkins/agent/remoting",
"2024-08-17 15:26:05.418+0000 [id=31136]\tWARNING\to.d.j.p.d.t.DatadogWebhookBuildLogic#toJson: Couldn't find a valid commit for pipelineID 'jenkins-Infra-plugin-site-api-generate-data-315256'. GIT_COMMIT environment variable was not found or has invalid SHA1 string:",
]
source = "message"
grok {
match_rules = <<-EOT
ParsingAccountAppLogForStatus %%{date("yyyy-MM-dd HH:mm:ss.SSS"):date}(.)%%{word:log_status}(.*)
ParsingJenkinsLogForStatus %%{date("yyyy-MM-dd HH:mm:ss.SSSZ"):date} \[id=%%{integerStr:jenkinsCorrelationId}\].%%{word:log_status}.*
ParsingFirstWordForStatus %%{word:log_status}: .*
EOT
support_rules = ""
}
}
}
processor {
status_remapper {
is_enabled = true
name = "CustomStatusRemapper"
sources = [
"log_status",
]
}
}
}
## Remap statuses of all logs from mirrorbits services
resource "datadog_logs_custom_pipeline" "mirrorbits_logs" {
filter {
query = "source:mirrorbits"
}
name = "Mirrorbits logs"
is_enabled = true
processor {
grok_parser {
is_enabled = true
name = "Log Level Status Parser"
samples = [
"2024/08/17 17:37:43.909 UTC /download/plugins/snakeyaml-api/index.html: MD5 d499f56fc98d7a5fe1e9e3ecce8b4c4f",
"2024/08/17 18:15:56.500 UTC -> Node updates-jenkins-io-mirrorbits-7c6bd54c84-gbxsh-27545 joined the cluster",
"2024/08/17 17:53:43.921 UTC HTTP error: write tcp 10.100.4.4:8080->10.100.4.1:36706: write: broken pipe",
"2024/08/17 18:16:16.409999 REDIRECT 302 \"/current/update-center.actual.json\" ip:89.84.210.161 mirror:westeurope asn:13335 distance:394.04km countries:FR",
"2024/08/17 18:15:56.839 UTC eastamerica Up! (305ms)",
]
source = "message"
grok {
support_rules = ""
# IMPORTANT: Escape '%%{}' datadog syntax as it is also Terraform template directive
# https://developer.hashicorp.com/terraform/language/expressions/strings#escape-sequences-1
match_rules = <<-EOT
parseMirrorbitsAccessLog %%{date("yyyy/MM/dd HH:mm:ss.SSSSSS"):date}\s+%%{word:request_result}\s+%%{integer:http_status}\s+"%%{notSpace:http_uri}"\s+%%{data::keyvalue(":")}
parseMirrorbitsScanLog %%{date("yyyy/MM/dd HH:mm:ss.SSS z"):date}\s+%%{notSpace:http_uri}:\s+%%{notSpace:hash_type}\s+%%{notSpace:hash_value}
parseMirrorbitsErrorLog %%{date("yyyy/MM/dd HH:mm:ss.SSS z"):date}\s+HTTP\s+%%{notSpace:log_status:uppercase}:%%{data:error_message}
parseMirrorbitHealthCheck %%{date("yyyy/MM/dd HH:mm:ss.SSS z"):date}\s+%%{notSpace:mirror_id}\s+%%{notSpace:mirror_status}!\s+\(%%{notSpace:mirror_latency}\)
parseMirrorNodeEvent %%{date("yyyy/MM/dd HH:mm:ss.SSS z"):date} -> Node %%{notSpace:node_id}\s+%%{data:node_event}
EOT
}
}
}
processor {
category_processor {
is_enabled = true
name = "Define Status"
target = "log_level"
category {
name = "INFO"
filter {
query = "@mirror_status:Up OR @request_result:REDIRECT OR @hash_type:*"
}
}
}
}
processor {
status_remapper {
is_enabled = true
name = "Status Remapper: Custom Status"
sources = [
"log_level",
]
}
}
}