Add system tests to logstash package (#4443)

* Add system tests for logstash package (cherry picked from commit 4df6b991a6db0cb851598c34063363d90f1c0749) * Update README * Test .env file * Set default env var value * Fix alias mapping * Fix mapping and use Hostname var on test config * Add missing filed to node_stats mapping * Fix docker-compose file * Fix test config and replaces .env with variantes.yml
elastic · Nov 1, 2022 · 094a533 · 094a533
1 parent c663da0
commit 094a533
Show file tree

Hide file tree

Showing 16 changed files with 868 additions and 504 deletions.
diff --git a/packages/logstash/_dev/deploy/docker/.env b/packages/logstash/_dev/deploy/docker/.env
diff --git a/packages/logstash/_dev/deploy/docker/docker-compose.yml b/packages/logstash/_dev/deploy/docker/docker-compose.yml
@@ -1,10 +1,11 @@
-version: '2.3'
+version: "2.3"
 services:
   logstash:
-    image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION}"
+    user: root
+    image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.5.0-SNAPSHOT}"
     volumes:
       - "./pipeline:/usr/share/logstash/pipeline"
       - "./config:/usr/share/logstash/config"
-      - ${SERVICE_LOGS_DIR}/logstash:/usr/share/logstash/logs
+      - ${SERVICE_LOGS_DIR}:/usr/share/logstash/logs
     ports:
       - "127.0.0.1:9600:9600"
diff --git a/packages/logstash/_dev/deploy/variants.yml b/packages/logstash/_dev/deploy/variants.yml
@@ -0,0 +1,4 @@
+variants:
+  logstash_8.5.0:
+    ELASTIC_VERSION: 8.5.0-SNAPSHOT
+default: logstash_8.5.0
diff --git a/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/log/_dev/test/system/test-default-config.yml
@@ -0,0 +1,5 @@
+input: logfile
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/logstash-json.log"
diff --git a/packages/logstash/data_stream/log/sample_event.json b/packages/logstash/data_stream/log/sample_event.json
@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2022-09-13T19:08:48.030Z",
+    "@timestamp": "2022-10-11T14:03:32.641Z",
     "agent": {
-        "ephemeral_id": "c028b260-d373-4eab-978c-08bc42b9dc7c",
-        "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0",
+        "ephemeral_id": "e1088945-f8e4-466b-83fd-f636ffbf9bdf",
+        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.5.0"
@@ -16,33 +16,34 @@
         "version": "1.10.0"
     },
     "elastic_agent": {
-        "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0",
+        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
         "snapshot": true,
         "version": "8.5.0"
     },
     "event": {
         "agent_id_status": "verified",
+        "created": "2022-10-11T14:03:32.641Z",
         "dataset": "logstash.log",
-        "ingested": "2022-09-13T19:08:58Z",
+        "ingested": "2022-10-11T14:03:44Z",
         "kind": "event",
         "type": "info"
     },
     "host": {
         "architecture": "x86_64",
-        "containerized": true,
+        "containerized": false,
         "hostname": "docker-fleet-agent",
-        "id": "8127511256f0493fa1abf625ca3e0609",
+        "id": "b6bc6723e51b43959ce07f0c3105c72d",
         "ip": [
-            "172.20.0.8"
+            "192.168.0.7"
         ],
         "mac": [
-            "02-42-AC-14-00-08"
+            "02-42-C0-A8-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.47-linuxkit",
+            "kernel": "5.10.124-linuxkit",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",

diff --git a/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml b/packages/logstash/data_stream/node/_dev/test/system/test-default-config.yml
@@ -0,0 +1,6 @@
+type: logstash/metrics
+dataset: logstash.stack_monitoring.node
+vars:
+  hosts:
+    - "http://{{Hostname}}:9600"
+data_stream: ~
diff --git a/packages/logstash/data_stream/node/fields/fields.yml b/packages/logstash/data_stream/node/fields/fields.yml
@@ -1,6 +1,8 @@
 - name: logstash
   type: group
   fields:
+    - name: cluster.id
+      type: keyword
     - name: elasticsearch.cluster.id
       type: keyword
     - name: node

diff --git a/packages/logstash/data_stream/node/fields/package-fields.yml b/packages/logstash/data_stream/node/fields/package-fields.yml
@@ -3,91 +3,7 @@
   path: logstash.elasticsearch.cluster.id
 - name: timestamp
   type: alias
-  path: '@timestamp'
-- name: logstash_stats
-  type: group
-  fields:
-    - name: timestamp
-      type: alias
-      path: '@timestamp'
-    - name: jvm
-      type: group
-      fields:
-        - name: mem
-          type: group
-          fields:
-            - name: heap_used_in_bytes
-              type: alias
-              path: logstash.node.stats.jvm.mem.heap_used_in_bytes
-            - name: heap_max_in_bytes
-              type: alias
-              path: logstash.node.stats.jvm.mem.heap_max_in_bytes
-        - name: uptime_in_millis
-          type: alias
-          path: logstash.node.stats.jvm.uptime_in_millis
-    - name: events
-      type: group
-      fields:
-        - name: in
-          type: alias
-          path: logstash.node.stats.events.in
-        - name: out
-          type: alias
-          path: logstash.node.stats.events.out
-        - name: duration_in_millis
-          type: alias
-          path: logstash.node.stats.events.duration_in_millis
-    - name: logstash
-      type: group
-      fields:
-        - name: uuid
-          type: alias
-          path: logstash.node.stats.logstash.uuid
-        - name: version
-          type: alias
-          path: logstash.node.stats.logstash.version
-    - name: os
-      type: group
-      fields:
-        - name: cpu
-          type: group
-          fields:
-            - name: stat
-              type: group
-              fields:
-                - name: number_of_elapsed_periods
-                  type: alias
-                  path: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods
-                - name: time_throttled_nanos
-                  type: alias
-                  path: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos
-                - name: number_of_times_throttled
-                  type: alias
-                  path: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled
-            - name: load_average
-              type: group
-              fields:
-                - name: 15m
-                  type: alias
-                  path: logstash.node.stats.os.cpu.load_average.15m
-                - name: 1m
-                  type: alias
-                  path: logstash.node.stats.os.cpu.load_average.1m
-                - name: 5m
-                  type: alias
-                  path: logstash.node.stats.os.cpu.load_average.5m
-        - name: cgroup
-          type: group
-          fields:
-            - name: cpuacct.usage_nanos
-              type: alias
-              path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos
-    - name: process.cpu.percent
-      type: alias
-      path: logstash.node.stats.process.cpu.percent
-    - name: queue.events_count
-      type: alias
-      path: logstash.node.stats.queue.events_count
+  path: "@timestamp"
 - name: logstash_state
   type: group
   fields: