Support list of dynamic counter name. (#4637)

* Support list of dynamic counter name.
elastic · Nov 15, 2022 · 0aa29ac · 0aa29ac
1 parent 9f37b3b
commit 0aa29ac
Show file tree

Hide file tree

Showing 9 changed files with 120 additions and 55 deletions.
diff --git a/packages/microsoft_sqlserver/_dev/build/docs/README.md b/packages/microsoft_sqlserver/_dev/build/docs/README.md
@@ -57,7 +57,12 @@ See: [View the SQL Server error log in SQL Server Management Studio](https://doc
 
 ### performance metrics
 
-Collects the `performance` counter metrics. Dynamic counter feature provides flexibility to collect metrics by providing the counter name as an input.
+Collects the `performance` counter metrics. Dynamic counter feature provides flexibility to collect metrics by providing the counter as an input.
+This input can be a regular expression which will filter results based on pattern.
+For example, if %grant% is given as input, it will enable metrics collection for all of the counters with name like 'Memory Grants Pending', 'Active memory grants count' etc.
+MSSQL supports limited set of RegExp, See [here] (https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms187489(v=sql.105)?redirectedfrom=MSDN) for details.
+
+>Note: Dynamic counters will go through some basic ingest pipeline post-processing to make counter names in lower case and remove special characters and these fields will not have any static field mappings.
 
 The feature `merge_results` has been introduced in 8.4 beats which create a single event by combining the metrics together in a single event. See [here](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-sql.html#_example_merge_multiple_queries_to_single_event) for details.
 

diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.0"
+  changes:
+    - description: Support list of dynamic counter name.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4637
 - version: "1.8.0"
   changes:
     - description: Added infrastructure category.

diff --git a/packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs b/packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs
@@ -6,7 +6,7 @@ period: {{period}}
 raw_data.enabled: true
 merge_results: true
 # Below dynamic_counter_name handles the dynamic counter name passing it to SQL query
-dynamic_counter_name: {{dynamic_counter_name}}
+dynamic_counter_name: "{{dynamic_counter_name}}"
 driver: "mssql"
 sql_queries:
   - query: "SELECT cntr_value As 'user_connections' FROM sys.dm_os_performance_counters WHERE counter_name= 'User Connections'"
@@ -41,5 +41,5 @@ sql_queries:
     response_format: table
   - query: "SELECT cntr_value As 're_compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Re-Compilations/sec'"
     response_format: table
-  - query: "SELECT counter_name As 'dynamic_counter.name', cntr_value As 'dynamic_counter.value' FROM sys.dm_os_performance_counters WHERE counter_name= '{{dynamic_counter_name}}'"
-    response_format: table
+  - query: "SELECT counter_name, cntr_value FROM sys.dm_os_performance_counters WHERE counter_name like '{{dynamic_counter_name}}'"
+    response_format: variables
diff --git a/...ges/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml b/...ges/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml
@@ -14,6 +14,66 @@ processors:
     target_field: mssql
     ignore_missing: true
     ignore_failure: true
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    processor:
+        trim:
+            field: "_ingest._key"
+            ignore_missing: true
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    processor:
+        gsub:
+            field: "_ingest._key"
+            pattern: " "
+            replacement: "_"
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    ignore_failure: true
+    processor:
+        gsub:
+            field: "_ingest._key"
+            pattern: "/"
+            replacement: "_"
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    ignore_failure: true
+    processor:
+        gsub:
+            field: "_ingest._key"
+            pattern: ">"
+            replacement: "_"
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    ignore_failure: true
+    processor:
+        gsub:
+            field: "_ingest._key"
+            pattern: "\\("
+            replacement: ""
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    ignore_failure: true
+    processor:
+        gsub:
+            field: "_ingest._key"
+            pattern: "%"
+            replacement: "pct"
+- foreach:
+    field: mssql.metrics
+    ignore_missing: true
+    ignore_failure: true
+    processor:
+        gsub:
+            field: "_ingest._key"
+            pattern: "\\)"
+            replacement: ""
 on_failure:
 - set:
     field: error.message

diff --git a/packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml b/packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml
@@ -62,9 +62,6 @@
         - name: buffer_target_pages
           type: long
           description: Ideal number of pages in the buffer pool.
-        - name: dynamic_counter.value
+        - name: memory_grants_pending
           type: long
-          description: Dynamic counter value is fetched from performance table for the dynamic counter name which is provided by user.
-        - name: dynamic_counter.name
-          type: keyword
-          description: Dynamic counter name is given by user.
+          description: This is generated from the default pattern given for Dynamic Counter Name variable. This counter tells us how many processes are waiting for the memory to be assigned to them so they can get started. 
diff --git a/packages/microsoft_sqlserver/data_stream/performance/manifest.yml b/packages/microsoft_sqlserver/data_stream/performance/manifest.yml
@@ -18,7 +18,7 @@ streams:
         multi: false
         required: false
         show_user: true
-        default: Memory Grants Pending
-        description: Collect the dynamic counter value for the provided counter name from the performance table.
+        default: 'Memory Grants Pend%'
+        description: Collect the values for dynamic counters based on the provided pattern from the performance table.
     title: Microsoft SQL Server performance metrics
     description: Collect Microsoft SQL Server performance metrics
diff --git a/packages/microsoft_sqlserver/data_stream/performance/sample_event.json b/packages/microsoft_sqlserver/data_stream/performance/sample_event.json
@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2022-11-04T12:33:59.171Z",
+    "@timestamp": "2022-11-15T10:59:06.048Z",
     "agent": {
-        "ephemeral_id": "54815e19-bc14-40d2-b1e0-c16234963b52",
-        "id": "bd4366c7-33ba-4d56-8c63-fc6a15c98c17",
+        "ephemeral_id": "c15c314e-c8ef-41f0-8697-eefd3aa8d0a1",
+        "id": "c305f9fa-07c8-41b9-b950-fa4652b743bb",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
         "version": "8.5.0"
@@ -31,15 +31,15 @@
         "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "bd4366c7-33ba-4d56-8c63-fc6a15c98c17",
+        "id": "c305f9fa-07c8-41b9-b950-fa4652b743bb",
         "snapshot": false,
         "version": "8.5.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "microsoft_sqlserver.performance",
-        "duration": 42729100,
-        "ingested": "2022-11-04T12:34:00Z",
+        "duration": 40261000,
+        "ingested": "2022-11-15T10:59:07Z",
         "module": "sql"
     },
     "host": {
@@ -48,10 +48,10 @@
         "hostname": "docker-fleet-agent",
         "id": "66392b0697b84641af8006d87aeb89f1",
         "ip": [
-            "172.18.0.7"
+            "172.31.0.7"
         ],
         "mac": [
-            "02-42-AC-12-00-07"
+            "02-42-AC-1F-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
@@ -71,20 +71,17 @@
     "mssql": {
         "metrics": {
             "active_temp_tables": 0,
-            "batch_requests_per_sec": 59,
-            "buffer_cache_hit_ratio": 990,
-            "buffer_checkpoint_pages_per_sec": 105,
+            "batch_requests_per_sec": 65,
+            "buffer_cache_hit_ratio": 74,
+            "buffer_checkpoint_pages_per_sec": 274,
             "buffer_database_pages": 2215,
-            "buffer_page_life_expectancy": 18,
+            "buffer_page_life_expectancy": 30,
             "buffer_target_pages": 2408448,
-            "compilations_per_sec": 81,
-            "dynamic_counter": {
-                "name": "Memory Grants Pending                                                                                                           ",
-                "value": 0
-            },
-            "lock_waits_per_sec": 6,
-            "logins_per_sec": 18,
-            "logouts_per_sec": 17,
+            "compilations_per_sec": 83,
+            "lock_waits_per_sec": 3,
+            "logins_per_sec": 20,
+            "logouts_per_sec": 19,
+            "memory_grants_pending": 0,
             "page_splits_per_sec": 9,
             "re_compilations_per_sec": 0,
             "transactions": 0,

diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md
@@ -57,7 +57,12 @@ See: [View the SQL Server error log in SQL Server Management Studio](https://doc
 
 ### performance metrics
 
-Collects the `performance` counter metrics. Dynamic counter feature provides flexibility to collect metrics by providing the counter name as an input.
+Collects the `performance` counter metrics. Dynamic counter feature provides flexibility to collect metrics by providing the counter as an input.
+This input can be a regular expression which will filter results based on pattern.
+For example, if %grant% is given as input, it will enable metrics collection for all of the counters with name like 'Memory Grants Pending', 'Active memory grants count' etc.
+MSSQL supports limited set of RegExp, See [here] (https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms187489(v=sql.105)?redirectedfrom=MSDN) for details.
+
+>Note: Dynamic counters will go through some basic ingest pipeline post-processing to make counter names in lower case and remove special characters and these fields will not have any static field mappings.
 
 The feature `merge_results` has been introduced in 8.4 beats which create a single event by combining the metrics together in a single event. See [here](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-sql.html#_example_merge_multiple_queries_to_single_event) for details.
 
@@ -346,10 +351,10 @@ An example event for `performance` looks as following:
 
 ```json
 {
-    "@timestamp": "2022-11-04T12:33:59.171Z",
+    "@timestamp": "2022-11-15T10:59:06.048Z",
     "agent": {
-        "ephemeral_id": "54815e19-bc14-40d2-b1e0-c16234963b52",
-        "id": "bd4366c7-33ba-4d56-8c63-fc6a15c98c17",
+        "ephemeral_id": "c15c314e-c8ef-41f0-8697-eefd3aa8d0a1",
+        "id": "c305f9fa-07c8-41b9-b950-fa4652b743bb",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
         "version": "8.5.0"
@@ -378,15 +383,15 @@ An example event for `performance` looks as following:
         "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "bd4366c7-33ba-4d56-8c63-fc6a15c98c17",
+        "id": "c305f9fa-07c8-41b9-b950-fa4652b743bb",
         "snapshot": false,
         "version": "8.5.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "microsoft_sqlserver.performance",
-        "duration": 42729100,
-        "ingested": "2022-11-04T12:34:00Z",
+        "duration": 40261000,
+        "ingested": "2022-11-15T10:59:07Z",
         "module": "sql"
     },
     "host": {
@@ -395,10 +400,10 @@ An example event for `performance` looks as following:
         "hostname": "docker-fleet-agent",
         "id": "66392b0697b84641af8006d87aeb89f1",
         "ip": [
-            "172.18.0.7"
+            "172.31.0.7"
         ],
         "mac": [
-            "02-42-AC-12-00-07"
+            "02-42-AC-1F-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
@@ -418,20 +423,17 @@ An example event for `performance` looks as following:
     "mssql": {
         "metrics": {
             "active_temp_tables": 0,
-            "batch_requests_per_sec": 59,
-            "buffer_cache_hit_ratio": 990,
-            "buffer_checkpoint_pages_per_sec": 105,
+            "batch_requests_per_sec": 65,
+            "buffer_cache_hit_ratio": 74,
+            "buffer_checkpoint_pages_per_sec": 274,
             "buffer_database_pages": 2215,
-            "buffer_page_life_expectancy": 18,
+            "buffer_page_life_expectancy": 30,
             "buffer_target_pages": 2408448,
-            "compilations_per_sec": 81,
-            "dynamic_counter": {
-                "name": "Memory Grants Pending                                                                                                           ",
-                "value": 0
-            },
-            "lock_waits_per_sec": 6,
-            "logins_per_sec": 18,
-            "logouts_per_sec": 17,
+            "compilations_per_sec": 83,
+            "lock_waits_per_sec": 3,
+            "logins_per_sec": 20,
+            "logouts_per_sec": 19,
+            "memory_grants_pending": 0,
             "page_splits_per_sec": 9,
             "re_compilations_per_sec": 0,
             "transactions": 0,
@@ -463,11 +465,10 @@ An example event for `performance` looks as following:
 | mssql.metrics.buffer_target_pages | Ideal number of pages in the buffer pool. | long |  |
 | mssql.metrics.compilations_per_sec | Number of SQL compilations per second. Indicates the number of times the compile code path is entered. Includes compiles caused by statement-level recompilations in SQL Server. After SQL Server user activity is stable, this value reaches a steady state. | float | gauge |
 | mssql.metrics.connection_reset_per_sec | Total number of logins started per second from the connection pool. | float | gauge |
-| mssql.metrics.dynamic_counter.name | Dynamic counter name is given by user. | keyword |  |
-| mssql.metrics.dynamic_counter.value | Dynamic counter value is fetched from performance table for the dynamic counter name which is provided by user. | long |  |
 | mssql.metrics.lock_waits_per_sec | Number of lock requests per second that required the caller to wait. | float | gauge |
 | mssql.metrics.logins_per_sec | Total number of logins started per second. This does not include pooled connections. | float | gauge |
 | mssql.metrics.logouts_per_sec | Total number of logout operations started per second. | float | gauge |
+| mssql.metrics.memory_grants_pending | This is generated from the default pattern given for Dynamic Counter Name variable. This counter tells us how many processes are waiting for the memory to be assigned to them so they can get started. | long |  |
 | mssql.metrics.page_splits_per_sec | Number of page splits per second that occur as the result of overflowing index pages. | float | gauge |
 | mssql.metrics.re_compilations_per_sec | Number of statement recompiles per second. Counts the number of times statement recompiles are triggered. Generally, you want the recompiles to be low. | float | gauge |
 | mssql.metrics.transactions | Total number of transactions | long |  |

diff --git a/packages/microsoft_sqlserver/manifest.yml b/packages/microsoft_sqlserver/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: microsoft_sqlserver
 title: "Microsoft SQL Server"
-version: "1.8.0"
+version: "1.9.0"
 license: basic
 description: Collect events from Microsoft SQL Server with Elastic Agent
 type: integration