diff --git a/postgres/datadog_checks/postgres/metadata.py b/postgres/datadog_checks/postgres/metadata.py index fc871e5d2150a..87f8aa3da3130 100644 --- a/postgres/datadog_checks/postgres/metadata.py +++ b/postgres/datadog_checks/postgres/metadata.py @@ -1,7 +1,6 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import json import time from typing import Dict, Optional, Tuple # noqa: F401 @@ -15,6 +14,7 @@ from datadog_checks.base import is_affirmative from datadog_checks.base.utils.db.utils import DBMAsyncJob, default_json_event_encoding +from datadog_checks.base.utils.serialization import json from datadog_checks.base.utils.tracking import tracked_method # default pg_settings collection interval in seconds diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 968e1119b6e2d..ea082ceb51ba7 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -236,6 +236,22 @@ files: type: boolean example: false display_default: false + - name: collect_settings + description: Configure collection of sys.configurations. This is an alpha feature. + options: + - name: enabled + description: | + Enable collection of sys.configurations. Requires `dbm: true`. + value: + type: boolean + example: false + - name: collection_interval + description: | + Set the database settings collection interval (in seconds). Each collection involves a single query to + `sys.configurations`. + value: + type: number + example: 600 - name: query_metrics description: Configure collection of query metrics options: diff --git a/sqlserver/datadog_checks/sqlserver/config_models/instance.py b/sqlserver/datadog_checks/sqlserver/config_models/instance.py index 99e2fbab02b80..501c2705f30a6 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/instance.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/instance.py @@ -37,6 +37,15 @@ class Azure(BaseModel): fully_qualified_domain_name: Optional[str] = None +class CollectSettings(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + + class CustomQuery(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -115,6 +124,7 @@ class InstanceConfig(BaseModel): availability_group: Optional[str] = None aws: Optional[Aws] = None azure: Optional[Azure] = None + collect_settings: Optional[CollectSettings] = None command_timeout: Optional[int] = None connection_string: Optional[str] = None connector: Optional[str] = None diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index ab1d73353d207..e0571b07a2e43 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -222,6 +222,21 @@ instances: # # dbm: false + ## Configure collection of sys.configurations. This is an alpha feature. + # + # collect_settings: + + ## @param enabled - boolean - optional - default: false + ## Enable collection of sys.configurations. Requires `dbm: true`. + # + # enabled: false + + ## @param collection_interval - number - optional - default: 600 + ## Set the database settings collection interval (in seconds). Each collection involves a single query to + ## `sys.configurations`. + # + # collection_interval: 600 + ## Configure collection of query metrics # # query_metrics: diff --git a/sqlserver/datadog_checks/sqlserver/metadata.py b/sqlserver/datadog_checks/sqlserver/metadata.py new file mode 100644 index 0000000000000..4b9dc929a11a6 --- /dev/null +++ b/sqlserver/datadog_checks/sqlserver/metadata.py @@ -0,0 +1,151 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import time + +from datadog_checks.base import is_affirmative +from datadog_checks.base.utils.db.utils import ( + DBMAsyncJob, + default_json_event_encoding, +) +from datadog_checks.base.utils.serialization import json +from datadog_checks.base.utils.tracking import tracked_method + +try: + import datadog_agent +except ImportError: + from ..stubs import datadog_agent + +from datadog_checks.sqlserver.const import STATIC_INFO_ENGINE_EDITION, STATIC_INFO_VERSION + +# default settings collection interval in seconds +DEFAULT_SETTINGS_COLLECTION_INTERVAL = 600 + +SETTINGS_QUERY = """\ +SELECT {columns} FROM sys.configurations +""" + +SQL_SERVER_SETTINGS_COLUMNS = [ + "name", + "value", + "minimum", + "maximum", + "value_in_use", + "is_dynamic", + "is_advanced", +] + +# some columns use the sql_varient type, which isn't supported +# by most pyodbc drivers, instead we can cast these values to VARCHAR +SQL_COLS_CAST_TYPE = { + "minimum": "varchar(max)", + "maximum": "varchar(max)", + "value_in_use": "varchar(max)", + "value": "varchar(max)", +} + + +def agent_check_getter(self): + return self.check + + +class SqlserverMetadata(DBMAsyncJob): + """ + Collects database metadata. Supports: + 1. collection of sqlserver instance settings + """ + + def __init__(self, check): + self.check = check + # do not emit any dd.internal metrics for DBM specific check code + self.tags = [t for t in self.check.tags if not t.startswith('dd.internal')] + self.log = check.log + self.collection_interval = check.settings_config.get( + 'collection_interval', DEFAULT_SETTINGS_COLLECTION_INTERVAL + ) + + super(SqlserverMetadata, self).__init__( + check, + run_sync=is_affirmative(check.settings_config.get('run_sync', False)), + enabled=is_affirmative(check.settings_config.get('enabled', False)), + expected_db_exceptions=(), + min_collection_interval=check.min_collection_interval, + dbms="sqlserver", + rate_limit=1 / float(self.collection_interval), + job_name="database-metadata", + shutdown_callback=self._close_db_conn, + ) + self.disable_secondary_tags = is_affirmative( + check.statement_metrics_config.get('disable_secondary_tags', False) + ) + self._conn_key_prefix = "dbm-metadata-" + self._settings_query = None + self._time_since_last_settings_query = 0 + self._max_query_metrics = check.statement_metrics_config.get("max_queries", 250) + + def _close_db_conn(self): + pass + + def run_job(self): + self.report_sqlserver_metadata() + + def _get_available_settings_columns(self, cursor, all_expected_columns): + cursor.execute("select top 0 * from sys.configurations") + all_columns = {i[0] for i in cursor.description} + available_columns = [c for c in all_expected_columns if c in all_columns] + missing_columns = set(all_expected_columns) - set(available_columns) + if missing_columns: + self.log.debug( + "missing the following expected settings columns from sys.configurations: %s", missing_columns + ) + self.log.debug("found available sys.configurations columns: %s", available_columns) + return available_columns + + def _get_settings_query_cached(self, cursor): + if self._settings_query: + return self._settings_query + available_columns = self._get_available_settings_columns(cursor, SQL_SERVER_SETTINGS_COLUMNS) + formatted_columns = [] + for column in available_columns: + if column in SQL_COLS_CAST_TYPE: + formatted_columns.append(f"CAST({column} AS {SQL_COLS_CAST_TYPE[column]}) AS {column}") + else: + formatted_columns.append(column) + self._settings_query = SETTINGS_QUERY.format( + columns=', '.join(formatted_columns), + ) + return self._settings_query + + @tracked_method(agent_check_getter=agent_check_getter, track_result_length=True) + def _load_settings_rows(self, cursor): + self.log.debug("collecting sql server instance settings") + query = self._get_settings_query_cached(cursor) + self.log.debug("Running query [%s] %s", query) + cursor.execute(query) + columns = [i[0] for i in cursor.description] + # construct row dicts manually as there's no DictCursor for pyodbc + rows = [dict(zip(columns, row)) for row in cursor.fetchall()] + self.log.debug("loaded sql server settings len(rows)=%s", len(rows)) + return rows + + @tracked_method(agent_check_getter=agent_check_getter) + def report_sqlserver_metadata(self): + with self.check.connection.open_managed_default_connection(key_prefix=self._conn_key_prefix): + with self.check.connection.get_managed_cursor(key_prefix=self._conn_key_prefix) as cursor: + settings_rows = self._load_settings_rows(cursor) + event = { + "host": self.check.resolved_hostname, + "agent_version": datadog_agent.get_version(), + "dbms": "sqlserver", + "kind": "sqlserver_configs", + "collection_interval": self.collection_interval, + 'dbms_version': "{},{}".format( + self.check.static_info_cache.get(STATIC_INFO_VERSION, ""), + self.check.static_info_cache.get(STATIC_INFO_ENGINE_EDITION, ""), + ), + "tags": self.tags, + "timestamp": time.time() * 1000, + "cloud_metadata": self.check.cloud_metadata, + "metadata": settings_rows, + } + self._check.database_monitoring_metadata(json.dumps(event, default=default_json_event_encoding)) diff --git a/sqlserver/datadog_checks/sqlserver/sqlserver.py b/sqlserver/datadog_checks/sqlserver/sqlserver.py index 5022b454e75e3..054f4cb540d2c 100644 --- a/sqlserver/datadog_checks/sqlserver/sqlserver.py +++ b/sqlserver/datadog_checks/sqlserver/sqlserver.py @@ -17,6 +17,7 @@ from datadog_checks.base.utils.db.utils import resolve_db_host from datadog_checks.base.utils.serialization import json from datadog_checks.sqlserver.activity import SqlserverActivity +from datadog_checks.sqlserver.metadata import SqlserverMetadata from datadog_checks.sqlserver.statements import SqlserverStatementMetrics from datadog_checks.sqlserver.utils import Database, parse_sqlserver_major_version @@ -120,7 +121,9 @@ def __init__(self, name, init_config, instances): # DBM self.dbm_enabled = self.instance.get('dbm', False) self.statement_metrics_config = self.instance.get('query_metrics', {}) or {} + self.settings_config = self.instance.get('collect_settings', {}) or {} self.statement_metrics = SqlserverStatementMetrics(self) + self.sql_metadata = SqlserverMetadata(self) self.activity_config = self.instance.get('query_activity', {}) or {} self.activity = SqlserverActivity(self) self.cloud_metadata = {} @@ -179,6 +182,7 @@ def __init__(self, name, init_config, instances): def cancel(self): self.statement_metrics.cancel() self.activity.cancel() + self.sql_metadata.cancel() def config_checks(self): if self.autodiscovery and self.instance.get('database'): @@ -741,6 +745,7 @@ def check(self, _): if self.dbm_enabled: self.statement_metrics.run_job_loop(self.tags) self.activity.run_job_loop(self.tags) + self.sql_metadata.run_job_loop(self.tags) else: self.log.debug("Skipping check") diff --git a/sqlserver/tests/test_high_cardinality.py b/sqlserver/tests/test_high_cardinality.py index 173f8de6502e7..5f8f76916b1af 100644 --- a/sqlserver/tests/test_high_cardinality.py +++ b/sqlserver/tests/test_high_cardinality.py @@ -19,6 +19,7 @@ def dbm_instance(instance_docker): instance_docker['dbm'] = True instance_docker['query_metrics'] = {'enabled': True, 'run_sync': True} instance_docker['query_activity'] = {'enabled': True, 'run_sync': True} + instance_docker['collect_settings'] = {'enabled': False} return copy(instance_docker) diff --git a/sqlserver/tests/test_metadata.py b/sqlserver/tests/test_metadata.py new file mode 100644 index 0000000000000..abd33931deafa --- /dev/null +++ b/sqlserver/tests/test_metadata.py @@ -0,0 +1,88 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import unicode_literals + +import logging +from copy import copy + +import pytest + +from datadog_checks.sqlserver import SQLServer + +from .common import CHECK_NAME + +try: + import pyodbc +except ImportError: + pyodbc = None + + +@pytest.fixture +def dbm_instance(instance_docker): + instance_docker['dbm'] = True + instance_docker['min_collection_interval'] = 1 + # set a very small collection interval so the tests go fast + instance_docker['collect_settings'] = { + 'enabled': True, + 'run_sync': True, + 'collection_interval': 0.1, + } + return copy(instance_docker) + + +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize( + "expected_columns,available_columns", + [ + [ + ["name", "value"], + ["name", "value"], + ], + [ + ["name", "value", "some_missing_column"], + ["name", "value"], + ], + ], +) +def test_get_available_settings_columns(dbm_instance, expected_columns, available_columns): + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + check.initialize_connection() + _conn_key_prefix = "dbm-metadata-" + with check.connection.open_managed_default_connection(key_prefix=_conn_key_prefix): + with check.connection.get_managed_cursor(key_prefix=_conn_key_prefix) as cursor: + result_available_columns = check.sql_metadata._get_available_settings_columns(cursor, expected_columns) + assert result_available_columns == available_columns + + +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_get_settings_query_cached(dbm_instance, caplog): + caplog.set_level(logging.DEBUG) + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + check.initialize_connection() + _conn_key_prefix = "dbm-metadata" + with check.connection.open_managed_default_connection(key_prefix=_conn_key_prefix): + with check.connection.get_managed_cursor(key_prefix=_conn_key_prefix) as cursor: + for _ in range(3): + query = check.sql_metadata._get_settings_query_cached(cursor) + assert query, "query should be non-empty" + times_columns_loaded = 0 + for r in caplog.records: + if r.message.startswith("found available sys.configurations columns"): + times_columns_loaded += 1 + assert times_columns_loaded == 1, "columns should have been loaded only once" + + +def test_sqlserver_collect_settings(aggregator, dd_run_check, dbm_instance): + check = SQLServer(CHECK_NAME, {}, [dbm_instance]) + # dd_run_check(check) + check.initialize_connection() + check.check(dbm_instance) + dbm_metadata = aggregator.get_event_platform_events("dbm-metadata") + event = dbm_metadata[0] + assert event['dbms'] == "sqlserver" + assert event['kind'] == "sqlserver_configs" + assert len(event["metadata"]) > 0