diff --git a/mysql/assets/configuration/spec.yaml b/mysql/assets/configuration/spec.yaml index ece7d2676f0bf..f4b27f3974804 100644 --- a/mysql/assets/configuration/spec.yaml +++ b/mysql/assets/configuration/spec.yaml @@ -335,6 +335,22 @@ files: type: boolean example: false display_default: false + - name: collect_settings + description: Configure collection of performance_schema.global_variables. This is an alpha feature. + options: + - name: enabled + description: | + Enable collection of performance_schema.global_variables. Requires `dbm: true`. + value: + type: boolean + example: false + - name: collection_interval + description: | + Set the database settings collection interval (in seconds). Each collection involves a single query to + `performance_schema.global_variables`. + value: + type: number + example: 600 - name: query_metrics description: Configure collection of query metrics options: diff --git a/mysql/datadog_checks/mysql/config.py b/mysql/datadog_checks/mysql/config.py index 51d9093d027b6..0590be8dc319e 100644 --- a/mysql/datadog_checks/mysql/config.py +++ b/mysql/datadog_checks/mysql/config.py @@ -39,6 +39,7 @@ def __init__(self, instance): ) self.statement_samples_config = instance.get('query_samples', instance.get('statement_samples', {})) or {} self.statement_metrics_config = instance.get('query_metrics', {}) or {} + self.settings_config = instance.get('collect_settings', {}) or {} self.activity_config = instance.get('query_activity', {}) or {} self.cloud_metadata = {} aws = instance.get('aws', {}) diff --git a/mysql/datadog_checks/mysql/config_models/instance.py b/mysql/datadog_checks/mysql/config_models/instance.py index d233b0ad48b9e..865bad7fab297 100644 --- a/mysql/datadog_checks/mysql/config_models/instance.py +++ b/mysql/datadog_checks/mysql/config_models/instance.py @@ -37,6 +37,15 @@ class Azure(BaseModel): fully_qualified_domain_name: Optional[str] = None +class CollectSettings(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + + class CustomQuery(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -157,6 +166,7 @@ class InstanceConfig(BaseModel): aws: Optional[Aws] = None azure: Optional[Azure] = None charset: Optional[str] = None + collect_settings: Optional[CollectSettings] = None connect_timeout: Optional[float] = None custom_queries: Optional[tuple[CustomQuery, ...]] = None dbm: Optional[bool] = None diff --git a/mysql/datadog_checks/mysql/data/conf.yaml.example b/mysql/datadog_checks/mysql/data/conf.yaml.example index dba884041310c..ea3653318c412 100644 --- a/mysql/datadog_checks/mysql/data/conf.yaml.example +++ b/mysql/datadog_checks/mysql/data/conf.yaml.example @@ -334,6 +334,21 @@ instances: # # dbm: false + ## Configure collection of performance_schema.global_variables. This is an alpha feature. + # + # collect_settings: + + ## @param enabled - boolean - optional - default: false + ## Enable collection of performance_schema.global_variables. Requires `dbm: true`. + # + # enabled: false + + ## @param collection_interval - number - optional - default: 600 + ## Set the database settings collection interval (in seconds). Each collection involves a single query to + ## `performance_schema.global_variables`. + # + # collection_interval: 600 + ## Configure collection of query metrics # # query_metrics: diff --git a/mysql/datadog_checks/mysql/metadata.py b/mysql/datadog_checks/mysql/metadata.py new file mode 100644 index 0000000000000..0bf68b9dfdea8 --- /dev/null +++ b/mysql/datadog_checks/mysql/metadata.py @@ -0,0 +1,128 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import time +from contextlib import closing +from operator import attrgetter + +import pymysql + +try: + import datadog_agent +except ImportError: + from ..stubs import datadog_agent + +from datadog_checks.base import is_affirmative +from datadog_checks.base.utils.db.utils import ( + DBMAsyncJob, + default_json_event_encoding, +) +from datadog_checks.base.utils.serialization import json +from datadog_checks.base.utils.tracking import tracked_method + +# default pg_settings collection interval in seconds +DEFAULT_SETTINGS_COLLECTION_INTERVAL = 600 + +MARIADB_TABLE_NAME = "information_schema.GLOBAL_VARIABLES" +MYSQL_TABLE_NAME = "performance_schema.global_variables" + +SETTINGS_QUERY = """ +SELECT + variable_name, + variable_value +FROM + {table_name} +""" + + +class MySQLMetadata(DBMAsyncJob): + """ + Collects database metadata. Supports: + 1. collection of performance_schema.global_variables + """ + + def __init__(self, check, config, connection_args): + self.collection_interval = float( + config.settings_config.get('collection_interval', DEFAULT_SETTINGS_COLLECTION_INTERVAL) + ) + super(MySQLMetadata, self).__init__( + check, + rate_limit=1 / self.collection_interval, + run_sync=is_affirmative(config.settings_config.get('run_sync', False)), + enabled=is_affirmative(config.settings_config.get('enabled', False)), + min_collection_interval=config.min_collection_interval, + dbms="mysql", + expected_db_exceptions=(pymysql.err.DatabaseError,), + job_name="database-metadata", + shutdown_callback=self._close_db_conn, + ) + self._check = check + self._config = config + self._version_processed = False + self._connection_args = connection_args + self._db = None + self._check = check + + def _get_db_connection(self): + """ + lazy reconnect db + pymysql connections are not thread safe so we can't reuse the same connection from the main check + :return: + """ + if not self._db: + self._db = pymysql.connect(**self._connection_args) + return self._db + + def _close_db_conn(self): + if self._db: + try: + self._db.close() + except Exception: + self._log.debug("Failed to close db connection", exc_info=1) + finally: + self._db = None + + def _cursor_run(self, cursor, query, params=None): + """ + Run and log the query. If provided, obfuscated params are logged in place of the regular params. + """ + try: + self._log.debug("Running query [{}] params={}".format(query, params)) + cursor.execute(query, params) + except pymysql.DatabaseError as e: + self._check.count( + "dd.mysql.db.error", + 1, + tags=self._tags + ["error:{}".format(type(e))] + self._check._get_debug_tags(), + hostname=self._check.resolved_hostname, + ) + raise + + def run_job(self): + self.report_mysql_metadata() + + @tracked_method(agent_check_getter=attrgetter('_check')) + def report_mysql_metadata(self): + settings = [] + table_name = MYSQL_TABLE_NAME if not self._check.is_mariadb else MARIADB_TABLE_NAME + query = SETTINGS_QUERY.format(table_name=table_name) + with closing(self._get_db_connection().cursor(pymysql.cursors.DictCursor)) as cursor: + self._cursor_run( + cursor, + query, + ) + rows = cursor.fetchall() + settings = [dict(row) for row in rows] + event = { + "host": self._check.resolved_hostname, + "agent_version": datadog_agent.get_version(), + "dbms": "mysql", + "kind": "mysql_variables", + "collection_interval": self.collection_interval, + 'dbms_version': self._check.version.version + '+' + self._check.version.build, + "tags": self._tags, + "timestamp": time.time() * 1000, + "cloud_metadata": self._config.cloud_metadata, + "metadata": settings, + } + self._check.database_monitoring_metadata(json.dumps(event, default=default_json_event_encoding)) diff --git a/mysql/datadog_checks/mysql/mysql.py b/mysql/datadog_checks/mysql/mysql.py index 653ece013cfda..82463cd75f9f4 100644 --- a/mysql/datadog_checks/mysql/mysql.py +++ b/mysql/datadog_checks/mysql/mysql.py @@ -44,6 +44,7 @@ VARIABLES_VARS, ) from .innodb_metrics import InnoDBMetrics +from .metadata import MySQLMetadata from .queries import ( QUERY_USER_CONNECTIONS, SQL_95TH_PERCENTILE, @@ -116,6 +117,7 @@ def __init__(self, name, init_config, instances): self._warnings_by_code = {} self._statement_metrics = MySQLStatementMetrics(self, self._config, self._get_connection_args()) self._statement_samples = MySQLStatementSamples(self, self._config, self._get_connection_args()) + self._mysql_metadata = MySQLMetadata(self, self._config, self._get_connection_args()) self._query_activity = MySQLActivity(self, self._config, self._get_connection_args()) self._runtime_queries = None @@ -274,6 +276,7 @@ def check(self, _): self._statement_metrics.run_job_loop(dbm_tags) self._statement_samples.run_job_loop(dbm_tags) self._query_activity.run_job_loop(dbm_tags) + self._mysql_metadata.run_job_loop(dbm_tags) # keeping track of these: self._put_qcache_stats() @@ -292,6 +295,7 @@ def cancel(self): self._statement_samples.cancel() self._statement_metrics.cancel() self._query_activity.cancel() + self._mysql_metadata.cancel() def _new_query_executor(self, queries): return QueryExecutor( diff --git a/mysql/tests/test_metadata.py b/mysql/tests/test_metadata.py new file mode 100644 index 0000000000000..f917123551868 --- /dev/null +++ b/mysql/tests/test_metadata.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + +from datadog_checks.mysql import MySql + +from . import common + + +@pytest.fixture +def dbm_instance(instance_complex): + instance_complex['dbm'] = True + instance_complex['query_samples'] = {'enabled': False} + instance_complex['query_metrics'] = {'enabled': False} + instance_complex['query_activity'] = {'enabled': False} + instance_complex['collect_settings'] = {'enabled': True, 'run_sync': True, 'collection_interval': 0.1} + return instance_complex + + +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_collect_mysql_settings(aggregator, dbm_instance, dd_run_check): + # test to make sure we continue to support the old key + mysql_check = MySql(common.CHECK_NAME, {}, instances=[dbm_instance]) + dd_run_check(mysql_check) + dbm_metadata = aggregator.get_event_platform_events("dbm-metadata") + event = dbm_metadata[0] + assert event['host'] == "stubbed.hostname" + assert event['dbms'] == "mysql" + assert event['kind'] == "mysql_variables" + assert len(event["metadata"]) > 0