[DBMON-6548] Add Mysql global wait_event query metric

mysql/changelog.d/23562.added

@@ -0,0 +1 @@
+Add collection of global wait event metrics from performance_schema for DBM-enabled instances.

mysql/datadog_checks/mysql/mysql.py

@@ -67,6 +67,7 @@
     QUERY_DEADLOCKS,
     QUERY_ERRORS_RAISED,
     QUERY_USER_CONNECTIONS,
+    QUERY_WAIT_EVENT_SUMMARY,
     SQL_95TH_PERCENTILE,
     SQL_AVG_QUERY_RUN_TIME,
     SQL_GROUP_REPLICATION_MEMBER,
@@ -468,8 +469,10 @@ def _get_runtime_queries(self, db):
 
         if self.global_variables.performance_schema_enabled:
             queries.extend([QUERY_USER_CONNECTIONS])
-            if not self.is_mariadb and self.version.version_compatible((8, 0, 0)) and self._config.dbm_enabled:
-                queries.extend([QUERY_ERRORS_RAISED])
+            if self._config.dbm_enabled:
+                queries.extend([QUERY_WAIT_EVENT_SUMMARY])
+                if not self.is_mariadb and self.version.version_compatible((8, 0, 0)):
+                    queries.extend([QUERY_ERRORS_RAISED])
         if self._index_metrics.include_index_metrics:
             queries.extend(self._index_metrics.queries)
         self._runtime_queries_cached = self._new_query_executor(queries)

mysql/datadog_checks/mysql/queries.py

@@ -262,6 +262,29 @@
     ],
 }
 
+QUERY_WAIT_EVENT_SUMMARY = {
+    'name': 'performance_schema.events_waits_summary_global_by_event_name',
+    'query': """
+        SELECT
+            event_name,
+            count_star,
+            sum_timer_wait / 1000,
+            avg_timer_wait / 1000,
+            max_timer_wait / 1000
+        FROM performance_schema.events_waits_summary_global_by_event_name
+        WHERE count_star > 0
+        ORDER BY sum_timer_wait DESC
+        LIMIT 200
+    """.strip(),
+    'columns': [
+        {'name': 'wait_event', 'type': 'tag'},
+        {'name': 'mysql.performance.wait_event.count', 'type': 'monotonic_count'},
+        {'name': 'mysql.performance.wait_event.time', 'type': 'monotonic_count'},
+        {'name': 'mysql.performance.wait_event.avg_time', 'type': 'gauge'},
+        {'name': 'mysql.performance.wait_event.max_time', 'type': 'gauge'},
+    ],
+}
+
 
 def show_replica_status_query(version, is_mariadb, channel=''):
     if version.version_compatible((10, 5, 1)) or not is_mariadb and version.version_compatible((8, 0, 22)):

mysql/metadata.csv

@@ -215,6 +215,10 @@ mysql.performance.threads_created,count,,thread,,"The number of threads created
 mysql.performance.threads_running,gauge,,thread,,The number of threads that are not sleeping.,0,mysql,threads running,
 mysql.performance.user_connections,gauge,,connection,,"The number of user connections. Tags: `processlist_db`, `processlist_host`, `processlist_state`, `processlist_user`",0,mysql,user conns,
 mysql.performance.user_time,gauge,,percent,,Percentage of CPU time spent in user space by MySQL.,-1,mysql,cpu user,
+mysql.performance.wait_event.avg_time,gauge,,nanosecond,,Average wait time per occurrence of a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event avg time,
+mysql.performance.wait_event.count,count,,event,,Total number of occurrences of a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event count,
+mysql.performance.wait_event.max_time,gauge,,nanosecond,,Maximum single-occurrence wait time for a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event max time,
+mysql.performance.wait_event.time,count,,nanosecond,,Total accumulated wait time for a wait event (DBM only). Tagged by `wait_event`.,0,mysql,wait event time,
 mysql.queries.count,count,,query,,The total count of executed queries per normalized query and schema. (DBM only),0,mysql,mysql queries count,cpu
 mysql.queries.created_tmp_disk_tables,count,,table,,The total count of temporary tables that exceeded tmp_table_size and were written to disk per normalized query and schema. (DBM only),0,mysql,mysql queries created tmp disk tables,
 mysql.queries.created_tmp_tables,count,,table,,The total count of in-memory temporary tables created during execution per normalized query and schema. (DBM only),0,mysql,mysql queries created tmp tables,

mysql/tests/test_mysql.py

@@ -1065,3 +1065,44 @@ def test_errors_raised_metric_with_dbm(aggregator, dd_run_check, instance_basic,
     else:
         # In all other cases the metric should not be present
         aggregator.assert_metric('mysql.performance.errors_raised', count=0)
+
+
+@pytest.mark.integration
+@pytest.mark.usefixtures('dd_environment')
+@pytest.mark.parametrize(
+    'dbm_enabled',
+    [
+        pytest.param(True, id="dbm_enabled"),
+        pytest.param(False, id="dbm_disabled"),
+    ],
+)
+def test_wait_event_summary_metrics_with_dbm(aggregator, dd_run_check, instance_basic, dbm_enabled):
+    instance_basic['dbm'] = dbm_enabled
+    if dbm_enabled:
+        instance_basic['collect_settings'] = {'enabled': False}
+        instance_basic['query_activity'] = {'enabled': False}
+        instance_basic['query_samples'] = {'enabled': False}
+        instance_basic['query_metrics'] = {'enabled': False}
+
+    mysql_check = MySql(common.CHECK_NAME, {}, [instance_basic])
+    dd_run_check(mysql_check)
+
+    aggregator.assert_service_check('mysql.can_connect', status=MySql.OK, count=1)
+
+    wait_event_metrics = [
+        'mysql.performance.wait_event.count',
+        'mysql.performance.wait_event.time',
+        'mysql.performance.wait_event.avg_time',
+        'mysql.performance.wait_event.max_time',
+    ]
+    if dbm_enabled:
+        for metric in wait_event_metrics:
+            aggregator.assert_metric(metric, at_least=1)
+        # Verify wait_event tag is present on submitted metrics
+        for metric in wait_event_metrics:
+            for m in aggregator.metrics(metric):
+                wait_event_tags = [t for t in m.tags if t.startswith('wait_event:')]
+                assert len(wait_event_tags) == 1, "Expected exactly one wait_event tag on {}".format(metric)
+    else:
+        for metric in wait_event_metrics:
+            aggregator.assert_metric(metric, count=0)