diff --git a/monitor-snapshot/master/ansible-monitor.tar.gz b/monitor-snapshot/master/ansible-monitor.tar.gz index 7d2a0f39..86c12cb4 100644 Binary files a/monitor-snapshot/master/ansible-monitor.tar.gz and b/monitor-snapshot/master/ansible-monitor.tar.gz differ diff --git a/monitor-snapshot/master/operator/dashboards/tikv_details.json b/monitor-snapshot/master/operator/dashboards/tikv_details.json index 2b6b8a22..16462a9d 100644 --- a/monitor-snapshot/master/operator/dashboards/tikv_details.json +++ b/monitor-snapshot/master/operator/dashboards/tikv_details.json @@ -39552,55 +39552,13 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "repeatDirection": null, - "span": null, - "targets": [], - "timeFrom": null, - "timeShift": null, - "title": "In Memory Engine", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "cacheTimeout": null, - "collapsed": true, - "datasource": null, - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 277, - "interval": null, - "links": [], - "maxDataPoints": 100, - "maxPerRow": null, - "minSpan": null, - "panels": [ + }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "", + "description": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one", "editable": true, "error": false, "fieldConfig": { @@ -39621,13 +39579,13 @@ }, "gridPos": { "h": 7, - "w": 12, + "w": 6, "x": 0, - "y": 0 + "y": 49 }, "height": null, "hideTimeOverride": false, - "id": 278, + "id": 277, "interval": null, "isNew": true, "legend": { @@ -39670,30 +39628,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "max((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "write-{{instance}}", - "metric": "", - "query": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "throttle-{{instance}}", + "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", + "query": "max((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" @@ -39702,7 +39645,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler flow", + "title": "Oldest Auto GC SafePoint", "tooltip": { "msResolution": true, "shared": true, @@ -39721,7 +39664,7 @@ "yaxes": [ { "decimals": null, - "format": "bytes", + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, @@ -39748,7 +39691,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "", + "description": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one", "editable": true, "error": false, "fieldConfig": { @@ -39769,13 +39712,13 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 0 + "w": 6, + "x": 6, + "y": 49 }, "height": null, "hideTimeOverride": false, - "id": 279, + "id": 278, "interval": null, "isNew": true, "legend": { @@ -39818,15 +39761,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", + "expr": "max((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", + "query": "max((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" @@ -39835,7 +39778,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler discard ratio", + "title": "Newest Auto GC SafePoint", "tooltip": { "msResolution": true, "shared": true, @@ -39854,7 +39797,7 @@ "yaxes": [ { "decimals": null, - "format": "percentunit", + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, @@ -39876,117 +39819,12 @@ "alignLevel": 0 } }, - { - "cacheTimeout": null, - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "tidb-cluster", - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "heatmap": {}, - "height": null, - "hideTimeOverride": false, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 280, - "interval": null, - "legend": { - "show": false - }, - "links": [], - "maxDataPoints": 512, - "maxPerRow": null, - "minSpan": null, - "options": {}, - "repeat": null, - "repeatDirection": null, - "reverseYBuckets": false, - "span": null, - "targets": [ - { - "datasource": "tidb-cluster", - "expr": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "metric": "", - "query": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", - "refId": "", - "step": 10, - "target": "" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Throttle duration", - "tooltip": { - "msResolution": true, - "shared": true, - "show": true, - "showHistogram": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "transparent": false, - "type": "heatmap", - "xAxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": null, + "description": "The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine", "editable": true, "error": false, "fieldConfig": { @@ -40007,13 +39845,13 @@ }, "gridPos": { "h": 7, - "w": 12, + "w": 6, "x": 12, - "y": 7 + "y": 49 }, "height": null, "hideTimeOverride": false, - "id": 281, + "id": 279, "interval": null, "isNew": true, "legend": { @@ -40056,15 +39894,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", + "expr": "(sum((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) - sum((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{cf}}", + "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", - "query": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", + "query": "(sum((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) - sum((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18))", "refId": "", "step": 10, "target": "" @@ -40073,7 +39911,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler throttled CF", + "title": "Auto GC SafePoint Gap", "tooltip": { "msResolution": true, "shared": true, @@ -40092,7 +39930,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -40119,7 +39957,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "", + "description": "The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point", "editable": true, "error": false, "fieldConfig": { @@ -40140,13 +39978,13 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 14 + "w": 6, + "x": 18, + "y": 49 }, "height": null, "hideTimeOverride": false, - "id": 282, + "id": 280, "interval": null, "isNew": true, "legend": { @@ -40189,15 +40027,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) ", + "expr": "max((\n tikv_safe_point_gap_with_in_memory_engine\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}}-{{cf}} {{$additional_groupby}}", + "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) ", + "query": "max((\n tikv_safe_point_gap_with_in_memory_engine\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" @@ -40206,7 +40044,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Flow controller actions", + "title": "Auto GC SafePoint Gap With TiKV", "tooltip": { "msResolution": true, "shared": true, @@ -40225,7 +40063,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -40246,7 +40084,49 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "repeatDirection": null, + "span": null, + "targets": [], + "timeFrom": null, + "timeShift": null, + "title": "In Memory Engine", + "transformations": [], + "transparent": false, + "type": "row" + }, + { + "cacheTimeout": null, + "collapsed": true, + "datasource": null, + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 281, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "panels": [ { "aliasColors": {}, "bars": false, @@ -40274,12 +40154,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 14 + "x": 0, + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 283, + "id": 282, "interval": null, "isNew": true, "legend": { @@ -40322,60 +40202,30 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{cf}}_l0_flow-{{instance}}", - "metric": "", - "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{cf}}_flush_flow-{{instance}}", - "metric": "", - "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "total_l0_flow-{{instance}}", + "legendFormat": "write-{{instance}}", "metric": "", - "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "query": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "total_flush_flow-{{instance}}", + "legendFormat": "throttle-{{instance}}", "metric": "", - "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "query": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", "refId": "", "step": 10, "target": "" @@ -40384,7 +40234,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Flush/L0 flow", + "title": "Scheduler flow", "tooltip": { "msResolution": true, "shared": true, @@ -40452,12 +40302,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 21 + "x": 12, + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 284, + "id": 283, "interval": null, "isNew": true, "legend": { @@ -40500,45 +40350,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "l0-{{instance}}", - "metric": "", - "query": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "memtable-{{instance}}", - "metric": "", - "query": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "avg_l0-{{instance}}", + "legendFormat": "{{type}}", "metric": "", - "query": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "query": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", "refId": "", "step": 10, "target": "" @@ -40547,7 +40367,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Flow controller factors", + "title": "Scheduler discard ratio", "tooltip": { "msResolution": true, "shared": true, @@ -40566,7 +40386,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -40589,11 +40409,23 @@ } }, { - "aliasColors": {}, - "bars": false, "cacheTimeout": null, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "tidb-cluster", - "description": "", + "description": null, "editable": true, "error": false, "fieldConfig": { @@ -40604,144 +40436,89 @@ } } }, - "fill": 1, - "fillGradient": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 21 + "x": 0, + "y": 7 }, + "heatmap": {}, "height": null, "hideTimeOverride": false, - "id": 285, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 284, "interval": null, - "isNew": true, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true + "show": false }, - "lines": true, - "linewidth": 1, "links": [], - "maxDataPoints": null, + "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true, - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": {}, "repeat": null, "repeatDirection": null, - "seriesOverrides": [], + "reverseYBuckets": false, "span": null, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", - "format": "time_series", + "expr": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", + "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{cf}} {{$additional_groupby}}", - "metric": "", - "query": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "pending-bytes {{$additional_groupby}}", + "legendFormat": "{{le}}", "metric": "", - "query": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000", + "query": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Compaction pending bytes", + "title": "Throttle duration", "tooltip": { "msResolution": true, "shared": true, + "show": true, + "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, - "type": "graph", - "xaxis": { + "type": "heatmap", + "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, - "yaxes": [ - { - "decimals": null, - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": 0 - } + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "Throttle time for txn storage commands in 1 minute.", + "description": null, "editable": true, "error": false, "fieldConfig": { @@ -40763,12 +40540,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 28 + "x": 12, + "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 286, + "id": 285, "interval": null, "isNew": true, "legend": { @@ -40811,15 +40588,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "expr": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}} {{$additional_groupby}}", + "legendFormat": "{{instance}}-{{cf}}", "metric": "", - "query": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "query": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", "refId": "", "step": 10, "target": "" @@ -40828,7 +40605,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Txn command throttled duration", + "title": "Scheduler throttled CF", "tooltip": { "msResolution": true, "shared": true, @@ -40847,7 +40624,7 @@ "yaxes": [ { "decimals": null, - "format": "\u00b5s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -40874,7 +40651,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -40896,12 +40673,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 28 + "x": 0, + "y": 14 }, "height": null, "hideTimeOverride": false, - "id": 287, + "id": 286, "interval": null, "isNew": true, "legend": { @@ -40944,15 +40721,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "expr": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}} {{$additional_groupby}}", + "legendFormat": "{{type}}-{{cf}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "query": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -40961,7 +40738,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Non-txn command throttled duration", + "title": "Flow controller actions", "tooltip": { "msResolution": true, "shared": true, @@ -40980,7 +40757,7 @@ "yaxes": [ { "decimals": null, - "format": "\u00b5s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -41001,55 +40778,13 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "repeatDirection": null, - "span": null, - "targets": [], - "timeFrom": null, - "timeShift": null, - "title": "Flow Control", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "cacheTimeout": null, - "collapsed": true, - "datasource": null, - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 288, - "interval": null, - "links": [], - "maxDataPoints": 100, - "maxPerRow": null, - "minSpan": null, - "panels": [ + }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The total number of commands on each stage", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -41071,12 +40806,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 0 + "x": 12, + "y": 14 }, "height": null, "hideTimeOverride": false, - "id": 289, + "id": 287, "interval": null, "isNew": true, "legend": { @@ -41119,30 +40854,60 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", + "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{stage}} {{$additional_groupby}}", + "legendFormat": "{{cf}}_l0_flow-{{instance}}", "metric": "", - "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", + "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", + "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{stage}} {{$additional_groupby}}", + "legendFormat": "{{cf}}_flush_flow-{{instance}}", "metric": "", - "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", + "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total_l0_flow-{{instance}}", + "metric": "", + "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total_flush_flow-{{instance}}", + "metric": "", + "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" @@ -41151,7 +40916,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler stage total", + "title": "Flush/L0 flow", "tooltip": { "msResolution": true, "shared": true, @@ -41170,7 +40935,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -41197,7 +40962,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The count of different priority commands", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -41219,12 +40984,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 21 }, "height": null, "hideTimeOverride": false, - "id": 290, + "id": 288, "interval": null, "isNew": true, "legend": { @@ -41267,15 +41032,45 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", + "expr": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{priority}} {{$additional_groupby}}", + "legendFormat": "l0-{{instance}}", "metric": "", - "query": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", + "query": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "memtable-{{instance}}", + "metric": "", + "query": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg_l0-{{instance}}", + "metric": "", + "query": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" @@ -41284,7 +41079,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler priority commands", + "title": "Flow controller factors", "tooltip": { "msResolution": true, "shared": true, @@ -41303,7 +41098,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -41330,7 +41125,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The count of pending commands per TiKV instance", + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -41352,12 +41147,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 7 + "x": 12, + "y": 21 }, "height": null, "hideTimeOverride": false, - "id": 291, + "id": 289, "interval": null, "isNew": true, "legend": { @@ -41400,15 +41195,30 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{cf}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "query": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pending-bytes {{$additional_groupby}}", + "metric": "", + "query": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000", "refId": "", "step": 10, "target": "" @@ -41417,7 +41227,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler pending commands", + "title": "Compaction pending bytes", "tooltip": { "msResolution": true, "shared": true, @@ -41436,7 +41246,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -41463,7 +41273,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The count of running commands per TiKV instance", + "description": "Throttle time for txn storage commands in 1 minute.", "editable": true, "error": false, "fieldConfig": { @@ -41485,12 +41295,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 7 + "x": 0, + "y": 28 }, "height": null, "hideTimeOverride": false, - "id": 292, + "id": 290, "interval": null, "isNew": true, "legend": { @@ -41533,15 +41343,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "query": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -41550,7 +41360,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler running commands", + "title": "Txn command throttled duration", "tooltip": { "msResolution": true, "shared": true, @@ -41569,7 +41379,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "\u00b5s", "label": null, "logBase": 1, "max": null, @@ -41596,7 +41406,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The total writing bytes of commands on each stage", + "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", "editable": true, "error": false, "fieldConfig": { @@ -41618,12 +41428,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 14 + "x": 12, + "y": 28 }, "height": null, "hideTimeOverride": false, - "id": 293, + "id": 291, "interval": null, "isNew": true, "legend": { @@ -41666,15 +41476,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "expr": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "query": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -41683,7 +41493,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler writing bytes", + "title": "Non-txn command throttled duration", "tooltip": { "msResolution": true, "shared": true, @@ -41702,7 +41512,7 @@ "yaxes": [ { "decimals": null, - "format": "bytes", + "format": "\u00b5s", "label": null, "logBase": 1, "max": null, @@ -41723,13 +41533,55 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "repeatDirection": null, + "span": null, + "targets": [], + "timeFrom": null, + "timeShift": null, + "title": "Flow Control", + "transformations": [], + "transparent": false, + "type": "row" + }, + { + "cacheTimeout": null, + "collapsed": true, + "datasource": null, + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 292, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The number of bytes used by scheduler", + "description": "The total number of commands on each stage", "editable": true, "error": false, "fieldConfig": { @@ -41751,12 +41603,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 14 + "x": 0, + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 294, + "id": 293, "interval": null, "isNew": true, "legend": { @@ -41799,30 +41651,30 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", + "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", + "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", + "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", - "hide": true, + "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", + "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -41831,7 +41683,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Scheduler memory quota", + "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, @@ -41850,7 +41702,7 @@ "yaxes": [ { "decimals": null, - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -41872,159 +41724,12 @@ "alignLevel": 0 } }, - { - "cacheTimeout": null, - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "tidb-cluster", - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 21 - }, - "heatmap": {}, - "height": null, - "hideTimeOverride": false, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 295, - "interval": null, - "legend": { - "show": false - }, - "links": [], - "maxDataPoints": 512, - "maxPerRow": null, - "minSpan": null, - "options": {}, - "repeat": null, - "repeatDirection": null, - "reverseYBuckets": false, - "span": null, - "targets": [ - { - "datasource": "tidb-cluster", - "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "metric": "", - "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", - "refId": "", - "step": 10, - "target": "" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Txn Scheduler Pool Wait Duration", - "tooltip": { - "msResolution": true, - "shared": true, - "show": true, - "showHistogram": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "transparent": false, - "type": "heatmap", - "xAxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - } - ], - "repeat": null, - "repeatDirection": null, - "span": null, - "targets": [], - "timeFrom": null, - "timeShift": null, - "title": "Scheduler", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "cacheTimeout": null, - "collapsed": true, - "datasource": null, - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 296, - "interval": null, - "links": [], - "maxDataPoints": 100, - "maxPerRow": null, - "minSpan": null, - "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.", + "description": "The count of different priority commands", "editable": true, "error": false, "fieldConfig": { @@ -42046,12 +41751,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, + "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 297, + "id": 294, "interval": null, "isNew": true, "legend": { @@ -42094,15 +41799,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", + "expr": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{level}} {{$additional_groupby}}", + "legendFormat": "{{priority}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", + "query": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -42111,7 +41816,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Time used by level", + "title": "Scheduler priority commands", "tooltip": { "msResolution": true, "shared": true, @@ -42130,7 +41835,7 @@ "yaxes": [ { "decimals": null, - "format": "\u00b5s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -42157,7 +41862,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.", + "description": "The count of pending commands per TiKV instance", "editable": true, "error": false, "fieldConfig": { @@ -42179,12 +41884,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 298, + "id": 295, "interval": null, "isNew": true, "legend": { @@ -42227,7 +41932,7 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) ", + "expr": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, @@ -42235,7 +41940,7 @@ "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", - "query": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) ", + "query": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" @@ -42244,7 +41949,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Level 0 chance", + "title": "Scheduler pending commands", "tooltip": { "msResolution": true, "shared": true, @@ -42263,7 +41968,7 @@ "yaxes": [ { "decimals": null, - "format": "percentunit", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -42290,7 +41995,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The number of concurrently running tasks in the yatp thread pool.", + "description": "The count of running commands per TiKV instance", "editable": true, "error": false, "fieldConfig": { @@ -42312,12 +42017,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, + "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 299, + "id": 296, "interval": null, "isNew": true, "legend": { @@ -42360,15 +42065,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) ", + "expr": "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}} {{$additional_groupby}}", + "legendFormat": "{{instance}}", "metric": "", - "query": "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) ", + "query": "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" @@ -42377,7 +42082,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Running tasks", + "title": "Scheduler running commands", "tooltip": { "msResolution": true, "shared": true, @@ -42419,23 +42124,11 @@ } }, { + "aliasColors": {}, + "bars": false, "cacheTimeout": null, - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": "tidb-cluster", - "description": null, + "description": "The total writing bytes of commands on each stage", "editable": true, "error": false, "fieldConfig": { @@ -42446,89 +42139,129 @@ } } }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 7 + "x": 0, + "y": 14 }, - "heatmap": {}, "height": null, "hideTimeOverride": false, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 300, + "id": 297, "interval": null, + "isNew": true, "legend": { - "show": false + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true }, + "lines": true, + "linewidth": 1, "links": [], - "maxDataPoints": 512, + "maxDataPoints": null, "maxPerRow": null, "minSpan": null, - "options": {}, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "repeat": null, "repeatDirection": null, - "reverseYBuckets": false, + "seriesOverrides": [], "span": null, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", - "format": "heatmap", + "expr": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", + "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{le}}", + "legendFormat": "{{instance}}", "metric": "", - "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", + "query": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], + "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Wait Duration", + "title": "Scheduler writing bytes", "tooltip": { "msResolution": true, "shared": true, - "show": true, - "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, - "type": "heatmap", - "xAxis": { + "type": "graph", + "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "Task execution time during one schedule.", + "description": "The number of bytes used by scheduler", "editable": true, "error": false, "fieldConfig": { @@ -42550,12 +42283,811 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, + "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, - "id": 301, + "id": 298, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "", + "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "", + "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Scheduler memory quota", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "cacheTimeout": null, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "tidb-cluster", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "heatmap": {}, + "height": null, + "hideTimeOverride": false, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 299, + "interval": null, + "legend": { + "show": false + }, + "links": [], + "maxDataPoints": 512, + "maxPerRow": null, + "minSpan": null, + "options": {}, + "repeat": null, + "repeatDirection": null, + "reverseYBuckets": false, + "span": null, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "metric": "", + "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Txn Scheduler Pool Wait Duration", + "tooltip": { + "msResolution": true, + "shared": true, + "show": true, + "showHistogram": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + } + ], + "repeat": null, + "repeatDirection": null, + "span": null, + "targets": [], + "timeFrom": null, + "timeShift": null, + "title": "Scheduler", + "transformations": [], + "transparent": false, + "type": "row" + }, + { + "cacheTimeout": null, + "collapsed": true, + "datasource": null, + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 300, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 301, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{level}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Time used by level", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "\u00b5s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 302, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "", + "query": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Level 0 chance", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": "The number of concurrently running tasks in the yatp thread pool.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "height": null, + "hideTimeOverride": false, + "id": 303, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}} {{$additional_groupby}}", + "metric": "", + "query": "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Running tasks", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "cacheTimeout": null, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "tidb-cluster", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "heatmap": {}, + "height": null, + "hideTimeOverride": false, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 304, + "interval": null, + "legend": { + "show": false + }, + "links": [], + "maxDataPoints": 512, + "maxPerRow": null, + "minSpan": null, + "options": {}, + "repeat": null, + "repeatDirection": null, + "reverseYBuckets": false, + "span": null, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "metric": "", + "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Wait Duration", + "tooltip": { + "msResolution": true, + "shared": true, + "show": true, + "showHistogram": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": "Task execution time during one schedule.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "height": null, + "hideTimeOverride": false, + "id": 305, "interval": null, "isNew": true, "legend": { @@ -42756,7 +43288,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 302, + "id": 306, "interval": null, "isNew": true, "legend": { @@ -42957,7 +43489,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 303, + "id": 307, "interval": null, "isNew": true, "legend": { @@ -43161,7 +43693,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 304, + "id": 308, "interval": null, "links": [], "maxDataPoints": 100, @@ -43200,7 +43732,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 305, + "id": 309, "interval": null, "isNew": true, "legend": { @@ -43348,7 +43880,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 306, + "id": 310, "interval": null, "isNew": true, "legend": { @@ -43549,7 +44081,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 307, + "id": 311, "interval": null, "isNew": true, "legend": { @@ -43750,7 +44282,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 308, + "id": 312, "interval": null, "isNew": true, "legend": { @@ -43951,7 +44483,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 309, + "id": 313, "interval": null, "isNew": true, "legend": { @@ -44152,7 +44684,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 310, + "id": 314, "interval": null, "isNew": true, "legend": { @@ -44285,7 +44817,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 311, + "id": 315, "interval": null, "isNew": true, "legend": { @@ -44418,7 +44950,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 312, + "id": 316, "interval": null, "isNew": true, "legend": { @@ -44551,7 +45083,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 313, + "id": 317, "interval": null, "isNew": true, "legend": { @@ -44684,7 +45216,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 314, + "id": 318, "interval": null, "isNew": true, "legend": { @@ -44892,7 +45424,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 315, + "id": 319, "interval": null, "legend": { "show": false @@ -44993,7 +45525,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 316, + "id": 320, "interval": null, "links": [], "maxDataPoints": 100, @@ -45039,7 +45571,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 317, + "id": 321, "interval": null, "legend": { "show": false @@ -45137,7 +45669,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 318, + "id": 322, "interval": null, "isNew": true, "legend": { @@ -45338,7 +45870,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 319, + "id": 323, "interval": null, "isNew": true, "legend": { @@ -45471,7 +46003,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 320, + "id": 324, "interval": null, "isNew": true, "legend": { @@ -45604,341 +46136,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 321, - "interval": null, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxDataPoints": null, - "maxPerRow": null, - "minSpan": null, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true, - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": null, - "seriesOverrides": [], - "span": null, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{req}} {{$additional_groupby}}", - "metric": "", - "query": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", - "refId": "", - "step": 10, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "KV Cursor Operations", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "transparent": false, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": 0 - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "datasource": "tidb-cluster", - "description": "", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "fill": 1, - "fillGradient": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "height": null, - "hideTimeOverride": false, - "id": 322, - "interval": null, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxDataPoints": null, - "maxPerRow": null, - "minSpan": null, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true, - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": null, - "seriesOverrides": [ - { - "alias": "count", - "bars": false, - "dashLength": 1, - "dashes": true, - "fill": 2, - "fillBelowTo": null, - "lines": true, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "bars": false, - "fill": 7, - "fillBelowTo": null, - "lines": true, - "yaxis": 1, - "zindex": 0 - } - ], - "span": null, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99%-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "count-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", - "refId": "", - "step": 10, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "KV Cursor Operations", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "transparent": false, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": 0 - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "datasource": "tidb-cluster", - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "fill": 1, - "fillGradient": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "height": null, - "hideTimeOverride": false, - "id": 323, + "id": 325, "interval": null, "isNew": true, "legend": { @@ -45981,15 +46179,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "expr": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "delete_skipped-{{req}} {{$additional_groupby}}", + "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "query": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -45998,7 +46196,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total RocksDB Perf Statistics", + "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, @@ -46044,7 +46242,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": null, + "description": "", "editable": true, "error": false, "fieldConfig": { @@ -46067,11 +46265,11 @@ "h": 7, "w": 12, "x": 12, - "y": 21 + "y": 14 }, "height": null, "hideTimeOverride": false, - "id": 324, + "id": 326, "interval": null, "isNew": true, "legend": { @@ -46107,22 +46305,90 @@ "renderer": "flot", "repeat": null, "repeatDirection": null, - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "count", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "avg", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%-{{req}} {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg-{{req}} {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count-{{req}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -46131,7 +46397,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total Response Size", + "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, @@ -46150,7 +46416,7 @@ "yaxes": [ { "decimals": null, - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -46177,7 +46443,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "Total bytes of memory used by coprocessor requests", + "description": null, "editable": true, "error": false, "fieldConfig": { @@ -46198,13 +46464,13 @@ }, "gridPos": { "h": 7, - "w": 24, + "w": 12, "x": 0, - "y": 28 + "y": 21 }, "height": null, "hideTimeOverride": false, - "id": 325, + "id": 327, "interval": null, "isNew": true, "legend": { @@ -46247,15 +46513,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", + "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "delete_skipped-{{req}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", + "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -46264,7 +46530,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Memory Quota", + "title": "Total RocksDB Perf Statistics", "tooltip": { "msResolution": true, "shared": true, @@ -46283,7 +46549,7 @@ "yaxes": [ { "decimals": null, - "format": "bytes", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -46304,55 +46570,13 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "repeatDirection": null, - "span": null, - "targets": [], - "timeFrom": null, - "timeShift": null, - "title": "Coprocessor Overview", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "cacheTimeout": null, - "collapsed": true, - "datasource": null, - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 326, - "interval": null, - "links": [], - "maxDataPoints": 100, - "maxPerRow": null, - "minSpan": null, - "panels": [ + }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The time consumed when handling coprocessor requests", + "description": null, "editable": true, "error": false, "fieldConfig": { @@ -46374,12 +46598,12 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 0 + "x": 12, + "y": 21 }, "height": null, "hideTimeOverride": false, - "id": 327, + "id": 328, "interval": null, "isNew": true, "legend": { @@ -46415,90 +46639,22 @@ "renderer": "flot", "repeat": null, "repeatDirection": null, - "seriesOverrides": [ - { - "alias": "count", - "bars": false, - "dashLength": 1, - "dashes": true, - "fill": 2, - "fillBelowTo": null, - "lines": true, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "bars": false, - "fill": 7, - "fillBelowTo": null, - "lines": true, - "yaxis": 1, - "zindex": 0 - } - ], + "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "expr": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "99%-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg-{{req}} {{$additional_groupby}}", - "metric": "", - "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "count-{{req}} {{$additional_groupby}}", + "legendFormat": "{{instance}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "query": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" @@ -46507,7 +46663,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Handle duration", + "title": "Total Response Size", "tooltip": { "msResolution": true, "shared": true, @@ -46526,7 +46682,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -46553,7 +46709,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The time consumed to handle coprocessor requests per TiKV instance", + "description": "Total bytes of memory used by coprocessor requests", "editable": true, "error": false, "fieldConfig": { @@ -46574,13 +46730,13 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 0 + "w": 24, + "x": 0, + "y": 28 }, "height": null, "hideTimeOverride": false, - "id": 328, + "id": 329, "interval": null, "isNew": true, "legend": { @@ -46616,90 +46772,22 @@ "renderer": "flot", "repeat": null, "repeatDirection": null, - "seriesOverrides": [ - { - "alias": "count", - "bars": false, - "dashLength": 1, - "dashes": true, - "fill": 2, - "fillBelowTo": null, - "lines": true, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "bars": false, - "fill": 7, - "fillBelowTo": null, - "lines": true, - "yaxis": 1, - "zindex": 0 - } - ], + "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99.99%-{{req}}-{{instance}}", - "metric": "", - "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "expr": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "99%-{{req}}-{{instance}}", - "metric": "", - "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg-{{req}}-{{instance}}", - "metric": "", - "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "count-{{req}}-{{instance}}", + "legendFormat": "{{instance}}-{{type}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", + "query": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" @@ -46708,7 +46796,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Handle duration by store", + "title": "Memory Quota", "tooltip": { "msResolution": true, "shared": true, @@ -46727,7 +46815,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -46748,13 +46836,55 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "repeatDirection": null, + "span": null, + "targets": [], + "timeFrom": null, + "timeShift": null, + "title": "Coprocessor Overview", + "transformations": [], + "transparent": false, + "type": "row" + }, + { + "cacheTimeout": null, + "collapsed": true, + "datasource": null, + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 330, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The time consumed when coprocessor requests are wait for being handled", + "description": "The time consumed when handling coprocessor requests", "editable": true, "error": false, "fieldConfig": { @@ -46777,11 +46907,11 @@ "h": 7, "w": 12, "x": 0, - "y": 7 + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 329, + "id": 331, "interval": null, "isNew": true, "legend": { @@ -46847,7 +46977,7 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, @@ -46855,14 +46985,14 @@ "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", - "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, @@ -46870,14 +47000,14 @@ "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", - "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", + "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, @@ -46885,14 +47015,14 @@ "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", - "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", + "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, @@ -46900,7 +47030,7 @@ "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -46909,7 +47039,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Wait duration", + "title": "Handle duration", "tooltip": { "msResolution": true, "shared": true, @@ -46955,7 +47085,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", + "description": "The time consumed to handle coprocessor requests per TiKV instance", "editable": true, "error": false, "fieldConfig": { @@ -46978,11 +47108,11 @@ "h": 7, "w": 12, "x": 12, - "y": 7 + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 330, + "id": 332, "interval": null, "isNew": true, "legend": { @@ -47048,7 +47178,7 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, @@ -47056,14 +47186,14 @@ "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", - "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, @@ -47071,14 +47201,14 @@ "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", - "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", + "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, @@ -47086,14 +47216,14 @@ "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", - "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", + "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", + "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, @@ -47101,7 +47231,7 @@ "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", + "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" @@ -47110,7 +47240,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Wait duration by store", + "title": "Handle duration by store", "tooltip": { "msResolution": true, "shared": true, @@ -47156,7 +47286,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": null, + "description": "The time consumed when coprocessor requests are wait for being handled", "editable": true, "error": false, "fieldConfig": { @@ -47179,11 +47309,11 @@ "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 331, + "id": 333, "interval": null, "isNew": true, "legend": { @@ -47219,22 +47349,90 @@ "renderer": "flot", "repeat": null, "repeatDirection": null, - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "count", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "avg", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{vec_type}} {{$additional_groupby}}", + "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%-{{req}} {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg-{{req}} {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count-{{req}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -47243,7 +47441,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total DAG Requests", + "title": "Wait duration", "tooltip": { "msResolution": true, "shared": true, @@ -47262,7 +47460,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -47289,7 +47487,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The total number of DAG executors", + "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", "editable": true, "error": false, "fieldConfig": { @@ -47312,11 +47510,11 @@ "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 332, + "id": 334, "interval": null, "isNew": true, "legend": { @@ -47352,22 +47550,90 @@ "renderer": "flot", "repeat": null, "repeatDirection": null, - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "count", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "avg", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}} {{$additional_groupby}}", + "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%-{{req}}-{{instance}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg-{{req}}-{{instance}}", + "metric": "", + "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count-{{req}}-{{instance}}", + "metric": "", + "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" @@ -47376,7 +47642,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total DAG Executors", + "title": "Wait duration by store", "tooltip": { "msResolution": true, "shared": true, @@ -47395,7 +47661,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -47445,11 +47711,11 @@ "h": 7, "w": 12, "x": 0, - "y": 21 + "y": 14 }, "height": null, "hideTimeOverride": false, - "id": 333, + "id": 335, "interval": null, "isNew": true, "legend": { @@ -47492,15 +47758,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", + "expr": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{tag}} {{$additional_groupby}}", + "legendFormat": "{{vec_type}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", + "query": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -47509,7 +47775,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total Ops Details (Table Scan)", + "title": "Total DAG Requests", "tooltip": { "msResolution": true, "shared": true, @@ -47528,7 +47794,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -47555,7 +47821,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": null, + "description": "The total number of DAG executors", "editable": true, "error": false, "fieldConfig": { @@ -47578,11 +47844,11 @@ "h": 7, "w": 12, "x": 12, - "y": 21 + "y": 14 }, "height": null, "hideTimeOverride": false, - "id": 334, + "id": 336, "interval": null, "isNew": true, "legend": { @@ -47625,15 +47891,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", + "expr": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{tag}} {{$additional_groupby}}", + "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", + "query": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -47642,7 +47908,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total Ops Details (Index Scan)", + "title": "Total DAG Executors", "tooltip": { "msResolution": true, "shared": true, @@ -47711,11 +47977,11 @@ "h": 7, "w": 12, "x": 0, - "y": 28 + "y": 21 }, "height": null, "hideTimeOverride": false, - "id": 335, + "id": 337, "interval": null, "isNew": true, "legend": { @@ -47758,15 +48024,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", + "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}", + "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", + "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -47775,7 +48041,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Total Ops Details by CF (Table Scan)", + "title": "Total Ops Details (Table Scan)", "tooltip": { "msResolution": true, "shared": true, @@ -47844,11 +48110,277 @@ "h": 7, "w": 12, "x": 12, - "y": 28 + "y": 21 }, "height": null, "hideTimeOverride": false, - "id": 336, + "id": 338, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tag}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Ops Details (Index Scan)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "height": null, + "hideTimeOverride": false, + "id": 339, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Ops Details by CF (Table Scan)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "height": null, + "hideTimeOverride": false, + "id": 340, "interval": null, "isNew": true, "legend": { @@ -47988,7 +48520,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 337, + "id": 341, "interval": null, "legend": { "show": false @@ -48086,7 +48618,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 338, + "id": 342, "interval": null, "isNew": true, "legend": { @@ -48290,7 +48822,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 339, + "id": 343, "interval": null, "links": [], "maxDataPoints": 100, @@ -48329,7 +48861,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 340, + "id": 344, "interval": null, "isNew": true, "legend": { @@ -48462,7 +48994,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 341, + "id": 345, "interval": null, "isNew": true, "legend": { @@ -48595,7 +49127,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 342, + "id": 346, "interval": null, "isNew": true, "legend": { @@ -48735,7 +49267,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 343, + "id": 347, "interval": null, "legend": { "show": false @@ -48833,7 +49365,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 344, + "id": 348, "interval": null, "isNew": true, "legend": { @@ -49034,7 +49566,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 345, + "id": 349, "interval": null, "isNew": true, "legend": { @@ -49235,7 +49767,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 346, + "id": 350, "interval": null, "isNew": true, "legend": { @@ -49439,7 +49971,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 347, + "id": 351, "interval": null, "links": [], "maxDataPoints": 100, @@ -49478,7 +50010,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 348, + "id": 352, "interval": null, "isNew": true, "legend": { @@ -49656,7 +50188,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 349, + "id": 353, "interval": null, "isNew": true, "legend": { @@ -49857,7 +50389,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 350, + "id": 354, "interval": null, "isNew": true, "legend": { @@ -49990,7 +50522,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 351, + "id": 355, "interval": null, "isNew": true, "legend": { @@ -50123,7 +50655,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 352, + "id": 356, "interval": null, "isNew": true, "legend": { @@ -50256,7 +50788,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 353, + "id": 357, "interval": null, "isNew": true, "legend": { @@ -50389,7 +50921,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 354, + "id": 358, "interval": null, "isNew": true, "legend": { @@ -50522,7 +51054,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 355, + "id": 359, "interval": null, "isNew": true, "legend": { @@ -50651,7 +51183,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 356, + "id": 360, "interval": null, "links": [], "maxDataPoints": 100, @@ -50726,7 +51258,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 357, + "id": 361, "interval": null, "links": [], "maxDataPoints": 100, @@ -50805,7 +51337,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 358, + "id": 362, "interval": null, "isNew": true, "legend": { @@ -51058,7 +51590,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 359, + "id": 363, "interval": null, "isNew": true, "legend": { @@ -51191,7 +51723,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 360, + "id": 364, "interval": null, "isNew": true, "legend": { @@ -51327,7 +51859,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 361, + "id": 365, "interval": null, "links": [], "maxDataPoints": 100, @@ -51366,7 +51898,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 362, + "id": 366, "interval": null, "isNew": true, "legend": { @@ -51514,7 +52046,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 363, + "id": 367, "interval": null, "isNew": true, "legend": { @@ -51647,7 +52179,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 364, + "id": 368, "interval": null, "isNew": true, "legend": { @@ -51848,7 +52380,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 365, + "id": 369, "interval": null, "isNew": true, "legend": { @@ -51996,7 +52528,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 366, + "id": 370, "interval": null, "isNew": true, "legend": { @@ -52197,7 +52729,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 367, + "id": 371, "interval": null, "isNew": true, "legend": { @@ -52330,7 +52862,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 368, + "id": 372, "interval": null, "isNew": true, "legend": { @@ -52463,7 +52995,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 369, + "id": 373, "interval": null, "isNew": true, "legend": { @@ -52596,7 +53128,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 370, + "id": 374, "interval": null, "isNew": true, "legend": { @@ -52729,7 +53261,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 371, + "id": 375, "interval": null, "isNew": true, "legend": { @@ -52869,7 +53401,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 372, + "id": 376, "interval": null, "legend": { "show": false @@ -52967,383 +53499,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 373, - "interval": null, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxDataPoints": null, - "maxPerRow": null, - "minSpan": null, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true, - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": null, - "seriesOverrides": [ - { - "alias": "count", - "bars": false, - "dashLength": 1, - "dashes": true, - "fill": 2, - "fillBelowTo": null, - "lines": true, - "spaceLength": 1, - "transform": "negative-Y", - "yaxis": 2, - "zindex": -3 - }, - { - "alias": "avg", - "bars": false, - "fill": 7, - "fillBelowTo": null, - "lines": true, - "yaxis": 1, - "zindex": 0 - } - ], - "span": null, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", - "metric": "", - "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "99%-{{type}} {{$additional_groupby}}", - "metric": "", - "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "avg-{{type}} {{$additional_groupby}}", - "metric": "", - "query": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", - "refId": "", - "step": 10, - "target": "" - }, - { - "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "count-{{type}} {{$additional_groupby}}", - "metric": "", - "query": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", - "refId": "", - "step": 10, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "In-memory scan lock read duration", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "transparent": false, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "s", - "label": null, - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": 0 - } - } - ], - "repeat": null, - "repeatDirection": null, - "span": null, - "targets": [], - "timeFrom": null, - "timeShift": null, - "title": "Pessimistic Locking", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "cacheTimeout": null, - "collapsed": true, - "datasource": null, - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 374, - "interval": null, - "links": [], - "maxDataPoints": 100, - "maxPerRow": null, - "minSpan": null, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "datasource": "tidb-cluster", - "description": "The number of tasks handled by worker", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "fill": 1, - "fillGradient": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 375, - "interval": null, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "maxDataPoints": null, - "maxPerRow": null, - "minSpan": null, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true, - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": null, - "seriesOverrides": [], - "span": null, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{name}} {{$additional_groupby}}", - "metric": "", - "query": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", - "refId": "", - "step": 10, - "target": "" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Worker handled tasks", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "transparent": false, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": 0 - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "datasource": "tidb-cluster", - "description": "Current pending and running tasks of worker", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "fill": 1, - "fillGradient": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 376, + "id": 377, "interval": null, "isNew": true, "legend": { @@ -53379,22 +53535,90 @@ "renderer": "flot", "repeat": null, "repeatDirection": null, - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "count", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "avg", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "tidb-cluster", - "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) ", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{name}} {{$additional_groupby}}", + "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", - "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) ", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%-{{type}} {{$additional_groupby}}", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg-{{type}} {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count-{{type}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -53403,7 +53627,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Worker pending tasks", + "title": "In-memory scan lock read duration", "tooltip": { "msResolution": true, "shared": true, @@ -53422,9 +53646,9 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -53443,13 +53667,55 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "repeatDirection": null, + "span": null, + "targets": [], + "timeFrom": null, + "timeShift": null, + "title": "Pessimistic Locking", + "transformations": [], + "transparent": false, + "type": "row" + }, + { + "cacheTimeout": null, + "collapsed": true, + "datasource": null, + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 378, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The number of tasks handled by future_pool", + "description": "The number of tasks handled by worker", "editable": true, "error": false, "fieldConfig": { @@ -53472,11 +53738,11 @@ "h": 7, "w": 12, "x": 0, - "y": 7 + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 377, + "id": 379, "interval": null, "isNew": true, "legend": { @@ -53519,7 +53785,7 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", + "expr": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, @@ -53527,7 +53793,7 @@ "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", + "query": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -53536,7 +53802,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "FuturePool handled tasks", + "title": "Worker handled tasks", "tooltip": { "msResolution": true, "shared": true, @@ -53582,7 +53848,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "Current pending and running tasks of future_pool", + "description": "Current pending and running tasks of worker", "editable": true, "error": false, "fieldConfig": { @@ -53605,11 +53871,11 @@ "h": 7, "w": 12, "x": 12, - "y": 7 + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 378, + "id": 380, "interval": null, "isNew": true, "legend": { @@ -53652,7 +53918,7 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) ", + "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, @@ -53660,7 +53926,7 @@ "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", - "query": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) ", + "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -53669,7 +53935,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "FuturePool pending tasks", + "title": "Worker pending tasks", "tooltip": { "msResolution": true, "shared": true, @@ -53709,55 +53975,13 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "repeatDirection": null, - "span": null, - "targets": [], - "timeFrom": null, - "timeShift": null, - "title": "Task", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "cacheTimeout": null, - "collapsed": true, - "datasource": null, - "description": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "height": null, - "hideTimeOverride": false, - "id": 379, - "interval": null, - "links": [], - "maxDataPoints": 100, - "maxPerRow": null, - "minSpan": null, - "panels": [ + }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The count of requests that TiKV sends to PD", + "description": "The number of tasks handled by future_pool", "editable": true, "error": false, "fieldConfig": { @@ -53780,11 +54004,11 @@ "h": 7, "w": 12, "x": 0, - "y": 0 + "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 380, + "id": 381, "interval": null, "isNew": true, "legend": { @@ -53827,15 +54051,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "expr": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}} {{$additional_groupby}}", + "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", - "query": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "query": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -53844,7 +54068,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "PD requests", + "title": "FuturePool handled tasks", "tooltip": { "msResolution": true, "shared": true, @@ -53890,7 +54114,7 @@ "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The time consumed by requests that TiKV sends to PD", + "description": "Current pending and running tasks of future_pool", "editable": true, "error": false, "fieldConfig": { @@ -53913,11 +54137,11 @@ "h": 7, "w": 12, "x": 12, - "y": 0 + "y": 7 }, "height": null, "hideTimeOverride": false, - "id": 381, + "id": 382, "interval": null, "isNew": true, "legend": { @@ -53960,15 +54184,15 @@ "targets": [ { "datasource": "tidb-cluster", - "expr": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", + "expr": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}} {{$additional_groupby}}", + "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", - "query": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", + "query": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" @@ -53977,7 +54201,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "PD request duration (average)", + "title": "FuturePool pending tasks", "tooltip": { "msResolution": true, "shared": true, @@ -53996,7 +54220,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -54017,13 +54241,55 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "repeatDirection": null, + "span": null, + "targets": [], + "timeFrom": null, + "timeShift": null, + "title": "Task", + "transformations": [], + "transparent": false, + "type": "row" + }, + { + "cacheTimeout": null, + "collapsed": true, + "datasource": null, + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 383, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "tidb-cluster", - "description": "The total number of PD heartbeat messages", + "description": "The count of requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { @@ -54046,11 +54312,277 @@ "h": 7, "w": 12, "x": 0, - "y": 7 + "y": 0 }, "height": null, "hideTimeOverride": false, - "id": 382, + "id": 384, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}} {{$additional_groupby}}", + "metric": "", + "query": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PD requests", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": "The time consumed by requests that TiKV sends to PD", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "height": null, + "hideTimeOverride": false, + "id": 385, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "tidb-cluster", + "expr": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}} {{$additional_groupby}}", + "metric": "", + "query": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PD request duration (average)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "tidb-cluster", + "description": "The total number of PD heartbeat messages", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "height": null, + "hideTimeOverride": false, + "id": 386, "interval": null, "isNew": true, "legend": { @@ -54198,7 +54730,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 383, + "id": 387, "interval": null, "isNew": true, "legend": { @@ -54331,7 +54863,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 384, + "id": 388, "interval": null, "isNew": true, "legend": { @@ -54464,7 +54996,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 385, + "id": 389, "interval": null, "isNew": true, "legend": { @@ -54597,7 +55129,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 386, + "id": 390, "interval": null, "isNew": true, "legend": { @@ -54733,7 +55265,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 387, + "id": 391, "interval": null, "links": [], "maxDataPoints": 100, @@ -54772,7 +55304,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 388, + "id": 392, "interval": null, "isNew": true, "legend": { @@ -54905,7 +55437,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 389, + "id": 393, "interval": null, "isNew": true, "legend": { @@ -55038,7 +55570,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 390, + "id": 394, "interval": null, "isNew": true, "legend": { @@ -55171,7 +55703,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 391, + "id": 395, "interval": null, "isNew": true, "legend": { @@ -55304,7 +55836,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 392, + "id": 396, "interval": null, "isNew": true, "legend": { @@ -55437,7 +55969,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 393, + "id": 397, "interval": null, "isNew": true, "legend": { @@ -55573,7 +56105,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 394, + "id": 398, "interval": null, "links": [], "maxDataPoints": 100, @@ -55612,7 +56144,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 395, + "id": 399, "interval": null, "isNew": true, "legend": { @@ -55745,7 +56277,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 396, + "id": 400, "interval": null, "isNew": true, "legend": { @@ -55878,7 +56410,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 397, + "id": 401, "interval": null, "isNew": true, "legend": { @@ -56026,7 +56558,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 398, + "id": 402, "interval": null, "isNew": true, "legend": { @@ -56189,7 +56721,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 399, + "id": 403, "interval": null, "isNew": true, "legend": { @@ -56322,7 +56854,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 400, + "id": 404, "interval": null, "isNew": true, "legend": { @@ -56455,7 +56987,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 401, + "id": 405, "interval": null, "isNew": true, "legend": { @@ -56603,7 +57135,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 402, + "id": 406, "interval": null, "isNew": true, "legend": { @@ -56751,7 +57283,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 403, + "id": 407, "interval": null, "isNew": true, "legend": { @@ -56887,7 +57419,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 404, + "id": 408, "interval": null, "links": [], "maxDataPoints": 100, @@ -56926,7 +57458,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 405, + "id": 409, "interval": null, "isNew": true, "legend": { @@ -57059,7 +57591,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 406, + "id": 410, "interval": null, "isNew": true, "legend": { @@ -57192,7 +57724,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 407, + "id": 411, "interval": null, "isNew": true, "legend": { @@ -57325,7 +57857,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 408, + "id": 412, "interval": null, "isNew": true, "legend": { @@ -57458,7 +57990,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 409, + "id": 413, "interval": null, "isNew": true, "legend": { @@ -57591,7 +58123,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 410, + "id": 414, "interval": null, "isNew": true, "legend": { @@ -57724,7 +58256,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 411, + "id": 415, "interval": null, "isNew": true, "legend": { @@ -57857,7 +58389,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 412, + "id": 416, "interval": null, "isNew": true, "legend": { @@ -57990,7 +58522,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 413, + "id": 417, "interval": null, "isNew": true, "legend": { @@ -58130,7 +58662,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 414, + "id": 418, "interval": null, "legend": { "show": false @@ -58228,7 +58760,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 415, + "id": 419, "interval": null, "isNew": true, "legend": { @@ -58361,7 +58893,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 416, + "id": 420, "interval": null, "isNew": true, "legend": { @@ -58509,7 +59041,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 417, + "id": 421, "interval": null, "isNew": true, "legend": { @@ -58657,7 +59189,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 418, + "id": 422, "interval": null, "isNew": true, "legend": { @@ -58797,7 +59329,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 419, + "id": 423, "interval": null, "legend": { "show": false @@ -58895,7 +59427,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 420, + "id": 424, "interval": null, "isNew": true, "legend": { @@ -59028,7 +59560,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 421, + "id": 425, "interval": null, "isNew": true, "legend": { @@ -59164,7 +59696,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 422, + "id": 426, "interval": null, "links": [], "maxDataPoints": 100, @@ -59203,7 +59735,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 423, + "id": 427, "interval": null, "isNew": true, "legend": { @@ -59336,7 +59868,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 424, + "id": 428, "interval": null, "isNew": true, "legend": { @@ -59499,7 +60031,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 425, + "id": 429, "interval": null, "isNew": true, "legend": { @@ -59647,7 +60179,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 426, + "id": 430, "interval": null, "isNew": true, "legend": { @@ -59780,7 +60312,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 427, + "id": 431, "interval": null, "isNew": true, "legend": { @@ -59920,7 +60452,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 428, + "id": 432, "interval": null, "legend": { "show": false @@ -60025,7 +60557,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 429, + "id": 433, "interval": null, "legend": { "show": false @@ -60130,7 +60662,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 430, + "id": 434, "interval": null, "legend": { "show": false @@ -60228,7 +60760,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 431, + "id": 435, "interval": null, "isNew": true, "legend": { @@ -60368,7 +60900,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 432, + "id": 436, "interval": null, "legend": { "show": false @@ -60473,7 +61005,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 433, + "id": 437, "interval": null, "legend": { "show": false @@ -60578,7 +61110,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 434, + "id": 438, "interval": null, "legend": { "show": false @@ -60676,7 +61208,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 435, + "id": 439, "interval": null, "isNew": true, "legend": { @@ -60809,7 +61341,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 436, + "id": 440, "interval": null, "isNew": true, "legend": { @@ -60942,7 +61474,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 437, + "id": 441, "interval": null, "isNew": true, "legend": { @@ -61082,7 +61614,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 438, + "id": 442, "interval": null, "legend": { "show": false @@ -61180,7 +61712,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 439, + "id": 443, "interval": null, "isNew": true, "legend": { @@ -61316,7 +61848,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 440, + "id": 444, "interval": null, "links": [], "maxDataPoints": 100, @@ -61355,7 +61887,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 441, + "id": 445, "interval": null, "isNew": true, "legend": { @@ -61518,7 +62050,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 442, + "id": 446, "interval": null, "isNew": true, "legend": { @@ -61651,7 +62183,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 443, + "id": 447, "interval": null, "isNew": true, "legend": { @@ -61791,7 +62323,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 444, + "id": 448, "interval": null, "legend": { "show": false @@ -61896,7 +62428,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 445, + "id": 449, "interval": null, "legend": { "show": false @@ -61994,7 +62526,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 446, + "id": 450, "interval": null, "isNew": true, "legend": { @@ -62149,7 +62681,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 447, + "id": 451, "interval": null, "legend": { "show": false @@ -62254,7 +62786,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 448, + "id": 452, "interval": null, "legend": { "show": false @@ -62359,7 +62891,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 449, + "id": 453, "interval": null, "legend": { "show": false @@ -62457,7 +62989,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 450, + "id": 454, "interval": null, "isNew": true, "legend": { @@ -62627,7 +63159,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 451, + "id": 455, "interval": null, "legend": { "show": false @@ -62725,7 +63257,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 452, + "id": 456, "interval": null, "isNew": true, "legend": { @@ -62926,7 +63458,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 453, + "id": 457, "interval": null, "isNew": true, "legend": { @@ -63127,7 +63659,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 454, + "id": 458, "interval": null, "isNew": true, "legend": { @@ -63260,7 +63792,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 455, + "id": 459, "interval": null, "isNew": true, "legend": { @@ -63423,7 +63955,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 456, + "id": 460, "interval": null, "isNew": true, "legend": { @@ -63556,7 +64088,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 457, + "id": 461, "interval": null, "isNew": true, "legend": { @@ -63689,7 +64221,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 458, + "id": 462, "interval": null, "isNew": true, "legend": { @@ -63890,7 +64422,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 459, + "id": 463, "interval": null, "isNew": true, "legend": { @@ -64023,7 +64555,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 460, + "id": 464, "interval": null, "isNew": true, "legend": { @@ -64163,7 +64695,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 461, + "id": 465, "interval": null, "legend": { "show": false @@ -64268,7 +64800,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 462, + "id": 466, "interval": null, "legend": { "show": false @@ -64373,7 +64905,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 463, + "id": 467, "interval": null, "legend": { "show": false @@ -64478,7 +65010,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 464, + "id": 468, "interval": null, "legend": { "show": false @@ -64583,7 +65115,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 465, + "id": 469, "interval": null, "legend": { "show": false @@ -64688,7 +65220,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 466, + "id": 470, "interval": null, "legend": { "show": false @@ -64793,7 +65325,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 467, + "id": 471, "interval": null, "legend": { "show": false @@ -64891,7 +65423,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 468, + "id": 472, "interval": null, "isNew": true, "legend": { @@ -65039,7 +65571,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 469, + "id": 473, "interval": null, "isNew": true, "legend": { @@ -65172,7 +65704,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 470, + "id": 474, "interval": null, "isNew": true, "legend": { @@ -65305,7 +65837,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 471, + "id": 475, "interval": null, "isNew": true, "legend": { @@ -65453,7 +65985,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 472, + "id": 476, "interval": null, "isNew": true, "legend": { @@ -65589,7 +66121,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 473, + "id": 477, "interval": null, "links": [], "maxDataPoints": 100, @@ -65640,7 +66172,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 474, + "id": 478, "interval": null, "links": [], "maxDataPoints": 100, @@ -65736,7 +66268,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 475, + "id": 479, "interval": null, "links": [], "maxDataPoints": 100, @@ -65811,7 +66343,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 476, + "id": 480, "interval": null, "links": [], "maxDataPoints": 100, @@ -65886,7 +66418,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 477, + "id": 481, "interval": null, "links": [], "maxDataPoints": 100, @@ -65961,7 +66493,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 478, + "id": 482, "interval": null, "links": [], "maxDataPoints": 100, @@ -66036,7 +66568,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 479, + "id": 483, "interval": null, "links": [], "maxDataPoints": 100, @@ -66111,7 +66643,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 480, + "id": 484, "interval": null, "links": [], "maxDataPoints": 100, @@ -66186,7 +66718,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 481, + "id": 485, "interval": null, "links": [], "maxDataPoints": 100, @@ -66265,7 +66797,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 482, + "id": 486, "interval": null, "isNew": true, "legend": { @@ -66398,7 +66930,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 483, + "id": 487, "interval": null, "isNew": true, "legend": { @@ -66531,7 +67063,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 484, + "id": 488, "interval": null, "isNew": true, "legend": { @@ -66664,7 +67196,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 485, + "id": 489, "interval": null, "isNew": true, "legend": { @@ -66797,7 +67329,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 486, + "id": 490, "interval": null, "isNew": true, "legend": { @@ -66930,7 +67462,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 487, + "id": 491, "interval": null, "isNew": true, "legend": { @@ -67078,7 +67610,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 488, + "id": 492, "interval": null, "isNew": true, "legend": { @@ -67211,7 +67743,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 489, + "id": 493, "interval": null, "isNew": true, "legend": { @@ -67344,7 +67876,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 490, + "id": 494, "interval": null, "isNew": true, "legend": { @@ -67510,7 +68042,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 491, + "id": 495, "interval": null, "legend": { "show": false @@ -67615,7 +68147,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 492, + "id": 496, "interval": null, "legend": { "show": false @@ -67720,7 +68252,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 493, + "id": 497, "interval": null, "legend": { "show": false @@ -67825,7 +68357,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 494, + "id": 498, "interval": null, "legend": { "show": false @@ -67930,7 +68462,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 495, + "id": 499, "interval": null, "legend": { "show": false @@ -68035,7 +68567,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 496, + "id": 500, "interval": null, "legend": { "show": false @@ -68140,7 +68672,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 497, + "id": 501, "interval": null, "legend": { "show": false @@ -68245,7 +68777,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 498, + "id": 502, "interval": null, "legend": { "show": false @@ -68343,7 +68875,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 499, + "id": 503, "interval": null, "isNew": true, "legend": { @@ -68476,7 +69008,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 500, + "id": 504, "interval": null, "isNew": true, "legend": { @@ -68609,7 +69141,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 501, + "id": 505, "interval": null, "isNew": true, "legend": { @@ -68742,7 +69274,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 502, + "id": 506, "interval": null, "isNew": true, "legend": { @@ -68875,7 +69407,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 503, + "id": 507, "interval": null, "isNew": true, "legend": { @@ -69008,7 +69540,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 504, + "id": 508, "interval": null, "isNew": true, "legend": { @@ -69141,7 +69673,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 505, + "id": 509, "interval": null, "isNew": true, "legend": { @@ -69274,7 +69806,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 506, + "id": 510, "interval": null, "isNew": true, "legend": { @@ -69414,7 +69946,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 507, + "id": 511, "interval": null, "legend": { "show": false @@ -69519,7 +70051,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 508, + "id": 512, "interval": null, "legend": { "show": false @@ -69617,7 +70149,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 509, + "id": 513, "interval": null, "isNew": true, "legend": { @@ -69750,7 +70282,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 510, + "id": 514, "interval": null, "isNew": true, "legend": { @@ -69883,7 +70415,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 511, + "id": 515, "interval": null, "isNew": true, "legend": { @@ -70016,7 +70548,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 512, + "id": 516, "interval": null, "isNew": true, "legend": { @@ -70149,7 +70681,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 513, + "id": 517, "interval": null, "isNew": true, "legend": { @@ -70282,7 +70814,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 514, + "id": 518, "interval": null, "isNew": true, "legend": { @@ -70418,7 +70950,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 515, + "id": 519, "interval": null, "links": [], "maxDataPoints": 100, @@ -70457,7 +70989,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 516, + "id": 520, "interval": null, "isNew": true, "legend": { @@ -70605,7 +71137,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 517, + "id": 521, "interval": null, "isNew": true, "legend": { @@ -70738,7 +71270,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 518, + "id": 522, "interval": null, "isNew": true, "legend": { @@ -70871,7 +71403,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 519, + "id": 523, "interval": null, "isNew": true, "legend": { @@ -71007,7 +71539,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 520, + "id": 524, "interval": null, "links": [], "maxDataPoints": 100, @@ -71046,7 +71578,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 521, + "id": 525, "interval": null, "isNew": true, "legend": { @@ -71179,7 +71711,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 522, + "id": 526, "interval": null, "isNew": true, "legend": { @@ -71312,7 +71844,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 523, + "id": 527, "interval": null, "isNew": true, "legend": { @@ -71445,7 +71977,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 524, + "id": 528, "interval": null, "isNew": true, "legend": { @@ -71578,7 +72110,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 525, + "id": 529, "interval": null, "isNew": true, "legend": { @@ -71711,7 +72243,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 526, + "id": 530, "interval": null, "isNew": true, "legend": { @@ -71847,7 +72379,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 527, + "id": 531, "interval": null, "links": [], "maxDataPoints": 100, @@ -71886,7 +72418,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 528, + "id": 532, "interval": null, "isNew": true, "legend": { @@ -72019,7 +72551,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 529, + "id": 533, "interval": null, "isNew": true, "legend": { @@ -72155,7 +72687,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 530, + "id": 534, "interval": null, "links": [], "maxDataPoints": 100, @@ -72194,7 +72726,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 531, + "id": 535, "interval": null, "isNew": true, "legend": { @@ -72395,7 +72927,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 532, + "id": 536, "interval": null, "isNew": true, "legend": { @@ -72531,7 +73063,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 533, + "id": 537, "interval": null, "links": [], "maxDataPoints": 100, @@ -72570,7 +73102,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 534, + "id": 538, "interval": null, "isNew": true, "legend": { @@ -72703,7 +73235,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 535, + "id": 539, "interval": null, "isNew": true, "legend": { @@ -72836,7 +73368,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 536, + "id": 540, "interval": null, "isNew": true, "legend": { @@ -72969,7 +73501,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 537, + "id": 541, "interval": null, "isNew": true, "legend": { @@ -73102,7 +73634,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 538, + "id": 542, "interval": null, "isNew": true, "legend": { @@ -73250,7 +73782,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 539, + "id": 543, "interval": null, "isNew": true, "legend": { @@ -73454,7 +73986,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 540, + "id": 544, "interval": null, "links": [], "maxDataPoints": 100, @@ -73493,7 +74025,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 541, + "id": 545, "interval": null, "isNew": true, "legend": { @@ -73626,7 +74158,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 542, + "id": 546, "interval": null, "isNew": true, "legend": { @@ -73759,7 +74291,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 543, + "id": 547, "interval": null, "isNew": true, "legend": { @@ -73892,7 +74424,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 544, + "id": 548, "interval": null, "isNew": true, "legend": { @@ -74025,7 +74557,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 545, + "id": 549, "interval": null, "isNew": true, "legend": { @@ -74222,7 +74754,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 546, + "id": 550, "interval": null, "links": [], "maxDataPoints": 100,