diff --git a/grafana/scylla-cpu.2.3.template.json b/grafana/scylla-cpu.2.3.template.json deleted file mode 100644 index 8d4967ba0..000000000 --- a/grafana/scylla-cpu.2.3.template.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "cpu-2-3", - "rows": [ - { - "class": "logo_row", - "panels": [ - { - "class": "text_panel", - "content": "\n
" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "percent_panel", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "CPU Utilization per [[by]]", - "description" : "the percentage of the time during which the CPU is utilized by Scylla. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 3, - "style": {} - }, - { - "class": "dashlist", - "tags": [ - "2.3" - ] - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "percent_panel", - "pointradius": 1, - "targets": [ - { - "expr": "avg(irate(scylla_scheduler_runtime_ms{group=\"main\",instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])/10 + avg(irate(scylla_scheduler_runtime_ms{group=\"statement\",instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])/10", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Foreground CPU Utilization by [[by]]", - "description": "Time spent handling foreground requests (like reads, writes, and some system tasks). The remaining time is either idle, or used by background load like compactions and repairs. Background load in Scylla is opportunistic: background requests will try to use all resources available to complete as fast as possible and rely on the schedulers to provide isolation. This graph is better understood in conjunction with the main CPU load graph. For example, if there are spikes in CPU load that are not present in this graph, that indicates that the foreground load itself is stable." - }, - { - "class": "ns_panel", - "pointradius": 1, - "targets": [ - { - "expr": "sum(irate(scylla_scheduler_time_spent_on_task_quota_violations_ns{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Time spent in task quota violations by [[by]]", - "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer. This may cause latency issues" - } - ], - "title": "New row" - } - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "Shard", - "value": "shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "type": "custom" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "template_variable_custom", - "current": { - "text": "2.3", - "value": "2.3" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "2.3", - "value": "2.3" - } - ], - "query": "2.3" - }, - { - "class": "monitor_version_var" - } - ] - }, - "tags": [ - "2.3" - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Scylla CPU Per Server Metrics", - "version": 5 - } -} diff --git a/grafana/scylla-cpu.3.0.template.json b/grafana/scylla-cpu.3.0.template.json deleted file mode 100644 index 30fb1af4b..000000000 --- a/grafana/scylla-cpu.3.0.template.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "cpu-3-0", - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "panels": [ - { - "class": "percent_panel", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "CPU Utilization per [[by]]", - "description" : "the percentage of the time during which the CPU is utilized by Scylla. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high" - }, - { - "class": "percent_panel", - "pointradius": 1, - "targets": [ - { - "expr": "avg(irate(scylla_scheduler_runtime_ms{group=\"main\",instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])/10 + avg(irate(scylla_scheduler_runtime_ms{group=\"statement\",instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])/10", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Foreground CPU Utilization by [[by]]", - "description": "Time spent handling foreground requests (like reads, writes, and some system tasks). The remaining time is either idle, or used by background load like compactions and repairs. Background load in Scylla is opportunistic: background requests will try to use all resources available to complete as fast as possible and rely on the schedulers to provide isolation. This graph is better understood in conjunction with the main CPU load graph. For example, if there are spikes in CPU load that are not present in this graph, that indicates that the foreground load itself is stable." - } - ] - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "plain_text", - "content": "

Task Quota Violation

" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "ms_panel", - "repeat": "group", - "span":3, - "pointradius": 1, - "targets": [ - { - "expr": "sum(irate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Time spent in task quota violations by [[by]] - $group", - "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer. This may cause latency issues" - } - ], - "title": "New row" - } - ], - "templating": { - "list": [ - { - "class":"by_template_var" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - } - , - { - "class": "template_variable_all", - "label": "group", - "name": "group", - "hide": 2, - "query": "label_values(scylla_scheduler_time_spent_on_task_quota_violations_ms,group)", - "sort": 3 - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "tags": [ - "3.0" - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "CPU Metrics", - "overwrite": true, - "version": 5 - } -} diff --git a/grafana/scylla-cql.3.0.template.json b/grafana/scylla-cql.3.0.template.json deleted file mode 100644 index 5afb59b20..000000000 --- a/grafana/scylla-cql.3.0.template.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "cql-3-0", - "originalTitle": "CQL", - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

CQL - Coordinator

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Insert" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Reads" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Deletes" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Updates" - }, - { - "class": "graph_panel", - "span": 3, - "pointradius": 1, - "targets": [ - { - "expr": "sum(scylla_transport_current_connections{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Client CQL connections by [[by]]", - "description": "amount of CQL connections currently established" - }, - { - "class": "graph_panel", - "span": 3, - "pointradius": 1, - "targets": [ - { - "expr": "sum(irate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Batches by [[by]]", - "description": "Number of CQL batches command, each batched command is counted once" - }, - { - "class": "graph_panel", - "span": 3, - "pointradius": 1, - "targets": [ - { - "expr": "sum(irate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "CQL Command In Batches by [[by]]", - "description": "Number of CQL command batched. Each batch would add the number of commands inside the batch" - }, - { - "class":"plain_text", - "content": "", - "span":3 - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Optimization

", - "style": {} - } - ], - "title": "New row" - }, - - { - "class": "row", - "panels": [ - - { - "class": "gauge_errors_panel", - "description": "All of the requests should be prepared\n\nPrepared statements remove the overhead of parsing the query every time and allow optimal routing of requests from client to server", - "targets": [ - { - "expr": "floor(100 *sum(irate(scylla_query_processor_statements_prepared{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) /(sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + sum(irate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + sum(irate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + sum(irate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "aaa", - "refId": "A" - } - ], - "title": "CQL Non-Prepared Statements" - }, - { - "class": "ops_panel", - "description": "All of the requests should be prepared\n", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_query_processor_statements_prepared{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "CQL Non-Prepared Statements" - }, - { - "class": "gauge_errors_panel", - "description": "All requests should be paged\n\nNon Paged request sources:\n- Client modifying the fetch size\n\nNon Paged requests require reading all the results and returning them in a single request.", - "targets": [ - { - "expr": "100 * (sum(irate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]))/sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]))) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "aaa", - "refId": "A" - } - ], - "title": "Non-Paged CQL Reads" - }, - { - "class": "ops_panel", - "span": 3, - "description": "Non-Paged requests require reading all the results and returning them in a single request", - "targets": [ - { - "expr": "sum(irate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Non-Paged CQL Reads" - }, - { - "class": "gauge_errors_panel", - "description": "All of the requests should be Token Aware\n\nNon Token Aware requests sources:\n* Non-Prepared Stamements\n* Client not using a Token Aware load balancing policy\n\nTokenAware requests are sent to a Scylla node that is also a replica. Token Un-Aware requests require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", - "targets": [ - { - "expr": "100 - floor(100*(sum(irate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) +sum(irate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])))/(sum(irate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) + irate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) + irate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) + irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])))) OR vector(0)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "aaa", - "refId": "A" - } - ], - "title": "Non-Token Aware" - }, - { - "class": "ops_panel", - "description": "Requests that are not token aware indicates that requests are not routed to the right node, which require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) - sum(irate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) + sum(irate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) + irate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) + irate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) - sum(irate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])\n", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Non-Token Aware Queries" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "gauge_errors_panel", - "description": "Reversed CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with ORDER BY that is different from the \"CLUSTERING ORDER BY\" of the table\nAlternatives:\n\n* Denormalize your data (use a Materialized View)", - "targets": [ - { - "expr": "100 * sum(irate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) / sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "aaa", - "refId": "A" - } - ], - "title": "Reversed CQL Reads" - }, - { - "class": "ops_panel", - "span": 3, - "description": "Reversed CQL Reads entail additional processing on server side and should be avoided", - "targets": [ - { - "expr": "sum(irate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Reversed CQL Reads" - }, - { - "class": "gauge_errors_panel", - "description": "ALLOW FILTERING CQL Reads, the percentage of read requests with 'ALLOW FILTERING'\n\nALLOW FILTERING CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned - check \n\"ALLOW FILTERING CQL Read Filtered Rows to check what percentage of the data is used\"\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", - "targets": [ - { - "expr": "100 * sum(irate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) / sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) OR vector(0)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "aaa", - "refId": "A" - } - ], - "title": "ALLOW FILTERING CQL Reads" - }, - { - "class": "rps_panel", - "span": 3, - "description": "Read requests with ALLOW FILTERING\n\nALLOW FILTERING CQL Reads entail additional processing on server side and should be avoided", - "targets": [ - { - "expr": "sum(irate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "ALLOW FILTERING CQL Reads" - }, - { - "class": "gauge_errors_panel", - "description": "ALLOW FILTERING Filtered rows, the percentage of rows that were read and then filtered.\n\nALLOW FILTERING CQL Reads entail additional processing on server side. \nReading a row and then filter it is a waste of resources.\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", - "targets": [ - { - "expr": "100 * sum(irate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) /sum(irate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) OR vector(0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "aaa", - "refId": "A" - } - ], - "title": "ALLOW FILTERING Filtered Rows" - }, - { - "class": "rps_panel", - "span": 3, - "description": "CQL Queries with ALLOW FILTERING should be avoided.\nDropped rows are rows that were read but were filtered by the server.\nWhen dropped rows is relatively high you should consider the alternatives", - "targets": [ - { - "expr": "sum(irate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "rows read $node $shard", - "refId": "A" - }, - { - "expr": "sum(irate(scylla_cql_filtered_rows_matched_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "rows matched $node $shard", - "refId": "B" - }, - { - "expr": "sum(irate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "rows dropped $node $shard", - "refId": "C" - } - ], - "title": "ALLOW FILTERING CQL Read Filtering" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "gauge_errors_panel", - "repeat": "dc", - "span": 2, - "description": "Cross DC traffic may cause additional latencies and network loads and in most cases, should be avoided.\n\nCross DC Read requests sources:\n- Consistency Level that is not LOCAL_XXX\n- Tables with read_repair_chance > 0\n\nNote:\n- If requests are supposed to be DC local - verify client is using a DCAware policy and a LOCAL_XX consistency level", - "targets": [ - { - "expr": "100*(sum(irate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) - sum(irate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", datacenter=~\"$dc\", shard=~\"[[shard]]\"}[60s])))/sum(irate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) OR vector(0)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Cross DC read requests $dc" - } - ] - } - ], - "tags": [ - "3.0" - ], - "templating": { - "list": [ - { - "class":"by_template_var" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "overwrite": true, - "title": "Scylla CQL" - } -} diff --git a/grafana/scylla-detailed.2.3.template.json b/grafana/scylla-detailed.2.3.template.json deleted file mode 100644 index f82725046..000000000 --- a/grafana/scylla-detailed.2.3.template.json +++ /dev/null @@ -1,1008 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "detail-2-3", - "rows": [ - { - "class": "logo_row", - "panels": [ - { - "class": "text_panel", - "content": "\n
" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "single_stat_panel", - "targets": [ - { - "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 1 - } - ], - "title": "Total Nodes" - }, - { - "class": "single_stat_panel_fail", - "targets": [ - { - "expr": "count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Unreachable", - "refId": "A", - "step": 1 - } - ], - "thresholds": "1,2", - "title": "Unreachable" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 3, - "style": {} - }, - { - "class": "ops_panel", - "bars": true, - "targets": [ - { - "expr": "sum(irate(scylla_transport_requests_served{cluster=~\"$cluster|$^\"}[60s])) + sum(irate(scylla_thrift_served{cluster=~\"$cluster|$^\"}[60s]))", - "intervalFactor": 1, - "legendFormat": "Total Requests", - "refId": "A", - "step": 1 - } - ], - "title": "Total Requests" - }, - { - "class": "dashlist", - "tags": [ - "2.3" - ] - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "percent_panel", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Load per [[by]]" - }, - { - "class": "ops_panel", - "targets": [ - { - "expr": "sum(irate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) + sum(irate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "description": "Amount of requests served as the coordinator. Imbalances here represent dispersion at the connection level, not your data model.", - "title": "Requests Served per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_header_panel", - "content": "

Reads and Writes

" - }, - { - "class": "text_header_panel", - "content": "

Timeouts and Errors

" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_foreground_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Foreground Writes per [[by]]" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_foreground_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Foreground Reads per [[by]]" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Write Timeouts per [[by]]" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Write Unavailable per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_background_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Background Writes per [[by]]" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_background_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "title": "Background Reads per [[by]]" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Read Timeouts per [[by]]" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Read Unavailable per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "plain_text", - "content": "

Replica

" - } - ] - }, - { - "class": "row", - "height": "auto", - "panels": [ - { - "class": "rps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads" - }, - { - "class": "wps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_database_active_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Active sstable reads" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_database_queued_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Queued sstable reads" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_database_requests_blocked_memory_current{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes currently blocked on dirty" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_commitlog_pending_allocations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes currently blocked on commitlog" - }, - { - "class": "text_panel", - "content": "", - "mode": "markdown", - "span": 3 - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads failed" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes blocked on dirty" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes blocked on commitlog" - }, - { - "class": "text_panel", - "content": "", - "mode": "markdown", - "span": 3 - }, - { - "class": "text_panel", - "content": "", - "mode": "markdown", - "span": 3 - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes failed" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes timed out" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Cache

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "rps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) - irate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads with no misses" - }, - { - "class": "rps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads with misses" - } - ] - }, - { - "class" : "row", - "panels": [ - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Hits" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Hits" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Misses" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Misses" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Insertions" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Insertions" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Evictions" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Evictions" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Merges" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Merges" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Removals" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Removals" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_cache_rows{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Rows" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_cache_partitions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partitions" - }, - { - "class": "bytes_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_cache_bytes_used{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Used Bytes" - }, - { - "class": "bytes_panel", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_cache_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Total Bytes" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Memory

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bytes_panel", - "span": 6, - "targets": [ - { - "expr": "sum(scylla_lsa_total_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "LSA total memory" - }, - { - "class": "bytes_panel", - "span": 6, - "targets": [ - { - "expr": "sum(scylla_lsa_non_lsa_used_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Non-LSA used memory" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Compaction

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "graph_panel", - "span": 12, - "targets": [ - { - "expr": "sum(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Running Compactions" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

CQL

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Insert" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Reads" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Deletes" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[300s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "CQL Updates" - }, - { - "class": "graph_panel", - "span": 3, - "pointradius": 1, - "targets": [ - { - "expr": "sum(scylla_transport_current_connections{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Client CQL connections by [[by]]", - "description": "amount of CQL connections currently established" - } - ], - "title": "New row" - } - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "Shard", - "value": "shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "type": "custom" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "template_variable_custom", - "current": { - "text": "2.3", - "value": "2.3" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "2.3", - "value": "2.3" - } - ], - "query": "2.3" - }, - { - "class": "monitor_version_var" - } - ] - }, - "tags": [ - "2.3" - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Scylla Per Server Metrics", - "version": 5 - } -} diff --git a/grafana/scylla-detailed.3.0.template.json b/grafana/scylla-detailed.3.0.template.json deleted file mode 100644 index 5d8865370..000000000 --- a/grafana/scylla-detailed.3.0.template.json +++ /dev/null @@ -1,1031 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "detailed-3-0", - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "panels": [ - { - "class": "percent_panel", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "span": 3, - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "title": "Load" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) + $func(irate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "description": "Amount of requests served as the coordinator. Imbalances here represent dispersion at the connection level, not your data model.", - "title": "Requests Served per [[by]] - Coordinator" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads per [[by]] - Replica" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes per [[by]] - Replica" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_header_panel", - "content": "

Reads and Writes - Coordinator

" - }, - { - "class": "text_header_panel", - "content": "

Timeouts and Errors - Coordinator

" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_foreground_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "description": "Foreground writes are writes that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", - "title": "Foreground Writes per [[by]]" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_foreground_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "description": "Foreground reads are reads that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", - "title": "Foreground Reads per [[by]]" - }, - { - "class": "wpm_panel", - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "title": "Write Timeouts/Minutes per [[by]]" - }, - { - "class": "wpm_panel", - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "description": "Requests that Scylla did not even try to write because replicas that were needed to execute this write were unavailable. Unavailable writes are counted in the node that received the request (the coordinator), not at the replicas.", - "title": "Write Unavailable/Minutes per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_background_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "description": "Background writes are writes that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", - "title": "Background Writes per [[by]]" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_background_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "description": "Background reads are reads that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", - "title": "Background Reads per [[by]]" - }, - { - "class": "rpm_panel", - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "title": "Read Timeouts/Minutes per [[by]]" - }, - { - "class": "rpm_panel", - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "description": "Requests that Scylla did not even try to read because replicas that were needed to execute this write were unavailable. Unavailable reads are counted in the node that received the request (the coordinator), not at the replicas.", - "title": "Read Unavailable/Minutes per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Hints Written per [[by]]" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "title": "Hints sent per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "plain_text", - "content": "

Replica

" - } - ] - }, - { - "class": "row", - "height": "auto", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_database_active_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Active sstable reads" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_database_queued_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Queued sstable reads" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_database_requests_blocked_memory_current{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes currently blocked on dirty" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_commitlog_pending_allocations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes currently blocked on commitlog" - }, - { - "class": "text_panel", - "content": "", - "mode": "markdown", - "span": 3 - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads failed" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes blocked on dirty" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes blocked on commitlog" - }, - { - "class": "text_panel", - "content": "", - "mode": "markdown", - "span": 3 - }, - { - "class": "text_panel", - "content": "", - "mode": "markdown", - "span": 3 - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes failed" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Writes timed out" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Cache - Replica

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "rps_panel", - "span": 6, - "targets": [ - { - "expr": "$func(irate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s]) - irate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads with no misses" - }, - { - "class": "rps_panel", - "span": 6, - "targets": [ - { - "expr": "$func(irate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Reads with misses" - } - ] - }, - { - "class" : "row", - "panels": [ - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Hits" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Hits" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Misses" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Misses" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Insertions" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Insertions" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Evictions" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Evictions" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Merges" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Merges" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Row Removals" - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partition Removals" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_cache_rows{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Rows" - }, - { - "class": "graph_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_cache_partitions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Partitions" - }, - { - "class": "bytes_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_cache_bytes_used{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Used Bytes" - }, - { - "class": "bytes_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_cache_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Total Bytes" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Materialized Views - Replica

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "View Local Update", - "description" : "Number of view update locally" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "View Remote Update", - "description" : "Number of view update remotely" - }, - { - "class": "bytes_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_database_view_update_backlog{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "View Update Backlog", - "description" : "Size in bytes of the view update backlog at each base replica." - }, - { - "class": "ops_panel", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Dropped View Updates", - "description" : "Number of dropped view updates due to an excessive view update backlog." - } ] - }, - { - "class": "row", - "panels": [ - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Hints for view", - "description" : "Number of hints sent for view." - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_current_throttled_base_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Throttled Base Writes", - "description" : "Currently throttled base writes, as a consequence of the respective view update backlog." - } - ] - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Memory - Replica

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bytes_panel", - "span": 6, - "targets": [ - { - "expr": "$func(scylla_lsa_total_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "LSA total memory" - }, - { - "class": "bytes_panel", - "span": 6, - "targets": [ - { - "expr": "$func(scylla_lsa_non_lsa_used_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Non-LSA used memory" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "text_panel", - "content": "

Compaction - Replica

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "graph_panel_int", - "span": 4, - "targets": [ - { - "expr": "$func(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Running Compactions" - }, - { - "class": "percent_panel", - "span": 4, - "targets": [ - { - "expr": "($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[60s])) by ([[by]]))/10", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "yaxes": [ - { - "format": "percent", - "logBase": 1, - "max": null, - "min": 0, - "show": true, - "decimals": 0 - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "description": "Percentage of CPU time used by compaction", - "title": "Compactions CPU Runtime" - }, - { - "class": "graph_panel", - "span": 4, - "targets": [ - { - "refId": "A", - "expr": "avg(scylla_scheduler_shares{group=\"compaction\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "format": "time_series" - } - ], - "description": "Shares assigned to the compaction", - "title": "Compactions Shares" - } - - ], - "title": "New row" - }, - { - "class": "user_panel_row_header" - }, - { - "class": "user_panels_row" - } - ], - "templating": { - "list": [ - { - "class":"by_template_var" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "aggregation_function" - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "tags": [ - "3.0" - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Detailed", - "overwrite": true, - "version": 5 - } -} diff --git a/grafana/scylla-errors.3.0.template.json b/grafana/scylla-errors.3.0.template.json deleted file mode 100644 index 0501ecea7..000000000 --- a/grafana/scylla-errors.3.0.template.json +++ /dev/null @@ -1,253 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "error-3-0", - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "panels": [ - { - "class": "rps_panel", - "span": 4, - "targets": [ - { - "expr": "avg(irate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Local Reads Error by [[by]]", - "description": "Number of Read requests that failed due to an 'unavailable' error" - }, - { - "class": "wps_panel", - "span": 4, - "targets": [ - { - "expr": "avg(irate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Local Write Error by [[by]]", - "description": "Number of write requests that failed due to an 'unavailable' error" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 4, - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "rps_panel", - "span": 4, - "targets": [ - { - "expr": "avg(irate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Reads Unavailable Error by [[by]]", - "description": "Number of Read requests that failed due to an 'unavailable' error" - }, - { - "class": "wps_panel", - "span": 4, - "targets": [ - { - "expr": "avg(irate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Write Unavailable Error by [[by]]", - "description": "Number of write requests that failed on a local Node" - }, - { - "class": "ops_panel", - "span": 4, - "targets": [ - { - "expr": "avg(irate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Range Unavailable Error by [[by]]", - "description": "Number of write requests that failed on a local Node" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "span": 4, - "targets": [ - { - "expr": "avg(irate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "AIO Error by [[by]]", - "description": "Number of AIO Errors" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 4, - "style": {} - }, - { - "class": "ops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "C++ Exceptions [[by]]", - "description": "Number of C++ exceptions thrown.\n\n A peak in the number of exceptions is an indication of a potential problem." - } - ], - "title": "New row" - } - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "Shard", - "value": "instance,shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "type": "custom" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "tags": [ - "3.0" - ], - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Scylla Errors Monitoring", - "overwrite": true, - "version": 5 - } -} diff --git a/grafana/scylla-io.2.3.template.json b/grafana/scylla-io.2.3.template.json deleted file mode 100644 index 49bb07d67..000000000 --- a/grafana/scylla-io.2.3.template.json +++ /dev/null @@ -1,597 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "io-2-3", - "rows": [ - { - "class": "small_row", - "panels": [ - { - "class": "text_panel", - "content": "\n
" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "150px", - "panels": [ - { - "class": "single_stat_panel", - "targets": [ - { - "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 240 - } - ], - "title": "Total Nodes" - }, - { - "class": "single_stat_panel_fail", - "targets": [ - { - "expr": "count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Unreachable", - "refId": "A", - "step": 120 - } - ], - "thresholds": "1,2", - "title": "Unreachable" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 3, - "style": {} - }, - { - "aliasColors": { - "{}": "#584477" - }, - "bars": true, - "class": "graph_panel", - "lines": false, - "seriesOverrides": [ - {} - ], - "targets": [ - { - "expr": "sum(irate(scylla_transport_requests_served{cluster=~\"$cluster|$^\"}[60s])) + sum(irate(scylla_thrift_served{cluster=~\"$cluster|$^\"}[60s]))", - "intervalFactor": 1, - "legendFormat": "Total Requests", - "refId": "A", - "step": 30 - } - ], - "title": "Total Requests", - "transparent": false - }, - { - "class": "dashlist", - "tags": [ - "2.3" - ] - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "graph_panel", - "seriesOverrides": [ - {} - ], - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 30 - } - ], - "title": "Load per [[by]]", - "transparent": false - }, - { - "class": "graph_panel", - "pointradius": 1, - "targets": [ - { - "expr": "sum(irate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) + sum(irate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 30 - } - ], - "title": "Requests Served per [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "plain_text", - "content": "

Disk Activity

" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "graph_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 20 - } - ], - "title": "Disk Writes per Server" - }, - { - "class": "graph_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "Disk Reads per Server" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_disk_written_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 20 - } - ], - "title": "Disk Writes Bps per Server" - }, - { - "class": "bps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_disk_read_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "Disk Read Bps per Server" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "plain_text", - "content": "

I/O Queue

" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_compaction_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Compactions I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_compaction_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Compactions I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_compaction_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Compactions I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_query_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Query I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_query_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Query I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_query_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Query I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_commitlog_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Commitlog I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_commitlog_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Commitlog I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_commitlog_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Commitlog I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_memtable_flush_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Memtable Flush I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_memtable_flush_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Memtable Flush I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_memtable_flush_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Memtable Flush I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_streaming_read_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Reads I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_read_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Reads I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_read_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Reads I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_streaming_write_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Writes I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_write_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Writes I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_write_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Writes I/O Queue IOPS by [[by]]" - } - ], - "title": "New row" - } - ], - "tags": [ - "2.3" - ], - "templating": { - "list": [ - { - "current": {}, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "monitor_disk", - "options": [], - "query": "node_disk_read_bytes_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "type": "query" - }, - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "Shard", - "value": "shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "type": "custom" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "template_variable_custom", - "current": { - "text": "2.3", - "value": "2.3" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "2.3", - "value": "2.3" - } - ], - "query": "2.3" - }, - { - "class": "monitor_version_var" - } - ] - }, - "title": "Scylla Per-Server Disk I/O" - } -} \ No newline at end of file diff --git a/grafana/scylla-io.3.0.template.json b/grafana/scylla-io.3.0.template.json deleted file mode 100644 index 87aaf88e9..000000000 --- a/grafana/scylla-io.3.0.template.json +++ /dev/null @@ -1,358 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "uid": "io-3-0", - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "height": "25px", - "gridPos": {"h": 2}, - "panels": [ - { - "class": "plain_text", - "content": "

I/O Queue

" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_compaction_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Compactions I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_compaction_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Compactions I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_compaction_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Compactions I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_query_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Query I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_query_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Query I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_query_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Query I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_commitlog_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Commitlog I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_commitlog_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Commitlog I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_commitlog_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Commitlog I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_memtable_flush_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Memtable Flush I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_memtable_flush_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Memtable Flush I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_memtable_flush_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Memtable Flush I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_streaming_read_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Reads I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_read_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Reads I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_read_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Reads I/O Queue IOPS by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "1000000*max(scylla_io_queue_streaming_write_delay{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Writes I/O Queue delay by [[by]]" - }, - { - "class": "bps_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_write_total_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Writes I/O Queue bandwidth by [[by]]" - }, - { - "class": "iops_panel", - "span": 4, - "targets": [ - { - "expr": "sum(irate(scylla_io_queue_streaming_write_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "seastar_io_queue_delay", - "refId": "A", - "step": 30 - } - ], - "title": "Streaming Writes I/O Queue IOPS by [[by]]" - } - ], - "title": "New row" - } - ], - "tags": [ - "3.0" - ], - "templating": { - "list": [ - { - "class":"by_template_var" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "overwrite": true, - "title": "I/O" - } -} diff --git a/grafana/scylla-os.2.3.template.json b/grafana/scylla-os.2.3.template.json deleted file mode 100644 index f01d496e8..000000000 --- a/grafana/scylla-os.2.3.template.json +++ /dev/null @@ -1,399 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "rows": [ - { - "class": "logo_row", - "panels": [ - { - "class": "text_panel", - "content": "\n
" - } - ] - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "pie_chart_panel", - "height": "250px", - "repeat": "node", - "targets": [ - { - "expr": "sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Free", - "metric": "", - "refId": "A", - "instant": true, - "step": 7200 - }, - { - "expr": "(sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}))", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "B", - "instant": true, - "step": 7200 - } - ], - "title": "Total Storage $node" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "text_panel", - "content": "

Disk $monitor_disk

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_writes_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Writes per Server" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_reads_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Reads per Server" - }, - { - "class": "bps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_written_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_bytes_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Writes Bps per Server" - }, - { - "class": "bps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_read_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_bytes_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Read Bps per Server" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "text_panel", - "content": "

Network $monitor_network_interface

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "pps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_receive_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_receive_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Rx Packets" - }, - { - "class": "pps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_transmit_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_transmit_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Tx Packets" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_receive_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_receive_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Rx Bps" - }, - { - "class": "bps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_transmit_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_transmit_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by (instance)", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Tx Bps" - } - ], - "title": "New row" - } - ], - "tags": [ - "2.3" - ], - "templating": { - "list": [ - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(node_filesystem_avail_bytes, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitor_disk", - "options": [], - "query": "node_disk_read_bytes_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitor_network_interface", - "options": [], - "query": "node_network_receive_packets_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "class": "template_variable_custom", - "current": { - "text": "2.3", - "value": "2.3" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "2.3", - "value": "2.3" - } - ], - "query": "2.3" - }, - { - "class": "monitor_version_var" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Scylla Per Machine Metrics", - "uid": "machine-2-3", - "version": 5 - } -} \ No newline at end of file diff --git a/grafana/scylla-os.3.0.template.json b/grafana/scylla-os.3.0.template.json deleted file mode 100644 index ef7125c92..000000000 --- a/grafana/scylla-os.3.0.template.json +++ /dev/null @@ -1,463 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "overwrite": true, - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "pie_chart_panel", - "height": "250px", - "repeat": "node", - "targets": [ - { - "expr": "sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Free", - "metric": "", - "refId": "A", - "instant": true, - "step": 7200 - }, - { - "expr": "(sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}))", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "B", - "instant": true, - "step": 7200 - } - ], - "title": "Total Storage $node" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "text_panel", - "content": "

Disk $monitor_disk

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bytes_panel", - "span": 3, - "targets": [ - { - "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Size by $by" - }, - { - "class": "wps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_writes_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Writes per $by" - }, - { - "class": "rps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_reads_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Reads per $by" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "text_panel", - "content": "", - "span": 3 - }, - { - "class": "bps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_written_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_bytes_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Writes Bps per $by" - }, - { - "class": "bps_panel", - "span": 3, - "targets": [ - { - "expr": "sum(irate(node_disk_read_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_disk_bytes_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Read Bps per $by" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "text_panel", - "content": "

Network $monitor_network_interface

", - "style": {} - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "pps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_receive_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_receive_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Rx Packets" - }, - { - "class": "pps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_transmit_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_transmit_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Tx Packets" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_receive_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_receive_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Rx Bps" - }, - { - "class": "bps_panel", - "span": 6, - "targets": [ - { - "expr": "sum(irate(node_network_transmit_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(irate(node_network_transmit_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Interface Tx Bps" - } - ], - "title": "New row" - } - ], - "tags": [ - "3.0" - ], - "templating": { - "list": [ - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(node_filesystem_avail_bytes, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - } - ], - "query": "Cluster,DC,Instance", - "type": "custom" - }, - { - "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitor_disk", - "options": [], - "query": "node_disk_read_bytes_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "isNone": true, - "text": "None", - "value": "" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "monitor_network_interface", - "options": [], - "query": "node_network_receive_packets_total", - "refresh": 2, - "regex": "/.*device=\"([^\\\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "OS Metrics", - "uid": "OS-3-0", - "version": 5 - } -} \ No newline at end of file diff --git a/grafana/scylla-overview.2.3.template.json b/grafana/scylla-overview.2.3.template.json deleted file mode 100644 index d79276b0a..000000000 --- a/grafana/scylla-overview.2.3.template.json +++ /dev/null @@ -1,526 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "originalTitle": "Scylla Cluster Metrics", - "rows": [ - { - "class": "logo_row", - "panels": [ - { - "class": "text_panel", - "content": "\n
" - } - ] - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "single_stat_panel", - "targets": [ - { - "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 40 - } - ], - "title": "Total Nodes" - }, - { - "class": "single_stat_panel_fail", - "targets": [ - { - "expr": "count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Unreachable", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "title": "Unreachable" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 3, - "style": {} - }, - { - "class": "alert_table", - "title": "Active Alerts" - }, - { - "class": "dashlist", - "tags": [ - "2.3" - ] - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "percent_panel", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Load", - "refId": "A", - "step": 4 - } - ], - "title": "Load" - }, - { - "class": "ops_panel", - "description": "Amount of requests served as the coordinator. Imbalances here represent dispersion at the client-side level or connection balancing level, not your data model.", - "targets": [ - { - "expr": "sum(irate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) + sum(irate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "Requests Served", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Requests Served" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]]) + 1)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average write latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile write latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile write latency by [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])/(sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]]) + 1)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average read latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile read latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile read latency by [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "text_header_panel", - "content": "

Reads and Writes

" - }, - { - "class": "text_header_panel", - "content": "

Timeouts and Errors

" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "queue_lenght_panel", - "description": "Foreground writes are writes that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_foreground_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Foreground Writes" - }, - { - "class": "queue_lenght_panel", - "description": "Foreground reads are reads that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_foreground_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "title": "Foreground Reads" - }, - { - "class": "wps_panel", - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Write Timeouts" - }, - { - "class": "wps_panel", - "description": "Requests that Scylla did not even try to write because replicas that were needed to execute this write were unavailable. Unavailable writes are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Write Unavailable" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "queue_lenght_panel", - "description": "Background writes are writes that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_background_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Background Writes" - }, - { - "class": "queue_lenght_panel", - "description": "Background reads are reads that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", - "span": 3, - "targets": [ - { - "expr": "sum(scylla_storage_proxy_coordinator_background_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Background Reads" - }, - { - "class": "rps_panel", - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Read Timeouts" - }, - { - "class": "rps_panel", - "description": "Requests that Scylla did not even try to read because replicas that were needed to execute this write were unavailable. Unavailable reads are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Read Unavailable" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "plain_text", - "content": "

Cache

", - "span": 6 - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "description": "Number of rows that were read from the cache, without needing to be fetched from storage.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cache Hits" - }, - { - "class": "ops_panel", - "description": "Number of rows that were not present in the cache, and had to be fetched from storage.", - "span": 3, - "targets": [ - { - "expr": "sum(irate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cache Misses" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [], - "title": "New row" - }, - { - "class": "monitoring_version_row" - } - ], - "tags": [ - "2.3" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "Instance", - "value": "instance" - }, - "hide": 0, - "includeAll": false, - "label": "by", - "multi": false, - "name": "by", - "options": [ - { - "selected": false, - "text": "Cluster", - "value": "cluster" - }, - { - "selected": false, - "text": "DC", - "value": "dc" - }, - { - "selected": true, - "text": "Instance", - "value": "instance" - }, - { - "selected": false, - "text": "Shard", - "value": "shard" - } - ], - "query": "Cluster,DC,Instance,Shard", - "type": "custom" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "class": "template_variable_custom", - "current": { - "text": "2-3", - "value": "2-3" - }, - "name": "dash_version", - "options": [ - { - "selected": true, - "text": "2-3", - "value": "2-3" - } - ], - "query": "2-3" - }, - { - "class": "template_variable_custom", - "current": { - "text": "2.3", - "value": "2.3" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "2.3", - "value": "2.3" - } - ], - "query": "2.3" - }, - { - "class": "monitor_version_var" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Scylla Overview Metrics", - "uid": "overview-2-3", - "version": 3 - } -} \ No newline at end of file diff --git a/grafana/scylla-overview.3.0.template.json b/grafana/scylla-overview.3.0.template.json deleted file mode 100644 index 3899fc880..000000000 --- a/grafana/scylla-overview.3.0.template.json +++ /dev/null @@ -1,877 +0,0 @@ -{ - "dashboard": { - "class": "dashboard", - "originalTitle": "Scylla Cluster Metrics", - "overwrite": true, - "rows": [ - { - "class": "logo_row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "single_stat_panel", - "targets": [ - { - "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", - "intervalFactor": 1, - "legendFormat": "Total Nodes", - "refId": "A", - "step": 40 - } - ], - "title": "Total Nodes" - }, - { - "class": "single_stat_panel_fail", - "targets": [ - { - "expr": "count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", - "intervalFactor": 1, - "legendFormat": "Unreachable", - "refId": "A", - "step": 20 - } - ], - "thresholds": "1,2", - "title": "Unreachable" - }, - { - "class": "text_panel", - "content": "## ", - "mode": "markdown", - "span": 2, - "style": {} - }, - { - "class": "percent_panel", - "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", - "span": 4, - "targets": [ - { - "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "title": "Load" - }, - { - "class": "single_value_table", - "description": "Nodes Information table", - "span": 4, - "styles": [ - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the detailed node information", - "linkUrl": "/d/detailed-[[dash_version]]/Detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell}", - "mappingType": 1, - "pattern": "instance", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "class": "hidden_column", - "pattern": "Time" - }, - { - "class": "hidden_column", - "pattern": "__name__" - }, - { - "class": "hidden_column", - "pattern": "exported_instance" - }, - { - "class": "hidden_column", - "pattern": "job" - }, - { - "class": "hidden_column", - "pattern": "version" - }, - { - "class": "hidden_column", - "pattern": "type" - }, - { - "class": "hidden_column", - "pattern": "Value" - }, - { - "class": "hidden_column", - "pattern": "cluster" - }, - { - "alias": "Version", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "${__cell_12}", - "linkUrl": "/d/OS-[[dash_version]]/OS-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_9}", - "mappingType": 1, - "pattern": "svr", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "OS", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the OS node information", - "linkUrl": "/d/OS-[[dash_version]]/OS-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_9}", - "mappingType": 1, - "pattern": "OS", - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "OS", - "value": "os" - } - ] - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the CQL information", - "linkUrl": "/d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_9}", - "mappingType": 1, - "pattern": "CQL", - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "CQL", - "value": "cql" - } - ] - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the Errors metrics information", - "linkUrl": "/d/error-[[dash_version]]/scylla-errors?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_9}", - "mappingType": 1, - "pattern": "Errors", - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "Errors", - "value": "errors" - } - ] - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the Errors metrics information", - "linkUrl": "/d/io-[[dash_version]]/i-o?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_9}", - "mappingType": 1, - "pattern": "IO", - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "IO", - "value": "io" - } - ] - }, - { - "alias": "CPU", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to the node CPU information", - "linkUrl": "/d/cpu-[[dash_version]]/CPU-Metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_9}", - "mappingType": 1, - "pattern": "CPU", - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "CPU", - "value": "cpu" - } - ] - } - ], - "targets": [ - { - "expr": "scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", - "format": "table", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Nodes" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "bytes_panel", - "span": 4, - "targets": [ - { - "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "B", - "step": 1 - } - ], - "title": "Disk Size by $by" - }, - { - "class": "ops_panel", - "description": "Amount of requests served as the coordinator. Imbalances here represent dispersion at the client-side level or connection balancing level, not your data model.", - "span": 4, - "targets": [ - { - "expr": "$func(irate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]]) + $func(irate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "title": "Requests Served - Coordinator" - }, - { - "class": "alert_table", - "span": 4, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": true, - "linkTooltip": "Jump to see the node details", - "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_4}&from=${__cell_0}", - "pattern": "Time", - "type": "date" - }, - { - "class": "hidden_column", - "pattern": "severity" - }, - { - "class": "hidden_column", - "pattern": "alertname" - }, - { - "class": "hidden_column", - "pattern": "cluster" - }, - { - "class": "hidden_column", - "pattern": "monitor" - }, - { - "class": "hidden_column", - "pattern": "summary" - }, - { - "alias": "Instance", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Jump to see the node details", - "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell}", - "mappingType": 1, - "pattern": "instance", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "title": "Active Alerts" - } - ], - "title": "New row" - }, - { - "class": "header_row", - "panels": [ - { - "class": "plain_text", - "content": "

Latencies - Coordinator

" - } - ] - }, - { - "class": "row", - "panels": [ - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]]) + 1)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average write latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile write latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile write latency by [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]]) + 1)", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 1 - } - ], - "title": "Average read latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "95th percentile read latency by [[by]]" - }, - { - "class": "us_panel", - "span": 4, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]], le))", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "99th percentile read latency by [[by]]" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "text_header_panel", - "content": "

Reads and Writes - Coordinator

" - }, - { - "class": "text_header_panel", - "content": "

Timeouts and Errors - Coordinator

" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "queue_lenght_panel", - "description": "Foreground writes are writes that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_foreground_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Foreground Writes" - }, - { - "class": "queue_lenght_panel", - "description": "Foreground reads are reads that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_foreground_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "title": "Foreground Reads" - }, - { - "class": "wps_panel", - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Write Timeouts" - }, - { - "class": "wps_panel", - "description": "Requests that Scylla did not even try to write because replicas that were needed to execute this write were unavailable. Unavailable writes are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Write Unavailable" - } - ], - "title": "New row" - }, - { - "class": "row", - "height": "200px", - "panels": [ - { - "class": "queue_lenght_panel", - "description": "Background writes are writes that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_background_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Background Writes" - }, - { - "class": "queue_lenght_panel", - "description": "Background reads are reads that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", - "span": 3, - "targets": [ - { - "expr": "$func(scylla_storage_proxy_coordinator_background_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Background Reads" - }, - { - "class": "rps_panel", - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Read Timeouts" - }, - { - "class": "rps_panel", - "description": "Requests that Scylla did not even try to read because replicas that were needed to execute this write were unavailable. Unavailable reads are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Read Unavailable" - } - ], - "title": "New row" - }, - { - "class": "row", - "gridPos": { - "h": 2 - }, - "height": "25px", - "panels": [ - { - "class": "plain_text", - "content": "

Cache - Replica

", - "span": 6 - }, - { - "class": "plain_text", - "content": "

Timeouts

", - "span": 6 - } - ], - "title": "New row" - }, - { - "class": "row", - "panels": [ - { - "class": "ops_panel", - "description": "Number of rows that were read from the cache, without needing to be fetched from storage.", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cache Hits" - }, - { - "class": "ops_panel", - "description": "Number of rows that were not present in the cache, and had to be fetched from storage.", - "span": 3, - "targets": [ - { - "expr": "$func(irate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cache Misses" - }, - { - "class": "wpm_panel", - "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Write Timeouts/Minutes by [[by]]" - }, - { - "class": "rpm_panel", - "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", - "span": 3, - "targets": [ - { - "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Read Timeouts/Minutes by [[by]]" - } - ], - "title": "New row" - }, - { - "class": "user_panel_row_header" - }, - { - "class": "user_panels_row" - }, - { - "class": "monitoring_version_row" - } - ], - "tags": [ - "3.0" - ], - "templating": { - "list": [ - { - "class": "by_template_var" - }, - { - "class": "template_variable_single", - "label": "cluster", - "name": "cluster", - "query": "label_values(scylla_reactor_utilization, cluster)" - }, - { - "class": "template_variable_all", - "label": "dc", - "name": "dc", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)" - }, - { - "class": "template_variable_all", - "label": "node", - "name": "node", - "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)" - }, - { - "class": "template_variable_all", - "label": "shard", - "name": "shard", - "query": "label_values(scylla_reactor_utilization,shard)", - "sort": 3 - }, - { - "allValue": null, - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "/var/lib/scylla", - "value": "/var/lib/scylla" - }, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Mount path", - "multi": false, - "name": "mount_point", - "options": [], - "query": "node_filesystem_avail_bytes", - "refresh": 2, - "regex": "/mountpoint=\"([^\"]*)\".*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "class": "aggregation_function" - }, - { - "class": "adhoc_filter" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3-0", - "value": "3-0" - }, - "name": "dash_version", - "options": [ - { - "selected": true, - "text": "3-0", - "value": "3-0" - } - ], - "query": "3-0" - }, - { - "class": "template_variable_custom", - "current": { - "text": "3.0", - "value": "3.0" - }, - "name": "scylla_dash_version", - "options": [ - { - "selected": true, - "text": "3.0", - "value": "3.0" - } - ], - "query": "3.0" - }, - { - "class": "monitor_version_var" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Overview", - "uid": "overview-3-0", - "version": 3 - } -} \ No newline at end of file