From 845f8447cf8910cfd0a21b494f7a97553aa19484 Mon Sep 17 00:00:00 2001 From: mphanias Date: Mon, 23 Oct 2023 12:55:39 +0530 Subject: [PATCH] OM129 - unique data usgae initial commit modified queries according to the 7.0 stat and metrics added all-cluster master-object and unique-usage-bytes view which are all connected to a prometheus instance --- config/grafana/dashboards/uniquedata.json | 300 ++++++++++++++++++++-- 1 file changed, 279 insertions(+), 21 deletions(-) diff --git a/config/grafana/dashboards/uniquedata.json b/config/grafana/dashboards/uniquedata.json index 12d9bf4..d2f0df3 100644 --- a/config/grafana/dashboards/uniquedata.json +++ b/config/grafana/dashboards/uniquedata.json @@ -60,6 +60,227 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 50, + "panels": [], + "title": "All Clusters - Summary ", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "description": "Display Unique Data Utilization across all cluster and namespaces connected to this prometheus instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 46, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$Q_MEMORY_OBJECTS * $QR_RULE_CAN_SHOW", + "hide": false, + "refId": "All Master Objects", + "type": "math" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum( # Running individual exporter for each node in a cluster\n (avg by (ns) ( aerospike_namespace_ns_cluster_size {job=\"$job_name\", } ))\n == bool \n (count by (ns) (aerospike_namespace_ns_cluster_size {job=\"$job_name\", } ))\n)> bool 0)\n* # migration check with RX metric\n(sum((sum by (ns) (aerospike_namespace_migrate_rx_partitions_remaining + aerospike_namespace_migrate_tx_partitions_remaining) != bool 0))\n== bool 0)\n* # Consistent Effective Replication factor across all nodes\n(sum(\n min by (ns) (aerospike_namespace_effective_replication_factor {job=\"$job_name\", }) \n == bool \n max by (ns) (aerospike_namespace_effective_replication_factor {job=\"$job_name\", })\n) > bool 0)", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "QR_RULE_CAN_SHOW" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum (aerospike_namespace_master_objects{job=\"$job_name\", })", + "format": "time_series", + "hide": true, + "instant": false, + "legendFormat": "All Master Objects", + "range": true, + "refId": "Q_MEMORY_OBJECTS" + } + ], + "title": "All Cluster - Master Objects (sum)", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "description": "Display Unique Data Utilization across all cluster and namespaces connected to this prometheus instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 48, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($6_Q_USAGE_BYTE + $7_Q_USAGE_BYTE) * $QR_RULE_CAN_SHOW", + "hide": false, + "refId": "Unique Data Bytes", + "type": "math" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum( # Running individual exporter for each node in a cluster\n (avg by (ns) ( aerospike_namespace_ns_cluster_size {job=\"$job_name\", } ))\n == bool \n (count by (ns) (aerospike_namespace_ns_cluster_size {job=\"$job_name\", } ))\n)> bool 0)\n* # migration check with RX metric\n(sum((sum by (ns) (aerospike_namespace_migrate_rx_partitions_remaining + aerospike_namespace_migrate_tx_partitions_remaining) != bool 0))\n== bool 0)\n* # Consistent Effective Replication factor across all nodes\n(sum(\n min by (ns) (aerospike_namespace_effective_replication_factor {job=\"$job_name\", }) \n == bool \n max by (ns) (aerospike_namespace_effective_replication_factor {job=\"$job_name\", })\n) > bool 0)", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "QR_RULE_CAN_SHOW" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum((sum((sum ( (((aerospike_namespace_memory_used_data_bytes {job=\"$job_name\", } + aerospike_namespace_memory_used_index_bytes {job=\"$job_name\", })\nunless # we pick namespace which are in memory but not using Disk or Pmem\n(aerospike_namespace_device_used_bytes {job=\"$job_name\", } or aerospike_namespace_pmem_used_bytes {job=\"$job_name\", } )\n) \n/aerospike_namespace_effective_replication_factor {job=\"$job_name\", }\n) \n- on (job, cluster_name, instance, service,ns) ( # begin calculate the header byte size 39bytes if as-version>=6 and 34 if as-version<6\n sum (\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{job=\"$job_name\", build=~\"^4.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{job=\"$job_name\", build=~\"^5.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{job=\"$job_name\", build!~\"^5.*\"}*39) ) \n by (job, cluster_name, instance, service,ns) \n ) # end of header-byte calculation by aerospike version\n) by (service, ns)) or vector(0) ) by (ns))) \n+ on (job, cluster_name, instance, service,ns)\nsum (\n ( # device-used-bytes + pmem-used-bytes\n ( # sum disk device used bytes\n ( sum by (ns) (aerospike_namespace_device_used_bytes{job=\"$job_name\", }) /\n ( sum by (ns) ( (aerospike_namespace_device_compression_ratio{job=\"$job_name\", } )) \n *\n avg by (ns) ( aerospike_namespace_effective_replication_factor{job=\"$job_name\", } ))\n )\n or\n ( sum by (ns) (aerospike_namespace_device_used_bytes{job=\"$job_name\", }) /\n ( avg by (ns) ( aerospike_namespace_effective_replication_factor{job=\"$job_name\", } ))\n )\n) # end devide-used-bytes\nor\n( \n ( sum by (ns) (aerospike_namespace_pmem_used_bytes{job=\"$job_name\", }) /\n ( sum by (ns) ( (aerospike_namespace_pmem_compression_ratio{job=\"$job_name\", } )) \n * avg by (ns) (aerospike_namespace_effective_replication_factor{job=\"$job_name\", }) ))\nor\n ( sum by (ns) (aerospike_namespace_pmem_used_bytes{job=\"$job_name\", }) /\n ( avg by (ns) ( aerospike_namespace_effective_replication_factor{job=\"$job_name\", }) )) \n)\n)\n- on (job, cluster_name, instance, service,ns) ( # begin calculate the header byte size 39bytes if as-version>=6 and 34 if as-version<6\n sum (\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{job=\"$job_name\", build=~\"^4.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{job=\"$job_name\", build=~\"^5.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{job=\"$job_name\", build!~\"^5.*\"}*39) ) \n by (ns)\n) # end of header-byte calculation by aerospike version\nor vector(0)) # end of disk+pmem used bytes\n\n) # end paranthesis for whole sum", + "hide": true, + "legendFormat": "Unique Data Bytes", + "range": true, + "refId": "6_Q_USAGE_BYTE" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum((sum((sum ( (((aerospike_namespace_data_used_bytes {job=\"$job_name\", })/\naerospike_namespace_data_compression_ratio {job=\"$job_name\", }\n) \n/aerospike_namespace_effective_replication_factor {job=\"$job_name\", }\n) \n- on (job, cluster_name, instance, service,ns) ( # begin calculate the header byte size 39bytes if as-version>=6 and 34 if as-version<6\n sum (\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build=~\"^4.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{ build=~\"^5.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", } * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build!~\"^5.*\"}*39) ) \n by (job, cluster_name, instance, service,ns) \n ) # end of header-byte calculation by aerospike version\n) by (service, ns)) or vector(0) ) by (ns))) \n\n) # end paranthesis for whole sum", + "hide": true, + "legendFormat": "Unique Data Bytes", + "range": true, + "refId": "7_Q_USAGE_BYTE" + } + ], + "title": "All Clusters - Unique Data Bytes (sum)", + "transformations": [], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 64, + "panels": [], + "title": "Unique Data Usage - $cluster", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -89,9 +310,9 @@ "h": 5, "w": 12, "x": 0, - "y": 0 + "y": 6 }, - "id": 4, + "id": 60, "links": [], "maxDataPoints": 100, "options": { @@ -116,7 +337,7 @@ "type": "__expr__", "uid": "__expr__" }, - "expression": "$Q_MEMORY_OBJECTS * $QR_RULE_CAN_SHOW", + "expression": "$Q_MEMORY_OBJECTS ", "hide": false, "refId": "All Master Objects", "type": "math" @@ -182,9 +403,9 @@ "h": 5, "w": 12, "x": 12, - "y": 0 + "y": 6 }, - "id": 27, + "id": 62, "links": [], "maxDataPoints": 100, "options": { @@ -209,7 +430,7 @@ "type": "__expr__", "uid": "__expr__" }, - "expression": "$Q_USAGE_BYTE * $QR_RULE_CAN_SHOW", + "expression": "($6_Q_USAGE_BYTE + $7_Q_USAGE_BYTE) * $QR_RULE_CAN_SHOW", "hide": false, "refId": "Unique Data Bytes", "type": "math" @@ -236,7 +457,19 @@ "hide": true, "legendFormat": "Unique Data Bytes", "range": true, - "refId": "Q_USAGE_BYTE" + "refId": "6_Q_USAGE_BYTE" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum((sum((sum ( (((aerospike_namespace_data_used_bytes {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} ) /\naerospike_namespace_data_compression_ratio {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}\n) \n/aerospike_namespace_effective_replication_factor {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}\n) \n- on (job, cluster_name, instance, service,ns) ( # begin calculate the header byte size 39bytes if as-version>=6 and 34 if as-version<6\n sum (\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build=~\"^4.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{ build=~\"^5.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build!~\"^5.*\"}*39) ) \n by (job, cluster_name, instance, service,ns) \n ) # end of header-byte calculation by aerospike version\n) by (service, ns)) or vector(0) ) by (ns))) \n\n) # end paranthesis for whole sum", + "hide": true, + "legendFormat": "Unique Data Bytes", + "range": true, + "refId": "7_Q_USAGE_BYTE" } ], "title": "Cluster - Unique Data Bytes ", @@ -249,7 +482,7 @@ "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 11 }, "id": 29, "panels": [ @@ -299,7 +532,8 @@ "mode": "absolute", "steps": [ { - "color": "#299c46" + "color": "#299c46", + "value": null } ] }, @@ -311,9 +545,9 @@ "h": 11, "w": 24, "x": 0, - "y": 6 + "y": 24 }, - "id": 36, + "id": 74, "interval": "5m", "links": [], "maxDataPoints": 150, @@ -336,7 +570,7 @@ "type": "__expr__", "uid": "__expr__" }, - "expression": "$Q_USAGE_BYTE * $QR_RULE_CAN_SHOW", + "expression": "($6_Q_USAGE_BYTE + $7_Q_USAGE_BYTE ) * $QR_RULE_CAN_SHOW", "hide": false, "refId": "Unique Data Bytes", "type": "math" @@ -363,15 +597,27 @@ "hide": true, "legendFormat": "Unique Data Bytes", "range": true, - "refId": "Q_USAGE_BYTE" + "refId": "6_Q_USAGE_BYTE" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum((sum((sum ( (((aerospike_namespace_data_used_bytes {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}+\naerospike_namespace_index_used_bytes {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} +\naerospike_namespace_sindex_used_bytes {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}) \n) \n/aerospike_namespace_effective_replication_factor {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}\n) \n- on (job, cluster_name, instance, service,ns) ( # begin calculate the header byte size 39bytes if as-version>=6 and 34 if as-version<6\n sum (\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build=~\"^4.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{ build=~\"^5.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build!~\"^5.*\"}*39) ) \n by (job, cluster_name, instance, service,ns) \n ) # end of header-byte calculation by aerospike version\n) by (service, ns)) or vector(0) ) by (ns))) \n\n) # end paranthesis for whole sum", + "hide": true, + "legendFormat": "Unique Data Bytes", + "range": true, + "refId": "7_Q_USAGE_BYTE" } ], - "title": "Cluster - Unique Data Usage ", + "title": "Historical - Usage Bytes", "transformations": [], "type": "timeseries" } ], - "title": "Historical - Usage Bytes", + "title": "Historical - Usage Bytes - $cluster", "type": "row" }, { @@ -380,11 +626,11 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 12 }, - "id": 2, + "id": 84, "panels": [], - "title": "Unique Data Usage", + "title": "Unique Data Usage - $cluster", "type": "row" }, { @@ -420,9 +666,9 @@ "h": 3, "w": 12, "x": 0, - "y": 7 + "y": 13 }, - "id": 6, + "id": 94, "links": [], "maxDataPoints": 100, "maxPerRow": 2, @@ -547,6 +793,18 @@ "legendFormat": "{{ns}} - in Memory", "range": true, "refId": "Q_MEMORY_USAGE" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_AEROSPIKE_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum((sum((sum ( (((aerospike_namespace_data_used_bytes {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} ) /\naerospike_namespace_data_compression_ratio {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}\n) \n/aerospike_namespace_effective_replication_factor {job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"}\n) \n- on (job, cluster_name, instance, service,ns) ( # begin calculate the header byte size 39bytes if as-version>=6 and 34 if as-version<6\n sum (\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build=~\"^4.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) (\n aerospike_node_up{ build=~\"^5.*\"}*35) \n or\n aerospike_namespace_master_objects{job=\"$job_name\", cluster_name=~\"$cluster\", service=~\"$node|$^\", ns=~\"$namespace|$^\"} * on (job, cluster_name, service) \n group_left(build) ( \n aerospike_node_up{ build!~\"^5.*\"}*39) ) \n by (job, cluster_name, instance, service,ns) \n ) # end of header-byte calculation by aerospike version\n) by (service, ns)) ) by (ns))) \n\n) # end paranthesis for whole sum", + "hide": false, + "legendFormat": "Unique Bytes -", + "range": true, + "refId": "7_DATA_USAGE" } ], "title": "Namespace - $namespace", @@ -691,6 +949,6 @@ "timezone": "", "title": "Unique Data Usage", "uid": "QFY8EJfVk", - "version": 2, + "version": 6, "weekStart": "" } \ No newline at end of file