Merge pull request #964 from jkroepke/fix-group_left

fix: Error "multiple matches for labels: many-to-one matching must beexplicit (group_left/group_right)" on Kubernetes/Networking Dashboards
kubernetes-monitoring · Aug 18, 2024 · 8c0479c · 8c0479c
2 parents 1b71e39 + 34dae8c
commit 8c0479c
Show file tree

Hide file tree

Showing 3 changed files with 414 additions and 54 deletions.
diff --git a/dashboards/network-usage/cluster-total.libsonnet b/dashboards/network-usage/cluster-total.libsonnet
@@ -66,8 +66,16 @@ local var = g.dashboard.variable;
         + tsPanel.standardOptions.withUnit('binBps')
         + tsPanel.queryOptions.withTargets([
           prometheus.new(
-            '${datasource}',
-            'sum by (namespace) (rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config
+            '${datasource}', |||
+              sum by (namespace) (
+                  rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
           )
           + prometheus.withLegendFormat('__auto'),
         ]),
@@ -76,44 +84,124 @@ local var = g.dashboard.variable;
         + tsPanel.standardOptions.withUnit('binBps')
         + tsPanel.queryOptions.withTargets([
           prometheus.new(
-            '${datasource}',
-            'sum by (namespace) (rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config
+            '${datasource}', |||
+              sum by (namespace) (
+                  rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
           )
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         table.new('Current Status')
         + table.gridPos.withW(24)
         + table.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'avg by (namespace) (rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            avg by (namespace) (
+                rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'avg by (namespace) (rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            avg by (namespace) (
+                rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_receive_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_receive_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_transmit_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_transmit_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_receive_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_receive_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
 
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_transmit_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_transmit_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withInstant(true)
           + prometheus.withFormat('table'),
         ])
@@ -213,8 +301,16 @@ local var = g.dashboard.variable;
         + tsPanel.standardOptions.withUnit('binBps')
         + tsPanel.queryOptions.withTargets([
           prometheus.new(
-            '${datasource}',
-            'avg by (namespace) (rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config
+            '${datasource}', |||
+              avg by (namespace) (
+                  rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
           )
           + prometheus.withLegendFormat('__auto'),
         ]),
@@ -223,8 +319,16 @@ local var = g.dashboard.variable;
         + tsPanel.standardOptions.withUnit('binBps')
         + tsPanel.queryOptions.withTargets([
           prometheus.new(
-            '${datasource}',
-            'avg by (namespace) (rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config
+            '${datasource}', |||
+              avg by (namespace) (
+                  rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
           )
           + prometheus.withLegendFormat('__auto'),
         ]),
@@ -233,8 +337,16 @@ local var = g.dashboard.variable;
         + tsPanel.standardOptions.withUnit('binBps')
         + tsPanel.queryOptions.withTargets([
           prometheus.new(
-            '${datasource}',
-            'sum by (namespace) (rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config
+            '${datasource}', |||
+              sum by (namespace) (
+                  rate(container_network_receive_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
           )
           + prometheus.withLegendFormat('__auto'),
         ]),
@@ -243,51 +355,117 @@ local var = g.dashboard.variable;
         + tsPanel.standardOptions.withUnit('binBps')
         + tsPanel.queryOptions.withTargets([
           prometheus.new(
-            '${datasource}',
-            'sum by (namespace) (rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config
+            '${datasource}', |||
+              sum by (namespace) (
+                  rate(container_network_transmit_bytes_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
           )
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         tsPanel.new('Rate of Received Packets')
         + tsPanel.standardOptions.withUnit('pps')
         + tsPanel.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_receive_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_receive_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         tsPanel.new('Rate of Transmitted Packets')
         + tsPanel.standardOptions.withUnit('pps')
         + tsPanel.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_transmit_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_transmit_packets_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         tsPanel.new('Rate of Received Packets Dropped')
         + tsPanel.standardOptions.withUnit('pps')
         + tsPanel.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_receive_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_receive_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         tsPanel.new('Rate of Transmitted Packets Dropped')
         + tsPanel.standardOptions.withUnit('pps')
         + tsPanel.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (namespace) (rate(container_network_transmit_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new('${datasource}', |||
+            sum by (namespace) (
+                rate(container_network_transmit_packets_dropped_total{%(clusterLabel)s="$cluster",namespace!=""}[%(grafanaIntervalVar)s])
+              * on (%(clusterLabel)s,namespace,pod) group_left ()
+                topk by (%(clusterLabel)s,namespace,pod) (
+                  1,
+                  max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                )
+            )
+          ||| % $._config)
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         tsPanel.new('Rate of TCP Retransmits out of all sent segments')
         + tsPanel.standardOptions.withUnit('percentunit')
         + tsPanel.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (instance) (rate(node_netstat_Tcp_RetransSegs{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s]) / rate(node_netstat_Tcp_OutSegs{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new(
+            '${datasource}', |||
+              sum by (instance) (
+                  rate(node_netstat_Tcp_RetransSegs{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s]) / rate(node_netstat_Tcp_OutSegs{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
+          )
           + prometheus.withLegendFormat('__auto'),
         ]),
 
         tsPanel.new('Rate of TCP SYN Retransmits out of all retransmits')
         + tsPanel.standardOptions.withUnit('percentunit')
         + tsPanel.queryOptions.withTargets([
-          prometheus.new('${datasource}', 'sum by (instance) (rate(node_netstat_TcpExt_TCPSynRetrans{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s]) / rate(node_netstat_Tcp_RetransSegs{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s]) * on (%(clusterLabel)s,namespace,pod) kube_pod_info{host_network="false"})' % $._config)
+          prometheus.new(
+            '${datasource}', |||
+              sum by (instance) (
+                  rate(node_netstat_TcpExt_TCPSynRetrans{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s]) / rate(node_netstat_Tcp_RetransSegs{%(clusterLabel)s="$cluster"}[%(grafanaIntervalVar)s])
+                * on (%(clusterLabel)s,namespace,pod) group_left ()
+                  topk by (%(clusterLabel)s,namespace,pod) (
+                    1,
+                    max by (%(clusterLabel)s,namespace,pod) (kube_pod_info{host_network="false"})
+                  )
+              )
+            ||| % $._config
+          )
           + prometheus.withLegendFormat('__auto'),
         ]),
       ];