chore(k8s/amour): organise vm rules with apps

uhthomas · Jan 8, 2024 · 76b037d · 76b037d
1 parent f04907e
commit 76b037d
Show file tree

Hide file tree

Showing 7 changed files with 468 additions and 434 deletions.
diff --git a/k8s/amour/kube_state_metrics/BUILD.bazel b/k8s/amour/kube_state_metrics/BUILD.bazel
@@ -12,11 +12,13 @@ cue_library(
         "service_account_list.cue",
         "service_list.cue",
         "stateful_set_list.cue",
+        "vm_rule_list.cue",
         "vm_service_scrape_list.cue",
     ],
     importpath = "github.com/uhthomas/automata/k8s/amour/kube_state_metrics",
     visibility = ["//visibility:public"],
     deps = [
+        "//cue.mod/gen/github.com/VictoriaMetrics/operator/api/victoriametrics/v1beta1:cue_v1beta1_library",
         "//cue.mod/gen/k8s.io/api/admissionregistration/v1:cue_v1_library",
         "//cue.mod/gen/k8s.io/api/apps/v1:cue_v1_library",
         "//cue.mod/gen/k8s.io/api/authentication/v1:cue_v1_library",

diff --git a/k8s/amour/kube_state_metrics/list.cue b/k8s/amour/kube_state_metrics/list.cue
@@ -39,5 +39,6 @@ _items: [
 	#ServiceAccountList.items,
 	#ServiceList.items,
 	#StatefulSetList.items,
+	#VMRuleList.items,
 	#VMServiceScrapeList.items,
 ]
diff --git a/k8s/amour/kube_state_metrics/vm_rule_list.cue b/k8s/amour/kube_state_metrics/vm_rule_list.cue
@@ -0,0 +1,78 @@
+package kube_state_metrics
+
+import victoriametricsv1beta1 "github.com/VictoriaMetrics/operator/api/victoriametrics/v1beta1"
+
+#VMRuleList: victoriametricsv1beta1.#VMRuleList & {
+	apiVersion: "operator.victoriametrics.com/v1beta1"
+	kind:       "VMRuleList"
+	items: [...{
+		apiVersion: "operator.victoriametrics.com/v1beta1"
+		kind:       "VMRule"
+	}]
+}
+
+#VMRuleList: items: [{
+	metadata: name: "kube-state-metrics"
+	spec: groups: [{
+		name: "kube-state-metrics"
+		rules: [{
+			alert: "KubeStateMetricsListErrors"
+			annotations: {
+				description: "kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all."
+				runbook_url: "https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricslisterrors"
+				summary:     "kube-state-metrics is experiencing errors in list operations."
+			}
+			expr: """
+				(sum(rate(kube_state_metrics_list_total{job=\"kube-state-metrics\",result=\"error\"}[5m]))
+				  /
+				sum(rate(kube_state_metrics_list_total{job=\"kube-state-metrics\"}[5m])))
+				> 0.01
+				"""
+
+			for: "15m"
+			labels: severity: "critical"
+		}, {
+			alert: "KubeStateMetricsWatchErrors"
+			annotations: {
+				description: "kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all."
+				runbook_url: "https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricswatcherrors"
+				summary:     "kube-state-metrics is experiencing errors in watch operations."
+			}
+			expr: """
+				(sum(rate(kube_state_metrics_watch_total{job=\"kube-state-metrics\",result=\"error\"}[5m]))
+				  /
+				sum(rate(kube_state_metrics_watch_total{job=\"kube-state-metrics\"}[5m])))
+				> 0.01
+				"""
+
+			for: "15m"
+			labels: severity: "critical"
+		}, {
+			alert: "KubeStateMetricsShardingMismatch"
+			annotations: {
+				description: "kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all."
+				runbook_url: "https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardingmismatch"
+				summary:     "kube-state-metrics sharding is misconfigured."
+			}
+			expr: "stdvar (kube_state_metrics_total_shards{job=\"kube-state-metrics\"}) != 0"
+			for:  "15m"
+			labels: severity: "critical"
+		}, {
+			alert: "KubeStateMetricsShardsMissing"
+			annotations: {
+				description: "kube-state-metrics shards are missing, some Kubernetes objects are not being exposed."
+				runbook_url: "https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardsmissing"
+				summary:     "kube-state-metrics shards are missing."
+			}
+			expr: """
+				2^max(kube_state_metrics_total_shards{job=\"kube-state-metrics\"}) - 1
+				  -
+				sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job=\"kube-state-metrics\"}) )
+				!= 0
+				"""
+
+			for: "15m"
+			labels: severity: "critical"
+		}]
+	}]
+}]
diff --git a/k8s/amour/node_exporter/BUILD.bazel b/k8s/amour/node_exporter/BUILD.bazel
@@ -8,11 +8,13 @@ cue_library(
         "namespace_list.cue",
         "service_account_list.cue",
         "service_list.cue",
+        "vm_rule_list.cue",
         "vm_service_scrape_list.cue",
     ],
     importpath = "github.com/uhthomas/automata/k8s/amour/node_exporter",
     visibility = ["//visibility:public"],
     deps = [
+        "//cue.mod/gen/github.com/VictoriaMetrics/operator/api/victoriametrics/v1beta1:cue_v1beta1_library",
         "//cue.mod/gen/k8s.io/api/apps/v1:cue_v1_library",
         "//cue.mod/gen/k8s.io/api/core/v1:cue_v1_library",
     ],

diff --git a/k8s/amour/node_exporter/list.cue b/k8s/amour/node_exporter/list.cue
@@ -33,5 +33,6 @@ _items: [
 	#NamespaceList.items,
 	#ServiceAccountList.items,
 	#ServiceList.items,
+	#VMRuleList.items,
 	#VMServiceScrapeList.items,
 ]