diff --git a/README.md b/README.md index c553313..41a72b9 100644 --- a/README.md +++ b/README.md @@ -232,21 +232,21 @@ Usage of k8s-image-availability-exporter: The following metrics for Prometheus are provided: -* `k8s_image_availability_exporter__available` — non-zero indicates *successful* image check. -* `k8s_image_availability_exporter__bad_image_format` — non-zero indicates incorrect `image` field format. -* `k8s_image_availability_exporter__absent` — non-zero indicates an image's manifest absence from container registry. -* `k8s_image_availability_exporter__registry_unavailable` — non-zero indicates general registry unavailiability, perhaps, due to network outage. -* `k8s_image_availability_exporter_deployment_registry_v1_api_not_supported` — non-zero indicates v1 Docker Registry API, these images are best ignored with `--ignored-images` cmdline parameter. -* `k8s_image_availability_exporter__authentication_failure` — non-zero indicates authentication error to container registry, verify imagePullSecrets. -* `k8s_image_availability_exporter__authorization_failure` — non-zero indicates authorization error to container registry, verify imagePullSecrets. -* `k8s_image_availability_exporter__unknown_error` — non-zero indicates an error that failed to be classified, consult exporter's logs for additional information. - -Each `` in the exporter's metrics name is replaced with the following values: - -* `deployment` -* `statefulset` -* `daemonset` -* `cronjob` +* `k8s_image_availability_exporter_available` — non-zero indicates *successful* image check. +* `k8s_image_availability_exporter_absent` — non-zero indicates an image's manifest absence from container registry. +* `k8s_image_availability_exporter_bad_image_format` — non-zero indicates incorrect `image` field format. +* `k8s_image_availability_exporter_registry_unavailable` — non-zero indicates general registry unavailiability, perhaps, due to network outage. +* `k8s_image_availability_exporter_authentication_failure` — non-zero indicates authentication error to container registry, verify imagePullSecrets. +* `k8s_image_availability_exporter_authorization_failure` — non-zero indicates authorization error to container registry, verify imagePullSecrets. +* `k8s_image_availability_exporter_unknown_error` — non-zero indicates an error that failed to be classified, consult exporter's logs for additional information. + +Each metric has the following labels: + +* `namespace` - namespace name +* `container` - container name +* `image` - image URL in the registry +* `kind` - Kubernetes controller kind, namely `deployment`, `statefulset`, `daemonset` or `cronjob` +* `name` - controller name ## Compatibility diff --git a/charts/k8s-image-availability-exporter/templates/prometheus-rule.yaml b/charts/k8s-image-availability-exporter/templates/prometheus-rule.yaml index 3cfdabe..859d0a2 100644 --- a/charts/k8s-image-availability-exporter/templates/prometheus-rule.yaml +++ b/charts/k8s-image-availability-exporter/templates/prometheus-rule.yaml @@ -10,52 +10,52 @@ spec: rules: - alert: DeploymentImageUnavailable expr: | - max by (namespace, deployment, container, image) ( - k8s_image_availability_exporter_deployment_available == 0 + max by (namespace, name, container, image) ( + k8s_image_availability_exporter_available{kind="deployment"} == 0 ) annotations: message: > Image {{`{{ $labels.image }}`}} from container {{`{{ $labels.container }}`}} - in deployment {{`{{ $labels.deployment }}`}} + in deployment {{`{{ $labels.name }}`}} from namespace {{`{{ $labels.namespace }}`}} is not available in docker registry. labels: severity: critical - alert: StatefulSetImageUnavailable expr: | - max by (namespace, statefulset, container, image) ( - k8s_image_availability_exporter_statefulset_available == 0 + max by (namespace, name, container, image) ( + k8s_image_availability_exporter_available{kind="statefulset"} == 0 ) annotations: message: > Image {{`{{ $labels.image }}`}} from container {{`{{ $labels.container }}`}} - in statefulSet {{`{{ $labels.statefulset }}`}} + in statefulSet {{`{{ $labels.name }}`}} from namespace {{`{{ $labels.namespace }}`}} is not available in docker registry. labels: severity: critical - alert: DaemonSetImageUnavailable expr: | - max by (namespace, daemonset, container, image) ( - k8s_image_availability_exporter_daemonset_available == 0 + max by (namespace, name, container, image) ( + k8s_image_availability_exporter_available{kind="daemonset"} == 0 ) annotations: message: > Image {{`{{ $labels.image }}`}} from container {{`{{ $labels.container }}`}} - in daemonSet {{`{{ $labels.daemonset }}`}} + in daemonSet {{`{{ $labels.name }}`}} from namespace {{`{{ $labels.namespace }}`}} is not available in docker registry. labels: severity: critical - alert: CronJobImageUnavailable expr: | - max by (namespace, cronjob, container, image) ( - k8s_image_availability_exporter_cronjob_available == 0 + max by (namespace, name, container, image) ( + k8s_image_availability_exporter_available{kind="cronjob"} == 0 ) annotations: message: > Image {{`{{ $labels.image }}`}} from container {{`{{ $labels.container }}`}} - in cronJob {{`{{ $labels.cronjob }}`}} + in cronJob {{`{{ $labels.name }}`}} from namespace {{`{{ $labels.namespace }}`}} is not available in docker registry. labels: diff --git a/pkg/store/image_store.go b/pkg/store/image_store.go index a2cc133..d1462e2 100644 --- a/pkg/store/image_store.go +++ b/pkg/store/image_store.go @@ -1,7 +1,6 @@ package store import ( - "fmt" "strings" "sync" "time" @@ -217,27 +216,14 @@ func newNamedConstMetrics(ownerKind, ownerName, namespace, container, image stri "namespace": namespace, "container": container, "image": image, + "kind": strings.ToLower(ownerKind), + "name": ownerName, } - switch ownerKind { - case "Deployment": - labels["deployment"] = ownerName - return getMetricByControllerKind(ownerKind, labels, avalMode) - case "StatefulSet": - labels["statefulset"] = ownerName - return getMetricByControllerKind(ownerKind, labels, avalMode) - case "DaemonSet": - labels["daemonset"] = ownerName - return getMetricByControllerKind(ownerKind, labels, avalMode) - case "CronJob": - labels["cronjob"] = ownerName - return getMetricByControllerKind(ownerKind, labels, avalMode) - default: - panic(fmt.Sprintf("received unknown metric name: %s", ownerKind)) - } + return getMetric(labels, avalMode) } -func getMetricByControllerKind(controllerKind string, labels map[string]string, mode AvailabilityMode) (ret []prometheus.Metric) { +func getMetric(labels map[string]string, mode AvailabilityMode) (ret []prometheus.Metric) { for availMode, desc := range AvailabilityModeDescMap { var value float64 if availMode == mode { @@ -245,7 +231,7 @@ func getMetricByControllerKind(controllerKind string, labels map[string]string, } ret = append(ret, prometheus.MustNewConstMetric( - prometheus.NewDesc("k8s_image_availability_exporter_"+strings.ToLower(controllerKind)+"_"+desc, "", nil, labels), + prometheus.NewDesc("k8s_image_availability_exporter_"+desc, "", nil, labels), prometheus.GaugeValue, value, )) diff --git a/pkg/store/image_store_test.go b/pkg/store/image_store_test.go index 83526a0..8735c18 100644 --- a/pkg/store/image_store_test.go +++ b/pkg/store/image_store_test.go @@ -5,38 +5,46 @@ import ( "strings" "testing" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func insertImagesIntoStore(t *testing.T, store *ImageStore, successfulChecks, failedChecks int) { +func insertImagesIntoStore(t *testing.T, store *ImageStore, successfulChecks, failedChecks int, info []ContainerInfo) { t.Helper() - dummyContainerInfos := []ContainerInfo{ - { - Namespace: "test", - ControllerKind: "Deployment", - ControllerName: "test", - Container: "test", - }, - } - for i := 0; i < successfulChecks; i++ { - store.ReconcileImage(fmt.Sprintf("test_%d", i), dummyContainerInfos) + store.ReconcileImage(fmt.Sprintf("test_%d", i), info) } for i := 0; i < failedChecks; i++ { - store.ReconcileImage(fmt.Sprintf("fail_%d", i), dummyContainerInfos) + store.ReconcileImage(fmt.Sprintf("fail_%d", i), info) } } func TestImageStore_AddOrUpdateImage(t *testing.T) { store := NewImageStore(reconcile(t), 2, 3) - insertImagesIntoStore(t, store, 3, 2) + info := []ContainerInfo{ + { + Namespace: "test", + ControllerKind: "Deployment", + ControllerName: "test", + Container: "test", + }, + { + Namespace: "test", + ControllerKind: "StatefulSet", + ControllerName: "test", + Container: "test", + }, + } + + insertImagesIntoStore(t, store, 3, 2, info) store.Check() metrics := store.ExtractMetrics() - require.Len(t, metrics, 35) + require.Len(t, metrics, 70) } func reconcile(t *testing.T) func(imageName string) AvailabilityMode { @@ -50,3 +58,348 @@ func reconcile(t *testing.T) func(imageName string) AvailabilityMode { return Available } } + +func TestImageStore_ExtractMetrics(t *testing.T) { + t.Parallel() + + t.Run("no images", func(t *testing.T) { + t.Parallel() + + store := NewImageStore(reconcile(t), 2, 3) + insertImagesIntoStore(t, store, 0, 0, nil) + store.Check() + + metrics := store.ExtractMetrics() + assert.Empty(t, metrics) + }) + + t.Run("one container", func(t *testing.T) { + t.Parallel() + + store := NewImageStore(reconcile(t), 2, 3) + + info := []ContainerInfo{ + { + Namespace: "test_ns", + ControllerKind: "Deployment", + ControllerName: "test_name", + Container: "test_container", + }, + } + + expectedMetrics := []*prometheus.Desc{ + prometheus.NewDesc( + "k8s_image_availability_exporter_registry_unavailable", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_authentication_failure", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_authorization_failure", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_unknown_error", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_available", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_absent", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_bad_image_format", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + } + + insertImagesIntoStore(t, store, 1, 0, info) + store.Check() + + metrics := store.ExtractMetrics() + require.Len(t, metrics, len(expectedMetrics)) + + expectedMetricsStr := make([]string, 0, len(expectedMetrics)) + for _, em := range expectedMetrics { + expectedMetricsStr = append(expectedMetricsStr, em.String()) + } + + returnedMetricsStr := make([]string, 0, len(metrics)) + for _, m := range metrics { + returnedMetricsStr = append(returnedMetricsStr, m.Desc().String()) + } + + assert.ElementsMatch(t, expectedMetricsStr, returnedMetricsStr) + }) + + t.Run("two containers, different kind", func(t *testing.T) { + t.Parallel() + + store := NewImageStore(reconcile(t), 2, 3) + + info := []ContainerInfo{ + { + Namespace: "test_ns", + ControllerKind: "Deployment", + ControllerName: "test_name", + Container: "test_container", + }, + { + Namespace: "test_ns2", + ControllerKind: "StatefulSet", + ControllerName: "test_name2", + Container: "test_container2", + }, + } + + expectedMetrics := []*prometheus.Desc{ + prometheus.NewDesc( + "k8s_image_availability_exporter_registry_unavailable", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_authentication_failure", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_authorization_failure", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_unknown_error", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_available", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_absent", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_bad_image_format", + "", + nil, + prometheus.Labels{ + "container": "test_container", + "image": "test_0", + "kind": "deployment", + "name": "test_name", + "namespace": "test_ns", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_registry_unavailable", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_authentication_failure", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_authorization_failure", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_unknown_error", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_available", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_absent", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + prometheus.NewDesc( + "k8s_image_availability_exporter_bad_image_format", + "", + nil, + prometheus.Labels{ + "container": "test_container2", + "image": "test_0", + "kind": "statefulset", + "name": "test_name2", + "namespace": "test_ns2", + }, + ), + } + + insertImagesIntoStore(t, store, 1, 0, info) + store.Check() + + metrics := store.ExtractMetrics() + require.Len(t, metrics, len(expectedMetrics)) + + expectedMetricsStr := make([]string, 0, len(expectedMetrics)) + for _, em := range expectedMetrics { + expectedMetricsStr = append(expectedMetricsStr, em.String()) + } + + returnedMetricsStr := make([]string, 0, len(metrics)) + for _, m := range metrics { + returnedMetricsStr = append(returnedMetricsStr, m.Desc().String()) + } + + assert.ElementsMatch(t, expectedMetricsStr, returnedMetricsStr) + }) +}