feat: add a new check error metric

[skip ci]
flanksource · Oct 25, 2024 · 79d3cd3 · 79d3cd3
1 parent ab0fc06
commit 79d3cd3
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 6 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -39,9 +39,7 @@ func InitContext() (context.Context, error) {
 	}
 
 	ctx.WithTracer(otel.GetTracerProvider().Tracer(app))
-	ctx.WithTracer(otel.GetTracerProvider().Tracer("canary-checker"))
-	ctx.WithTracer(otel.GetTracerProvider().Tracer("canary-checker"))
-	ctx.DB().Use(db.NewPlugin())
+	ctx.DB().Use(db.NewOopsPlugin())
 
 	return ctx, nil
 }

diff --git a/go.mod b/go.mod
@@ -54,6 +54,7 @@ require (
 	github.com/robertkrimen/otto v0.3.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/samber/lo v1.47.0
+	github.com/samber/oops v1.13.1
 	github.com/sethvargo/go-retry v0.3.0
 	github.com/sevennt/echo-pprof v0.1.1-0.20220616082843-66a461746b5f
 	github.com/spf13/cobra v1.8.0
@@ -250,7 +251,6 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rodaine/table v1.3.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	github.com/samber/oops v1.13.1 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
@@ -327,7 +327,7 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
 
-replace github.com/flanksource/duty => ../duty
+// replace github.com/flanksource/duty => ../duty
 
 // replace github.com/flanksource/artifacts => ../artifacts
 

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -115,6 +115,7 @@ var (
 	OpsCount        *prometheus.CounterVec
 	OpsFailedCount  *prometheus.CounterVec
 	OpsSuccessCount *prometheus.CounterVec
+	OpsErrorCount   *prometheus.CounterVec
 	RequestLatency  *prometheus.HistogramVec
 )
 
@@ -278,7 +279,13 @@ func Record(
 		Gauge.WithLabelValues(gaugeLabels...).Set(1)
 
 		CanaryCheckInfo.WithLabelValues(checkMetricLabels...).Set(1)
-		OpsFailedCount.WithLabelValues(checkMetricLabels...).Inc()
+
+		if result.InternalError {
+			OpsErrorCount.WithLabelValues(checkMetricLabels...).Inc()
+		} else {
+			fail.Append(1)
+			OpsFailedCount.WithLabelValues(checkMetricLabels...).Inc()
+		}
 	}
 
 	_uptime = types.Uptime{Passed: int(pass.Reduce(rolling.Sum)), Failed: int(fail.Reduce(rolling.Sum))}