Merge pull request #174 from hmariset/exporter_update

feat(RHTAPWATCH-570): Update exporters code with actual metrics

exporters/dsexporter/dsexporter.go

@@ -1,39 +1,108 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
+	"errors"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
 )
 
+const service = "grafana"
+const check = "prometheus-appstudio-ds"
+var allDataSources = GetDataSources
+
 type CustomCollector struct {
-	requestCounter prometheus.Counter
+	konfluxUp *prometheus.GaugeVec
 }
 
 // Creating a new instance of CustomCollector.
 func NewCustomCollector() *CustomCollector {
 	return &CustomCollector{
-		requestCounter: prometheus.NewCounter(prometheus.CounterOpts{
-			Name: "request_count",
-			Help: "Number of requests handled by the handler",
-		}),
+		konfluxUp: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "konflux_up",
+			Help: "Availability of the Konflux default grafana datasource",
+		},
+		[]string{"service", "check"}),
 	}
 }
 
 // Describe method sends descriptions of the metrics to Prometheus.
 // When Prometheus scrapes the /metrics endpoint of the exporter,
 // it first calls the Describe method to get a description of all the metrics.
 func (e *CustomCollector) Describe(ch chan<- *prometheus.Desc) {
-	e.requestCounter.Describe(ch)
+	e.konfluxUp.Describe(ch)
 }
 
 // Collect method sends the current values of the metrics to Prometheus.
 // After Prometheus understands what metrics are available (using the `Describe` method),
 // it then calls the `Collect` method to actually get the values of those metrics.
 func (e *CustomCollector) Collect(ch chan<- prometheus.Metric) {
-	e.requestCounter.Collect(ch)
+	var availability float64
+	clientset := kubernetes.NewForConfigOrDie(config.GetConfigOrDie())
+
+	if grafanaRes, err := GetGrafanaResource(clientset); err == nil {
+		if datasources, errB := allDataSources(grafanaRes); errB == nil {
+			availability = GetDataSourceAvailability(datasources, check)
+		}
+	}
+
+	e.konfluxUp.WithLabelValues(service, check).Set(availability)
+	e.konfluxUp.Collect(ch)
+}
+
+// get the grafana resource as a map
+func GetGrafanaResource(clientset *kubernetes.Clientset) (map[string]interface{}, error) {
+	data, err := clientset.RESTClient().
+		Get().
+		AbsPath("/apis/grafana.integreatly.org/v1beta1").
+		Namespace("appstudio-grafana").
+		Resource("grafanas").
+		Name("grafana-oauth").
+		DoRaw(context.TODO())
+	var grafanaResource map[string]interface{}
+	err = json.Unmarshal(data, &grafanaResource)
+	if err != nil {
+		return nil, errors.New("Error getting grafana resource")
+	}
+
+	return grafanaResource, nil
+}
+
+// get datasources from grafana resource
+func GetDataSources(grafanaResource map[string]interface{}) ([]string, error) {
+	// return empty string slice if datasources are not defined
+	if v, exists := grafanaResource["status"].(map[string]any); exists {
+		if v["datasources"] == nil {
+			return []string{}, nil
+		}
+	} else {
+		return nil, errors.New("Error retrieving status key") 
+	}
+	datasourcesIfc := grafanaResource["status"].(map[string]any)["datasources"].([]interface{})
+	datasources := make([]string, len(datasourcesIfc))
+	for i, v := range datasourcesIfc {
+		datasources[i] = v.(string)
+	}
+	return datasources, nil
+}
+
+// check if datasource exists, return 1 if yes, 0 if not
+func GetDataSourceAvailability(datasources []string, dsToCheck string) float64 {
+	for _, datasource := range datasources {
+		if strings.Contains(datasource, dsToCheck) {
+			fmt.Println("Datasource", datasource, "exists")
+			return 1
+		}
+	}
+	fmt.Println("Datasource", dsToCheck, "does not exist")
+	return 0
 }
 
 // Using a separate pedantic registry ensures that only your custom metric is exposed on the "/metrics" endpoint,
@@ -50,6 +119,7 @@ func main() {
 			Registry:          reg,
 		},
 	))
+
 	fmt.Println("Server is listening on http://localhost:8090/metrics")
 	http.ListenAndServe(":8090", nil)
 }

exporters/dsexporter/dsexporter_test.go

@@ -2,45 +2,136 @@ package main
 
 import (
 	"testing"
+	"reflect"
+	"net/http"
+	"net/http/httptest"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )
 
-func TestCustomCollector(t *testing.T) {
-	exporter := NewCustomCollector()
-	prometheus.MustRegister(exporter)
-
-	// Simulate collecting metrics and check the exported metric value.
-	metrics := prometheus.DefaultGatherer
-	metricFamilies, err := metrics.Gather()
-	assert.NoError(t, err)
-
-	var RequestCountValue float64
-	for _, mf := range metricFamilies {
-		if mf.GetName() == "request_count" {
-			RequestCountValue = mf.GetMetric()[0].GetCounter().GetValue()
-			break
+func TestGetDataSourceAvailability(t *testing.T) {
+	datasources := []string{"appstudio-grafana/prometheus-appstudio-ds/b83cfcd5-3012-4e6a-b044-6923e1fef2d8", "prometheus-appstudio/b83cfcd5-3012-4e6a-b044-6923e1fef2d8", "appstudio-ds"}
+	check := "prometheus-appstudio-ds"
+	result := GetDataSourceAvailability(datasources, check)
+	if result != 1 {
+		t.Errorf("Expected result 1, but got %f", result)
+	}
+
+	check = "prometheus-appstudio-dsexporter"
+	result = GetDataSourceAvailability(datasources, check)
+	if result != 0 {
+		t.Errorf("Expected result 0, but got %f", result)
+	}
+}
+
+func TestGetGrafanaResource(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/apis/grafana.integreatly.org/v1beta1/namespaces/appstudio-grafana/grafanas/grafana-oauth" {
+			t.Errorf("Unexpected request path: %s", r.URL.Path)
 		}
+		response := `{"test": "data"}`
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(response))
+	}))
+	defer server.Close()
+
+	config := &rest.Config{
+		Host: server.URL,
 	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if(err != nil) {
+		t.Fatalf("Error: %v", err)
+	}
+	result, errB := GetGrafanaResource(clientset)
+	expectedResult := map[string]interface{}{"test": "data"}
 
-	// Check whether the exported metric value is initially 0.
-	assert.Equal(t, float64(0), RequestCountValue)
+	if errB != nil {
+		t.Errorf("Error %v when getting grafana resources", errB)
+	}
 
-	// Increment the requestCounter by calling the Inc method.
-	exporter.requestCounter.Inc()
+	if !reflect.DeepEqual(result, expectedResult) {
+		t.Errorf("Expected %v, but got %v", expectedResult, result)
+	}
+}
 
-	// Collecting metrics again
-	metricFamilies, err = metrics.Gather()
-	assert.NoError(t, err)
+func TestGetDataSources(t *testing.T) {
+	grafanaRes := map[string]interface{}{
+		"status": map[string]interface{}{
+			"datasources": []interface{}{"appstudio-ds1", "appstudio-ds2"},
+		},
+	}
 
-	for _, mf := range metricFamilies {
-		if mf.GetName() == "request_count" {
-			RequestCountValue = mf.GetMetric()[0].GetCounter().GetValue()
-			break
-		}
+	expectedResult := []string{"appstudio-ds1", "appstudio-ds2"}
+	result, err := GetDataSources(grafanaRes)
+
+	if err != nil {
+		t.Errorf("Error %v when getting datasources", err)
+	}
+
+	if !reflect.DeepEqual(result, expectedResult) {
+		t.Errorf("Test-1 failed, Expected %v, but got %v", expectedResult, result)
 	}
 
-	// Check whether the exported metric value is now 1 after incrementing.
-	assert.Equal(t, float64(1), RequestCountValue)
+	// empty datasources
+	grafanaRes = map[string]interface{}{
+		"status": map[string]interface{}{
+			"datasources": []interface{}{},
+		},
+	}
+
+	expectedResult = []string{}
+	result, _ = GetDataSources(grafanaRes)
+
+	if !reflect.DeepEqual(result, expectedResult) {
+		t.Errorf("Test-2 failed, Expected %v, but got %v", expectedResult, result)
+	}
+
+	grafanaRes = map[string]interface{}{
+		"data": map[string]interface{}{
+			"datasources": []interface{}{"appstudio-ds1", "appstudio-ds2"},
+		},
+	}
+
+	_, err = GetDataSources(grafanaRes)
+	assert.ErrorContains(t, err, "Error retrieving status key")
+}
+
+func MockGetDataSources(grafanaResource map[string]interface{}) ([]string, error) {
+	return []string{"appstudio-ds1", "appstudio-ds2"}, nil
+}
+
+func MockGetDataSourcesExists(grafanaResource map[string]interface{}) ([]string, error) {
+	return []string{"appstudio-ds1", "appstudio-grafana/prometheus-appstudio-ds/b83cfcd5-3012-4e6a-b044-6923e1fef2d8"}, nil
+}
+
+func TestMain(t *testing.T) {
+	reg := prometheus.NewPedanticRegistry()
+	exporter := NewCustomCollector()
+	reg.MustRegister(exporter)
+
+	allDataSources = MockGetDataSources
+	handler := promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler.ServeHTTP(w, r)
+	}))
+	defer server.Close()
+
+	res, err := http.Get(server.URL + "/metrics")
+	if err != nil {
+		t.Fatalf("Failed to perform GET request: %v", err)
+	}
+	defer res.Body.Close()
+
+	assert.Equal(t, http.StatusOK, res.StatusCode)
+	assert.Equal(t, 1, testutil.CollectAndCount(exporter))
+	assert.Equal(t, float64(0), testutil.ToFloat64(exporter))
+
+	allDataSources = MockGetDataSourcesExists
+	assert.Equal(t, float64(1), testutil.ToFloat64(exporter))
 }

go.mod

@@ -1,21 +1,62 @@
 module github.com/redhat-appstudio/o11y.git
 
-go 1.20
+go 1.21
 
-require github.com/prometheus/client_golang v1.17.0
+require github.com/prometheus/client_golang v1.18.0
+
+require (
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/api v0.29.0 // indirect
+	k8s.io/apimachinery v0.29.0 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
-	golang.org/x/sys v0.11.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/sys v0.16.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/client-go v0.29.0 // indirect
+	sigs.k8s.io/controller-runtime v0.17.0 // indirect
 )