feat: telemetry

chart/templates/deployment.yaml

@@ -69,6 +69,12 @@ spec:
             {{- if .Values.db.runMigrations}}
             - --db-migrations
             {{- end}}
+            {{- if (tpl .Values.otel.collector .) }}
+            - --otel-collector-url={{ tpl .Values.otel.collector . | default "" }}
+            {{- end }}
+            {{- if (tpl .Values.otel.serviceName .) }}
+            - --otel-service-name={{ tpl .Values.otel.serviceName . | default "config-db" }}
+            {{- end }}
           {{- if .Values.upstream.enabled}}
           envFrom:
             - secretRef:
@@ -90,6 +96,10 @@ spec:
             - name: UPSTREAM_PAGE_SIZE
               value: '{{ .Values.upstream.pageSize }}'
             {{- end}}
+            {{- if .Values.otel.labels}}
+            - name: OTEL_LABELS
+              value: '{{ .Values.otel.labels }}'
+            {{- end}}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

chart/values.yaml

@@ -5,7 +5,7 @@ global:
   db:
     connectionPooler:
       enabled: false
-      secretKeyRef: 
+      secretKeyRef:
         name: mission-control-connection-pooler
         key: DB_URL
 
@@ -30,6 +30,13 @@ configAnalysisRetentionDays: 60
 scrapeRuleConfigMaps:
   - config-db-rules
 
+otel:
+  # OpenTelemetry gRPC collector endpoint in host:port format
+  collector: ''
+  serviceName: config-db
+  # labels in "a=b,c=d" format
+  labels: 
+
 db:
   runMigrations: true
   embedded:
@@ -89,15 +96,15 @@ serviceAccount:
   rbac:
     # Whether to create cluster-wide or namespaced roles
     clusterRole: true
-    
+
     # for secret management with valueFrom
     tokenRequest: true
     secrets: true
     configmaps: true
-    
+
     # for use with kubernetes resource lookups
     readAll: true
-    
+
     # for kubernetesFile lookups
     exec: true
 

cmd/root.go

@@ -11,6 +11,7 @@ import (
 	"github.com/flanksource/config-db/jobs"
 	"github.com/flanksource/config-db/scrapers"
 	"github.com/flanksource/config-db/scrapers/kubernetes"
+	"github.com/flanksource/config-db/telemetry"
 	"github.com/flanksource/config-db/utils/kube"
 	"github.com/flanksource/duty"
 	"github.com/spf13/cobra"
@@ -30,6 +31,12 @@ var (
 	date    = "unknown"
 )
 
+var (
+	// Telemetry flag vars
+	otelcollectorURL string
+	otelServiceName  string
+)
+
 func readFromEnv(v string) string {
 	val := os.Getenv(v)
 	if val != "" {
@@ -59,6 +66,11 @@ var Root = &cobra.Command{
 		}
 		db.Schema = readFromEnv(db.Schema)
 		db.PGRSTLogLevel = readFromEnv(db.PGRSTLogLevel)
+
+		if otelcollectorURL != "" {
+			logger.Infof("Sending traces to %s", otelcollectorURL)
+			_ = telemetry.InitTracer(otelServiceName, otelcollectorURL, true) // TODO: Setup runner
+		}
 	},
 }
 
@@ -78,6 +90,9 @@ func ServerFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&publicEndpoint, "public-endpoint", "http://localhost:8080", "Public endpoint that this instance is exposed under")
 	flags.IntVar(&kubernetes.BufferSize, "watch-event-buffer", kubernetes.BufferSize, "Buffer size for kubernetes events")
 
+	flags.StringVar(&otelcollectorURL, "otel-collector-url", "", "OpenTelemetry gRPC Collector URL in host:port format")
+	flags.StringVar(&otelServiceName, "otel-service-name", "config-db", "OpenTelemetry service name for the resource")
+
 	// Flags for push/pull
 	var upstreamPageSizeDefault = 500
 	if val, exists := os.LookupEnv("UPSTREAM_PAGE_SIZE"); exists {

cmd/server.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/config-db/api"
@@ -18,6 +19,7 @@ import (
 	"github.com/labstack/echo/v4/middleware"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho"
 
 	"github.com/flanksource/config-db/scrapers"
 )
@@ -44,14 +46,18 @@ var Serve = &cobra.Command{
 }
 
 func serve(ctx context.Context, configFiles []string) {
-
 	e := echo.New()
+	e.Use(otelecho.Middleware("config-db", otelecho.WithSkipper(tracingURLSkipper)))
+
 	// PostgREST needs to know how it is exposed to create the correct links
 	db.HTTPEndpoint = publicEndpoint + "/db"
 
 	if logger.IsTraceEnabled() {
-		e.Use(middleware.Logger())
+		echoLogConfig := middleware.DefaultLoggerConfig
+		echoLogConfig.Skipper = tracingURLSkipper
+		e.Use(middleware.LoggerWithConfig(echoLogConfig))
 	}
+
 	if !disablePostgrest {
 		go db.StartPostgrest()
 		forward(e, "/db", db.PostgRESTEndpoint())
@@ -135,3 +141,14 @@ func forward(e *echo.Echo, prefix string, target string) {
 func init() {
 	ServerFlags(Serve.Flags())
 }
+
+// tracingURLSkipper ignores metrics route on some middleware
+func tracingURLSkipper(c echo.Context) bool {
+	pathsToSkip := []string{"/health", "/metrics"}
+	for _, p := range pathsToSkip {
+		if strings.HasPrefix(c.Path(), p) {
+			return true
+		}
+	}
+	return false
+}

go.mod

@@ -71,6 +71,12 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/uber/athenadriver v1.1.14
 	github.com/xo/dburl v0.13.1
+	go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0
+	go.opentelemetry.io/otel v1.26.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0
+	go.opentelemetry.io/otel/sdk v1.22.0
+	go.opentelemetry.io/otel/trace v1.26.0
 	gopkg.in/flanksource/yaml.v3 v3.2.3
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/gorm v1.25.5
@@ -156,13 +162,8 @@ require (
 	github.com/zclconf/go-cty v1.14.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel v1.22.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
-	go.opentelemetry.io/otel/metric v1.22.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/tools v0.20.0 // indirect
@@ -281,7 +282,7 @@ require (
 	google.golang.org/api v0.162.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
-	google.golang.org/grpc v1.63.2 // indirect
+	google.golang.org/grpc v1.63.2
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -297,6 +298,10 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
 
+replace go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.22.0
+
+replace go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.22.0
+
 // replace github.com/flanksource/duty => ../duty
 
 // replace github.com/flanksource/ketall => ../ketall

go.sum

@@ -1445,10 +1445,14 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0 h1:477zSmIXZy+324mzDsXGm1DPhHKWTFrT6iUIAlpI9f4=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0/go.mod h1:oZWJX6UZ7QPGvMSM/2T5yXdKOtFaey2IV/IMHY+tryg=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
+go.opentelemetry.io/contrib/propagators/b3 v1.26.0 h1:wgFbVA+bK2k+fGVfDOCOG4cfDAoppyr5sI2dVlh8MWM=
+go.opentelemetry.io/contrib/propagators/b3 v1.26.0/go.mod h1:DDktFXxA+fyItAAM0Sbl5OBH7KOsCTjvbBdPKtoIf/k=
 go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
 go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
@@ -1459,8 +1463,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCy
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
+go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
+go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
 go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
 go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
 go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=

hack/generate-schemas/go.mod

@@ -92,9 +92,9 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel v1.22.0 // indirect
-	go.opentelemetry.io/otel/metric v1.22.0 // indirect
-	go.opentelemetry.io/otel/trace v1.22.0 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect

hack/generate-schemas/go.sum

@@ -549,14 +549,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
-go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
-go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
+go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
+go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
+go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
+go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
 go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
 go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
-go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
+go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
+go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=

telemetry/tracer.go

@@ -0,0 +1,68 @@
+package telemetry
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"google.golang.org/grpc/credentials"
+
+	"github.com/flanksource/commons/collections"
+	"github.com/flanksource/commons/logger"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func InitTracer(serviceName, collectorURL string, insecure bool) func(context.Context) error {
+	var secureOption otlptracegrpc.Option
+	if !insecure {
+		secureOption = otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, ""))
+	} else {
+		secureOption = otlptracegrpc.WithInsecure()
+	}
+
+	exporter, err := otlptrace.New(
+		context.Background(),
+		otlptracegrpc.NewClient(
+			secureOption,
+			otlptracegrpc.WithEndpoint(collectorURL),
+		),
+	)
+
+	if err != nil {
+		logger.Errorf("failed to create opentelemetry exporter: %v", err)
+		return func(_ context.Context) error { return nil }
+	}
+
+	attributes := []attribute.KeyValue{attribute.String("service.name", serviceName)}
+	if val, ok := os.LookupEnv("OTEL_LABELS"); ok {
+		kv := collections.KeyValueSliceToMap(strings.Split(val, ","))
+		for k, v := range kv {
+			attributes = append(attributes, attribute.String(k, v))
+		}
+	}
+
+	resources, err := resource.New(context.Background(), resource.WithAttributes(attributes...))
+	if err != nil {
+		logger.Errorf("could not set opentelemetry resources: %v", err)
+		return func(_ context.Context) error { return nil }
+	}
+
+	otel.SetTracerProvider(
+		sdktrace.NewTracerProvider(
+			sdktrace.WithSampler(sdktrace.AlwaysSample()),
+			sdktrace.WithBatcher(exporter),
+			sdktrace.WithResource(resources),
+		),
+	)
+
+	// Register the TraceContext propagator globally.
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+
+	return exporter.Shutdown
+}