diff --git a/cmd/root.go b/cmd/root.go index 5fd788567..7e175627e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( "github.com/flanksource/config-db/jobs" "github.com/flanksource/config-db/scrapers" "github.com/flanksource/config-db/scrapers/kubernetes" + "github.com/flanksource/config-db/telemetry" "github.com/flanksource/config-db/utils/kube" "github.com/flanksource/duty" "github.com/spf13/cobra" @@ -30,6 +31,12 @@ var ( date = "unknown" ) +var ( + // Telemetry flag vars + otelcollectorURL string + otelServiceName string +) + func readFromEnv(v string) string { val := os.Getenv(v) if val != "" { @@ -59,6 +66,11 @@ var Root = &cobra.Command{ } db.Schema = readFromEnv(db.Schema) db.PGRSTLogLevel = readFromEnv(db.PGRSTLogLevel) + + if otelcollectorURL != "" { + logger.Infof("Sending traces to %s", otelcollectorURL) + _ = telemetry.InitTracer(otelServiceName, otelcollectorURL, true) // TODO: Setup runner + } }, } @@ -78,6 +90,9 @@ func ServerFlags(flags *pflag.FlagSet) { flags.StringVar(&publicEndpoint, "public-endpoint", "http://localhost:8080", "Public endpoint that this instance is exposed under") flags.IntVar(&kubernetes.BufferSize, "watch-event-buffer", kubernetes.BufferSize, "Buffer size for kubernetes events") + flags.StringVar(&otelcollectorURL, "otel-collector-url", "", "OpenTelemetry gRPC Collector URL in host:port format") + flags.StringVar(&otelServiceName, "otel-service-name", "config-db", "OpenTelemetry service name for the resource") + // Flags for push/pull var upstreamPageSizeDefault = 500 if val, exists := os.LookupEnv("UPSTREAM_PAGE_SIZE"); exists { diff --git a/cmd/server.go b/cmd/server.go index 74018c710..c3c91fb05 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + "strings" "github.com/flanksource/commons/logger" "github.com/flanksource/config-db/api" @@ -18,6 +19,7 @@ import ( "github.com/labstack/echo/v4/middleware" prom "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" + "go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho" "github.com/flanksource/config-db/scrapers" ) @@ -44,14 +46,18 @@ var Serve = &cobra.Command{ } func serve(ctx context.Context, configFiles []string) { - e := echo.New() + e.Use(otelecho.Middleware("config-db", otelecho.WithSkipper(tracingURLSkipper))) + // PostgREST needs to know how it is exposed to create the correct links db.HTTPEndpoint = publicEndpoint + "/db" if logger.IsTraceEnabled() { - e.Use(middleware.Logger()) + echoLogConfig := middleware.DefaultLoggerConfig + echoLogConfig.Skipper = tracingURLSkipper + e.Use(middleware.LoggerWithConfig(echoLogConfig)) } + if !disablePostgrest { go db.StartPostgrest() forward(e, "/db", db.PostgRESTEndpoint()) @@ -135,3 +141,14 @@ func forward(e *echo.Echo, prefix string, target string) { func init() { ServerFlags(Serve.Flags()) } + +// tracingURLSkipper ignores metrics route on some middleware +func tracingURLSkipper(c echo.Context) bool { + pathsToSkip := []string{"/health", "/metrics"} + for _, p := range pathsToSkip { + if strings.HasPrefix(c.Path(), p) { + return true + } + } + return false +} diff --git a/go.mod b/go.mod index 220517628..83cb7af6e 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,12 @@ require ( github.com/stretchr/testify v1.9.0 github.com/uber/athenadriver v1.1.14 github.com/xo/dburl v0.13.1 + go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0 + go.opentelemetry.io/otel v1.26.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 + go.opentelemetry.io/otel/sdk v1.22.0 + go.opentelemetry.io/otel/trace v1.26.0 gopkg.in/flanksource/yaml.v3 v3.2.3 gopkg.in/yaml.v3 v3.0.1 gorm.io/gorm v1.25.5 @@ -155,13 +161,8 @@ require ( github.com/zclconf/go-cty v1.14.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect - go.opentelemetry.io/otel v1.22.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect - go.opentelemetry.io/otel/metric v1.22.0 // indirect - go.opentelemetry.io/otel/sdk v1.22.0 // indirect - go.opentelemetry.io/otel/trace v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/tools v0.20.0 // indirect @@ -280,7 +281,7 @@ require ( google.golang.org/api v0.162.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/grpc v1.63.2 // indirect + google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -296,6 +297,10 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) +replace go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.22.0 + +replace go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.22.0 + // replace github.com/flanksource/duty => ../duty // replace github.com/flanksource/ketall => ../ketall diff --git a/go.sum b/go.sum index d4b5c02df..194912b93 100644 --- a/go.sum +++ b/go.sum @@ -1443,10 +1443,14 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0 h1:477zSmIXZy+324mzDsXGm1DPhHKWTFrT6iUIAlpI9f4= +go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0/go.mod h1:oZWJX6UZ7QPGvMSM/2T5yXdKOtFaey2IV/IMHY+tryg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/contrib/propagators/b3 v1.26.0 h1:wgFbVA+bK2k+fGVfDOCOG4cfDAoppyr5sI2dVlh8MWM= +go.opentelemetry.io/contrib/propagators/b3 v1.26.0/go.mod h1:DDktFXxA+fyItAAM0Sbl5OBH7KOsCTjvbBdPKtoIf/k= go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= @@ -1457,8 +1461,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCy go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E= -go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= -go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod index 34e9890a3..0730dc20c 100644 --- a/hack/generate-schemas/go.mod +++ b/hack/generate-schemas/go.mod @@ -92,9 +92,9 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect - go.opentelemetry.io/otel v1.22.0 // indirect - go.opentelemetry.io/otel/metric v1.22.0 // indirect - go.opentelemetry.io/otel/trace v1.22.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum index cab602a53..e3bb20214 100644 --- a/hack/generate-schemas/go.sum +++ b/hack/generate-schemas/go.sum @@ -549,14 +549,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= -go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= -go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= -go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= -go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= -go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/telemetry/tracer.go b/telemetry/tracer.go new file mode 100644 index 000000000..2d3e845bc --- /dev/null +++ b/telemetry/tracer.go @@ -0,0 +1,68 @@ +package telemetry + +import ( + "context" + "os" + "strings" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "google.golang.org/grpc/credentials" + + "github.com/flanksource/commons/collections" + "github.com/flanksource/commons/logger" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func InitTracer(serviceName, collectorURL string, insecure bool) func(context.Context) error { + var secureOption otlptracegrpc.Option + if !insecure { + secureOption = otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, "")) + } else { + secureOption = otlptracegrpc.WithInsecure() + } + + exporter, err := otlptrace.New( + context.Background(), + otlptracegrpc.NewClient( + secureOption, + otlptracegrpc.WithEndpoint(collectorURL), + ), + ) + + if err != nil { + logger.Errorf("failed to create opentelemetry exporter: %v", err) + return func(_ context.Context) error { return nil } + } + + attributes := []attribute.KeyValue{attribute.String("service.name", serviceName)} + if val, ok := os.LookupEnv("OTEL_LABELS"); ok { + kv := collections.KeyValueSliceToMap(strings.Split(val, ",")) + for k, v := range kv { + attributes = append(attributes, attribute.String(k, v)) + } + } + + resources, err := resource.New(context.Background(), resource.WithAttributes(attributes...)) + if err != nil { + logger.Errorf("could not set opentelemetry resources: %v", err) + return func(_ context.Context) error { return nil } + } + + otel.SetTracerProvider( + sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(resources), + ), + ) + + // Register the TraceContext propagator globally. + otel.SetTextMapPropagator(propagation.TraceContext{}) + + return exporter.Shutdown +}