Skip to content

Commit

Permalink
feat: telemetry (#540)
Browse files Browse the repository at this point in the history
* feat: telemetry

* tenant id in chart

* Update chart/values.yaml

---------

Co-authored-by: Moshe Immerman <[email protected]>
  • Loading branch information
adityathebe and moshloop authored May 13, 2024
1 parent ac34964 commit 6fb2b6b
Show file tree
Hide file tree
Showing 9 changed files with 150 additions and 24 deletions.
10 changes: 10 additions & 0 deletions chart/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ spec:
{{- if .Values.db.runMigrations}}
- --db-migrations
{{- end}}
{{- if (tpl .Values.otel.collector .) }}
- --otel-collector-url={{ tpl .Values.otel.collector . | default "" }}
{{- end }}
{{- if (tpl .Values.otel.serviceName .) }}
- --otel-service-name={{ tpl .Values.otel.serviceName . | default "config-db" }}
{{- end }}
{{- if .Values.upstream.enabled}}
envFrom:
- secretRef:
Expand All @@ -90,6 +96,10 @@ spec:
- name: UPSTREAM_PAGE_SIZE
value: '{{ .Values.upstream.pageSize }}'
{{- end}}
{{- if .Values.otel.labels}}
- name: OTEL_LABELS
value: '{{ .Values.otel.labels }}'
{{- end}}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
Expand Down
15 changes: 11 additions & 4 deletions chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ global:
db:
connectionPooler:
enabled: false
secretKeyRef:
secretKeyRef:
name: mission-control-connection-pooler
key: DB_URL

Expand All @@ -30,6 +30,13 @@ configAnalysisRetentionDays: 60
scrapeRuleConfigMaps:
- config-db-rules

otel:
# OpenTelemetry gRPC collector endpoint in host:port format
collector: ''
serviceName: config-db
# labels in "a=b,c=d" format
labels:

db:
runMigrations: true
embedded:
Expand Down Expand Up @@ -89,15 +96,15 @@ serviceAccount:
rbac:
# Whether to create cluster-wide or namespaced roles
clusterRole: true

# for secret management with valueFrom
tokenRequest: true
secrets: true
configmaps: true

# for use with kubernetes resource lookups
readAll: true

# for kubernetesFile lookups
exec: true

Expand Down
15 changes: 15 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/flanksource/config-db/jobs"
"github.com/flanksource/config-db/scrapers"
"github.com/flanksource/config-db/scrapers/kubernetes"
"github.com/flanksource/config-db/telemetry"
"github.com/flanksource/config-db/utils/kube"
"github.com/flanksource/duty"
"github.com/spf13/cobra"
Expand All @@ -30,6 +31,12 @@ var (
date = "unknown"
)

var (
// Telemetry flag vars
otelcollectorURL string
otelServiceName string
)

func readFromEnv(v string) string {
val := os.Getenv(v)
if val != "" {
Expand Down Expand Up @@ -59,6 +66,11 @@ var Root = &cobra.Command{
}
db.Schema = readFromEnv(db.Schema)
db.PGRSTLogLevel = readFromEnv(db.PGRSTLogLevel)

if otelcollectorURL != "" {
logger.Infof("Sending traces to %s", otelcollectorURL)
_ = telemetry.InitTracer(otelServiceName, otelcollectorURL, true) // TODO: Setup runner
}
},
}

Expand All @@ -78,6 +90,9 @@ func ServerFlags(flags *pflag.FlagSet) {
flags.StringVar(&publicEndpoint, "public-endpoint", "http://localhost:8080", "Public endpoint that this instance is exposed under")
flags.IntVar(&kubernetes.BufferSize, "watch-event-buffer", kubernetes.BufferSize, "Buffer size for kubernetes events")

flags.StringVar(&otelcollectorURL, "otel-collector-url", "", "OpenTelemetry gRPC Collector URL in host:port format")
flags.StringVar(&otelServiceName, "otel-service-name", "config-db", "OpenTelemetry service name for the resource")

// Flags for push/pull
var upstreamPageSizeDefault = 500
if val, exists := os.LookupEnv("UPSTREAM_PAGE_SIZE"); exists {
Expand Down
21 changes: 19 additions & 2 deletions cmd/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"net/url"
"strings"

"github.com/flanksource/commons/logger"
"github.com/flanksource/config-db/api"
Expand All @@ -18,6 +19,7 @@ import (
"github.com/labstack/echo/v4/middleware"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/spf13/cobra"
"go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho"

"github.com/flanksource/config-db/scrapers"
)
Expand All @@ -44,14 +46,18 @@ var Serve = &cobra.Command{
}

func serve(ctx context.Context, configFiles []string) {

e := echo.New()
e.Use(otelecho.Middleware("config-db", otelecho.WithSkipper(tracingURLSkipper)))

// PostgREST needs to know how it is exposed to create the correct links
db.HTTPEndpoint = publicEndpoint + "/db"

if logger.IsTraceEnabled() {
e.Use(middleware.Logger())
echoLogConfig := middleware.DefaultLoggerConfig
echoLogConfig.Skipper = tracingURLSkipper
e.Use(middleware.LoggerWithConfig(echoLogConfig))
}

if !disablePostgrest {
go db.StartPostgrest()
forward(e, "/db", db.PostgRESTEndpoint())
Expand Down Expand Up @@ -135,3 +141,14 @@ func forward(e *echo.Echo, prefix string, target string) {
func init() {
ServerFlags(Serve.Flags())
}

// tracingURLSkipper ignores metrics route on some middleware
func tracingURLSkipper(c echo.Context) bool {
pathsToSkip := []string{"/health", "/metrics"}
for _, p := range pathsToSkip {
if strings.HasPrefix(c.Path(), p) {
return true
}
}
return false
}
19 changes: 12 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,12 @@ require (
github.com/stretchr/testify v1.9.0
github.com/uber/athenadriver v1.1.14
github.com/xo/dburl v0.13.1
go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0
go.opentelemetry.io/otel v1.26.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0
go.opentelemetry.io/otel/sdk v1.22.0
go.opentelemetry.io/otel/trace v1.26.0
gopkg.in/flanksource/yaml.v3 v3.2.3
gopkg.in/yaml.v3 v3.0.1
gorm.io/gorm v1.25.5
Expand Down Expand Up @@ -156,13 +162,8 @@ require (
github.com/zclconf/go-cty v1.14.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
go.opentelemetry.io/otel v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.22.0 // indirect
go.opentelemetry.io/otel/sdk v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.26.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/tools v0.20.0 // indirect
Expand Down Expand Up @@ -281,7 +282,7 @@ require (
google.golang.org/api v0.162.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/grpc v1.63.2 // indirect
google.golang.org/grpc v1.63.2
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand All @@ -297,6 +298,10 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
)

replace go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.22.0

replace go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.22.0

// replace github.com/flanksource/duty => ../duty

// replace github.com/flanksource/ketall => ../ketall
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1445,10 +1445,14 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0 h1:477zSmIXZy+324mzDsXGm1DPhHKWTFrT6iUIAlpI9f4=
go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.51.0/go.mod h1:oZWJX6UZ7QPGvMSM/2T5yXdKOtFaey2IV/IMHY+tryg=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
go.opentelemetry.io/contrib/propagators/b3 v1.26.0 h1:wgFbVA+bK2k+fGVfDOCOG4cfDAoppyr5sI2dVlh8MWM=
go.opentelemetry.io/contrib/propagators/b3 v1.26.0/go.mod h1:DDktFXxA+fyItAAM0Sbl5OBH7KOsCTjvbBdPKtoIf/k=
go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
Expand All @@ -1459,8 +1463,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCy
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E=
go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
Expand Down
6 changes: 3 additions & 3 deletions hack/generate-schemas/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,9 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
go.opentelemetry.io/otel v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.22.0 // indirect
go.opentelemetry.io/otel/trace v1.22.0 // indirect
go.opentelemetry.io/otel v1.26.0 // indirect
go.opentelemetry.io/otel/metric v1.26.0 // indirect
go.opentelemetry.io/otel/trace v1.26.0 // indirect
go.uber.org/goleak v1.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
Expand Down
12 changes: 6 additions & 6 deletions hack/generate-schemas/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -549,14 +549,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
Expand Down
68 changes: 68 additions & 0 deletions telemetry/tracer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package telemetry

import (
"context"
"os"
"strings"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"google.golang.org/grpc/credentials"

"github.com/flanksource/commons/collections"
"github.com/flanksource/commons/logger"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

func InitTracer(serviceName, collectorURL string, insecure bool) func(context.Context) error {
var secureOption otlptracegrpc.Option
if !insecure {
secureOption = otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, ""))
} else {
secureOption = otlptracegrpc.WithInsecure()
}

exporter, err := otlptrace.New(
context.Background(),
otlptracegrpc.NewClient(
secureOption,
otlptracegrpc.WithEndpoint(collectorURL),
),
)

if err != nil {
logger.Errorf("failed to create opentelemetry exporter: %v", err)
return func(_ context.Context) error { return nil }
}

attributes := []attribute.KeyValue{attribute.String("service.name", serviceName)}
if val, ok := os.LookupEnv("OTEL_LABELS"); ok {
kv := collections.KeyValueSliceToMap(strings.Split(val, ","))
for k, v := range kv {
attributes = append(attributes, attribute.String(k, v))
}
}

resources, err := resource.New(context.Background(), resource.WithAttributes(attributes...))
if err != nil {
logger.Errorf("could not set opentelemetry resources: %v", err)
return func(_ context.Context) error { return nil }
}

otel.SetTracerProvider(
sdktrace.NewTracerProvider(
sdktrace.WithSampler(sdktrace.AlwaysSample()),
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(resources),
),
)

// Register the TraceContext propagator globally.
otel.SetTextMapPropagator(propagation.TraceContext{})

return exporter.Shutdown
}

0 comments on commit 6fb2b6b

Please sign in to comment.