feat: register db.sql metrics (#2305)

Fixes #2285

Register the standard otel database metrics using:
https://github.com/XSAM/otelsql
Found call sites by grepping for `sql.Open` and skipping all test code.

Sample:
```
ScopeMetrics #2
ScopeMetrics SchemaURL: 
InstrumentationScope github.com/XSAM/otelsql 0.32.0

Metric #0
Descriptor:
     -> Name: db.sql.latency
     -> Description: The latency of calls in milliseconds
     -> Unit: ms
     -> DataType: Histogram
     -> AggregationTemporality: Cumulative

HistogramDataPoints #0
Data point attributes:
     -> method: Str(sql.conn.begin_tx)
     -> status: Str(ok)
StartTimestamp: 2024-08-08 23:18:01.011762 +0000 UTC
Timestamp: 2024-08-08 23:18:11.012177 +0000 UTC
Count: 23
Sum: 17.082795
Min: 0.137584
Max: 3.233917
```

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

backend/controller/sql/databasetesting/devel.go

@@ -8,7 +8,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/XSAM/otelsql"
 	_ "github.com/jackc/pgx/v5/stdlib" // pgx driver
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 
 	"github.com/TBD54566975/ftl/backend/controller/sql"
 	"github.com/TBD54566975/ftl/internal/log"
@@ -29,9 +31,13 @@ func CreateForDevel(ctx context.Context, dsn string, recreate bool) (*stdsql.DB,
 
 	var conn *stdsql.DB
 	for range 10 {
-		conn, err = stdsql.Open("pgx", noDBDSN.String())
+		conn, err = otelsql.Open("pgx", noDBDSN.String())
 		if err == nil {
 			defer conn.Close()
+			err = otelsql.RegisterDBStatsMetrics(conn, otelsql.WithAttributes(semconv.DBSystemPostgreSQL))
+			if err != nil {
+				panic(err)
+			}
 			break
 		}
 		logger.Debugf("Waiting for database to be ready: %v", err)
@@ -72,9 +78,13 @@ func CreateForDevel(ctx context.Context, dsn string, recreate bool) (*stdsql.DB,
 		return nil, err
 	}
 
-	realConn, err := stdsql.Open("pgx", dsn)
+	realConn, err := otelsql.Open("pgx", dsn)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to open database: %w", err)
+	}
+	err = otelsql.RegisterDBStatsMetrics(realConn, otelsql.WithAttributes(semconv.DBSystemPostgreSQL))
+	if err != nil {
+		return nil, fmt.Errorf("failed to register db metrics: %w", err)
 	}
 	// Reset transient state in the database to a clean state for development purposes.
 	// This includes things like resetting the state of async calls, leases,

cmd/ftl-controller/main.go

@@ -2,15 +2,16 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"os"
 	"strconv"
 	"time"
 
+	"github.com/XSAM/otelsql"
 	"github.com/alecthomas/kong"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 
 	"github.com/TBD54566975/ftl"
 	"github.com/TBD54566975/ftl/backend/controller"
@@ -51,7 +52,9 @@ func main() {
 	kctx.FatalIfErrorf(err, "failed to initialize observability")
 
 	// The FTL controller currently only supports DB as a configuration provider/resolver.
-	conn, err := sql.Open("pgx", cli.ControllerConfig.DSN)
+	conn, err := otelsql.Open("pgx", cli.ControllerConfig.DSN)
+	kctx.FatalIfErrorf(err)
+	err = otelsql.RegisterDBStatsMetrics(conn, otelsql.WithAttributes(semconv.DBSystemPostgreSQL))
 	kctx.FatalIfErrorf(err)
 	dal, err := dal.New(ctx, conn, encryptors)
 	kctx.FatalIfErrorf(err)

cmd/ftl/cmd_box_run.go

@@ -2,13 +2,14 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"net/url"
 	"time"
 
+	"github.com/XSAM/otelsql"
 	_ "github.com/jackc/pgx/v5/stdlib" // pgx driver
 	"github.com/jpillora/backoff"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/TBD54566975/ftl/backend/controller"
@@ -58,10 +59,14 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er
 	}
 
 	// Bring up the DB connection and DAL.
-	conn, err := sql.Open("pgx", config.DSN)
+	conn, err := otelsql.Open("pgx", config.DSN)
 	if err != nil {
 		return fmt.Errorf("failed to bring up DB connection: %w", err)
 	}
+	err = otelsql.RegisterDBStatsMetrics(conn, otelsql.WithAttributes(semconv.DBSystemPostgreSQL))
+	if err != nil {
+		return fmt.Errorf("failed to register DB metrics: %w", err)
+	}
 	encryptors, err := config.EncryptionKeys.Encryptors(false)
 	if err != nil {
 		return fmt.Errorf("failed to create encryptors: %w", err)

cmd/ftl/cmd_serve.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"fmt"
 	"net"
@@ -15,8 +14,10 @@ import (
 	"time"
 
 	"connectrpc.com/connect"
+	"github.com/XSAM/otelsql"
 	"github.com/alecthomas/types/optional"
 	_ "github.com/jackc/pgx/v5/stdlib" // pgx driver
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/TBD54566975/ftl"
@@ -148,10 +149,14 @@ func (s *serveCmd) run(ctx context.Context, projConfig projectconfig.Config, ini
 		controllerCtx = cf.ContextWithSecrets(controllerCtx, sm)
 
 		// Bring up the DB connection and DAL.
-		conn, err := sql.Open("pgx", config.DSN)
+		conn, err := otelsql.Open("pgx", config.DSN)
 		if err != nil {
 			return fmt.Errorf("failed to bring up DB connection: %w", err)
 		}
+		err = otelsql.RegisterDBStatsMetrics(conn, otelsql.WithAttributes(semconv.DBSystemPostgreSQL))
+		if err != nil {
+			return fmt.Errorf("failed to register DB metrics: %w", err)
+		}
 		encryptors, err := config.EncryptionKeys.Encryptors(false)
 		if err != nil {
 			return fmt.Errorf("failed to create encryptors: %w", err)

examples/go/echo/go.mod

@@ -10,6 +10,7 @@ require (
 	connectrpc.com/connect v1.16.2 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.1 // indirect
+	github.com/XSAM/otelsql v0.32.0 // indirect
 	github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect

go-runtime/ftl/reflection/testdata/go/runtimereflection/go.mod

@@ -11,6 +11,7 @@ require (
 	connectrpc.com/connect v1.16.2 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.1 // indirect
+	github.com/XSAM/otelsql v0.32.0 // indirect
 	github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/BurntSushi/toml v1.4.0
 	github.com/TBD54566975/golang-tools v0.2.1
 	github.com/TBD54566975/scaffolder v1.0.0
+	github.com/XSAM/otelsql v0.32.0
 	github.com/alecthomas/assert/v2 v2.10.0
 	github.com/alecthomas/atomic v0.1.0-alpha2
 	github.com/alecthomas/chroma/v2 v2.14.0
@@ -68,6 +69,7 @@ require (
 	golang.org/x/sync v0.8.0
 	golang.org/x/term v0.23.0
 	google.golang.org/protobuf v1.34.2
+	gotest.tools/v3 v3.5.1
 	modernc.org/sqlite v1.32.0
 )
 
@@ -87,6 +89,7 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
@@ -110,7 +113,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	golang.org/x/tools v0.23.0 // indirect
-	gotest.tools/v3 v3.5.1 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 )
 

internal/modulecontext/database.go

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/XSAM/otelsql"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 )
 
@@ -19,9 +22,13 @@ type Database struct {
 
 // NewDatabase creates a Database that can be added to ModuleContext
 func NewDatabase(dbType DBType, dsn string) (Database, error) {
-	db, err := sql.Open("pgx", dsn)
+	db, err := otelsql.Open("pgx", dsn)
 	if err != nil {
-		return Database{}, err
+		return Database{}, fmt.Errorf("failed to open database: %w", err)
+	}
+	err = otelsql.RegisterDBStatsMetrics(db, otelsql.WithAttributes(semconv.DBSystemPostgreSQL))
+	if err != nil {
+		return Database{}, fmt.Errorf("failed to register db metrics: %w", err)
 	}
 	return Database{
 		DSN:    dsn,

internal/observability/client_test.go

@@ -10,5 +10,5 @@ import (
 
 func TestSchemaMismatch(t *testing.T) {
 	dflt := resource.Default()
-	assert.Equal(t, dflt.SchemaURL(), schemaURL, `change import in client.go to: semconv "go.opentelemetry.io/otel/semconv/v%s"`, path.Base(dflt.SchemaURL()))
+	assert.Equal(t, dflt.SchemaURL(), schemaURL, `in every file that imports go.opentelemetry.io/otel/semconv, change the import to: semconv "go.opentelemetry.io/otel/semconv/v%s"`, path.Base(dflt.SchemaURL()))
 }

internal/reflect/reflect.go

@@ -13,6 +13,7 @@
 package reflect
 
 import (
+	"container/list"
 	"fmt"
 	"reflect"
 	"strings"
@@ -131,6 +132,11 @@ func copyAny(src any, ptrs map[uintptr]any, copyConf *copyConfig) (dst any) {
 		return src
 	}
 
+	// Special case list.List to handle its internal structure
+	if reflect.TypeOf(src) == reflect.TypeFor[*list.List]() {
+		return copyList(src.(*list.List), ptrs, copyConf)
+	}
+
 	// Look up the corresponding copy function.
 	switch v.Kind() {
 	case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
@@ -139,7 +145,11 @@ func copyAny(src any, ptrs map[uintptr]any, copyConf *copyConfig) (dst any) {
 		reflect.Complex64, reflect.Complex128, reflect.Func:
 		dst = copyPremitive(src, ptrs, copyConf)
 	case reflect.String:
-		dst = strings.Clone(src.(string))
+		if v.Type() == reflect.TypeFor[string]() {
+			dst = strings.Clone(src.(string))
+		} else {
+			dst = copyStringAlias(src, ptrs, copyConf)
+		}
 	case reflect.Slice:
 		dst = copySlice(src, ptrs, copyConf)
 	case reflect.Array:
@@ -160,6 +170,18 @@ func copyAny(src any, ptrs map[uintptr]any, copyConf *copyConfig) (dst any) {
 	return
 }
 
+func copyList(src *list.List, ptrs map[uintptr]any, copyConf *copyConfig) *list.List {
+	if src == nil {
+		return nil
+	}
+	dst := list.New()
+	for e := src.Front(); e != nil; e = e.Next() {
+		copiedValue := copyAny(e.Value, ptrs, copyConf)
+		dst.PushBack(copiedValue)
+	}
+	return dst
+}
+
 func copyPremitive(src any, ptr map[uintptr]any, copyConf *copyConfig) (dst any) {
 	kind := reflect.ValueOf(src).Kind()
 	switch kind {
@@ -170,6 +192,13 @@ func copyPremitive(src any, ptr map[uintptr]any, copyConf *copyConfig) (dst any)
 	return
 }
 
+func copyStringAlias(src any, ptr map[uintptr]any, copyConf *copyConfig) any {
+	v := reflect.ValueOf(src)
+	dc := reflect.New(v.Type()).Elem()
+	dc.Set(v)
+	return dc.Interface()
+}
+
 func copySlice(x any, ptrs map[uintptr]any, copyConf *copyConfig) any {
 	v := reflect.ValueOf(x)
 	kind := v.Kind()

internal/reflect/reflect_test.go

@@ -1,9 +1,30 @@
 package reflect
 
 import (
+	"container/list"
 	"testing"
+
+	"gotest.tools/v3/assert"
 )
 
+type mystring string
+type structWithMystring struct {
+	str mystring
+}
+
+func TestAliasedString(t *testing.T) {
+	output := DeepCopy(structWithMystring{"asdf"})
+	assert.Equal(t, output, structWithMystring{"asdf"})
+}
+
+func TestListElements(t *testing.T) {
+	l := list.New()
+	l.PushBack("one")
+	output := DeepCopy(l)
+	assert.Equal(t, output.Front().Value, l.Front().Value)
+	assert.Equal(t, output.Len(), l.Len())
+}
+
 type structOfPointers struct {
 	intPtr    *int
 	floatPtr  *float64