From eab7297f6007df7a3080bd94bbb55db8006fd34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Tue, 28 May 2024 14:58:23 +0200 Subject: [PATCH 01/14] go.mod: upgrade github.com/mattn/go-sqlite3 up to v1.14.18 (#2693) (#2714) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a9685e5d03..a2139e2844 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/labstack/echo v3.3.10+incompatible github.com/labstack/echo/v4 v4.11.1 github.com/lib/pq v1.10.2 - github.com/mattn/go-sqlite3 v1.14.16 + github.com/mattn/go-sqlite3 v1.14.18 github.com/microsoft/go-mssqldb v0.21.0 github.com/miekg/dns v1.1.55 github.com/mitchellh/mapstructure v1.5.0 diff --git a/go.sum b/go.sum index 8d59e14951..1ed1051b37 100644 --- a/go.sum +++ b/go.sum @@ -1659,8 +1659,8 @@ github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= +github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= From 464e0780336c823786a07438c893d810e5f835d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Tue, 28 May 2024 15:07:14 +0200 Subject: [PATCH 02/14] go.mod: upgrade github.com/go-jose/go-jose/v3 to v3.0.3 (#2700) --- go.mod | 2 +- go.sum | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a2139e2844..1c6daf57f5 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,7 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect diff --git a/go.sum b/go.sum index 1ed1051b37..058eeea647 100644 --- a/go.sum +++ b/go.sum @@ -1150,8 +1150,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= -github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= +github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -2174,7 +2174,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2196,6 +2195,7 @@ golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2342,6 +2342,7 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2536,6 +2537,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2550,6 +2553,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2567,6 +2572,7 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From b913c87d19978737a083f5590e721516ea795a34 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Tue, 28 May 2024 10:30:00 -0400 Subject: [PATCH 03/14] w3c: ensure _dd.parent_id is set using x-datadog-parent-id (#2696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dario Castañé --- ddtrace/tracer/textmap.go | 72 ++++++++++++++++++++++------------ ddtrace/tracer/textmap_test.go | 38 ++++++++++++++++-- ddtrace/tracer/tracer.go | 7 ---- 3 files changed, 82 insertions(+), 35 deletions(-) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 30b2b2f7ff..5f9dc5a8fa 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -274,11 +274,23 @@ func (p *chainedPropagator) Extract(carrier interface{}) (ddtrace.SpanContext, e for _, v := range p.extractors { if ctx != nil { // A local trace context has already been extracted. - p, isW3C := v.(*propagatorW3c) + pw3c, isW3C := v.(*propagatorW3c) if !isW3C { continue // Ignore other propagators. } - p.propagateTracestate(ctx.(*spanContext), carrier) + w3cCtx, err := pw3c.Extract(carrier) + if err == nil && w3cCtx.(*spanContext).TraceID128() == ctx.(*spanContext).TraceID128() { + pw3c.propagateTracestate(ctx.(*spanContext), w3cCtx.(*spanContext)) + if w3cCtx.SpanID() != ctx.SpanID() { + var ddCtx *spanContext + if ddp := getDatadogPropagator(p); ddp != nil { + if ddSpanCtx, err := ddp.Extract(carrier); err == nil { + ddCtx, _ = ddSpanCtx.(*spanContext) + } + } + overrideDatadogParentID(ctx.(*spanContext), w3cCtx.(*spanContext), ddCtx) + } + } break } var err error @@ -306,15 +318,8 @@ func (p *chainedPropagator) Extract(carrier interface{}) (ddtrace.SpanContext, e // provided by the given *spanContext. If it matches, then the tracestate // will be re-composed based on the composition of the given *spanContext, // but will include the non-DD vendors in the W3C trace context's tracestate. -func (p *propagatorW3c) propagateTracestate(ctx *spanContext, carrier interface{}) { - w3cCtx, _ := p.Extract(carrier) - if w3cCtx == nil { - return // It's not valid, so ignore it. - } - if ctx.TraceID() != w3cCtx.TraceID() { - return // The trace-ids must match. - } - if w3cCtx.(*spanContext).trace == nil { +func (p *propagatorW3c) propagateTracestate(ctx *spanContext, w3cCtx *spanContext) { + if w3cCtx.trace == nil { return // this shouldn't happen, since it should have a propagating tag already } if ctx.trace == nil { @@ -324,11 +329,10 @@ func (p *propagatorW3c) propagateTracestate(ctx *spanContext, carrier interface{ // it to the span context that will be returned. // Note: Other trace context fields like sampling priority, propagated tags, // and origin will remain unchanged. - ts := w3cCtx.(*spanContext).trace.propagatingTag(tracestateHeader) + ts := w3cCtx.trace.propagatingTag(tracestateHeader) priority, _ := ctx.SamplingPriority() setPropagatingTag(ctx, tracestateHeader, composeTracestate(ctx, priority, ts)) - ctx.reparentID = w3cCtx.(*spanContext).reparentID - ctx.isRemote = (w3cCtx.(*spanContext).isRemote) + ctx.isRemote = (w3cCtx.isRemote) } // propagator implements Propagator and injects/extracts span contexts @@ -489,6 +493,28 @@ func validateTID(tid string) error { return nil } +// getDatadogPropagator returns the Datadog Propagator +func getDatadogPropagator(cp *chainedPropagator) *propagator { + for _, e := range cp.extractors { + p, isDatadog := (e).(*propagator) + if isDatadog { + return p + } + } + return nil +} + +// overrideDatadogParentID overrides the span ID of a context with the ID extracted from tracecontext headers +// if the reparenting ID is not set on the context, the span ID from datadog headers is used. +func overrideDatadogParentID(ctx, w3cCtx, ddCtx *spanContext) { + ctx.spanID = w3cCtx.spanID + if w3cCtx.reparentID != "" && w3cCtx.reparentID != "0000000000000000" { + ctx.reparentID = w3cCtx.reparentID + } else if ddCtx != nil { + ctx.reparentID = fmt.Sprintf("%016x", ddCtx.SpanID()) + } +} + // unmarshalPropagatingTags unmarshals tags from v into ctx func unmarshalPropagatingTags(ctx *spanContext, v string) { if ctx.trace == nil { @@ -844,15 +870,12 @@ func composeTracestate(ctx *spanContext, priority int, oldState string) string { // if the context is not remote, set p as context.spanId // this ID can be used by downstream tracers to set a _dd.parent_id tag // to allow the backend to reparent orphaned spans if necessary - - if ctx.isRemote && ctx.reparentID != "" { + if !ctx.isRemote { + b.WriteString(fmt.Sprintf(";p:%016x", ctx.SpanID())) + } else if ctx.reparentID != "" && ctx.reparentID != "0000000000000000" { b.WriteString(fmt.Sprintf(";p:%s", ctx.reparentID)) } - if !ctx.isRemote || ctx.isRemote && ctx.trace.root != nil { - b.WriteString(fmt.Sprintf(";p:%016x", ctx.spanID)) - } - ctx.trace.iteratePropagatingTags(func(k, v string) bool { if !strings.HasPrefix(k, "_dd.p.") { return true @@ -1069,10 +1092,7 @@ func parseTracestate(ctx *spanContext, header string) { dropDM = true } } else if key == "p" { - if val != "" { - ctx.reparentID = val - } - + ctx.reparentID = val } else if strings.HasPrefix(key, "t.dm") { if ctx.trace.hasPropagatingTag(keyDecisionMaker) || dropDM { continue @@ -1084,6 +1104,10 @@ func parseTracestate(ctx *spanContext, header string) { setPropagatingTag(ctx, "_dd.p."+keySuffix, val) } } + // if dd list-member is present and last parent is not set, set it to zeros + if ctx.reparentID == "" { + ctx.reparentID = "0000000000000000" + } } } diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index f29f9eea81..6306cd12e6 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -111,6 +111,7 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { name, propagationStyle, traceparent string onlyExtractFirst bool // value of DD_TRACE_PROPAGATION_EXTRACT_FIRST wantTracestatePropagation bool + conflictingParentID bool }{ { /* @@ -131,6 +132,7 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { propagationStyle: "datadog,b3,tracecontext", traceparent: "00-00000000000000000000000000000004-2222222222222222-01", wantTracestatePropagation: true, + conflictingParentID: true, }, { /* @@ -141,6 +143,7 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { propagationStyle: "datadog,tracecontext", traceparent: "00-00000000000000000000000000000004-2222222222222222-01", wantTracestatePropagation: true, + conflictingParentID: true, }, { /* @@ -197,7 +200,7 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { originHeader: "synthetics", b3TraceIDHeader: "0021dc1807524785", traceparentHeader: tc.traceparent, - tracestateHeader: "dd=s:2;o:rum;p:0000000000000005;t.tid:1230000000000000~~,othervendor=t61rcWkgMzE", + tracestateHeader: "dd=s:2;o:rum;p:0000000000000001;t.tid:1230000000000000~~,othervendor=t61rcWkgMzE", }) ctx, err := tracer.Extract(headers) @@ -205,10 +208,16 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { sctx, ok := ctx.(*spanContext) assert.True(ok) assert.Equal("00000000000000000000000000000004", sctx.traceID.HexEncoded()) - assert.Equal(uint64(1), sctx.spanID) // should use x-datadog-parent-id, not the id in the tracestate + if tc.conflictingParentID == true { + // tracecontext span id should be used + assert.Equal(uint64(0x2222222222222222), sctx.spanID) + } else { + // should use x-datadog-parent-id, not the id in the tracestate + assert.Equal(uint64(1), sctx.spanID) + } assert.Equal("synthetics", sctx.origin) // should use x-datadog-origin, not the origin in the tracestate if tc.wantTracestatePropagation { - assert.Equal("0000000000000005", sctx.reparentID) + assert.Equal("0000000000000001", sctx.reparentID) assert.Equal("dd=s:0;o:synthetics;p:0000000000000001,othervendor=t61rcWkgMzE", sctx.trace.propagatingTag(tracestateHeader)) } else if sctx.trace != nil { assert.False(sctx.trace.hasPropagatingTag(tracestateHeader)) @@ -1348,7 +1357,7 @@ func TestEnvVars(t *testing.T) { { inHeaders: TextMapCarrier{ traceparentHeader: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", - tracestateHeader: "foo=1,dd=s:-1", + tracestateHeader: "foo=1,dd=s:-1;p:00f067aa0ba902b7", }, outHeaders: TextMapCarrier{ traceparentHeader: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", @@ -1908,6 +1917,27 @@ func checkSameElements(assert *assert.Assertions, want, got string) { assert.ElementsMatch(gotInnerList, wantInnerList) } +func TestTraceContextPrecedence(t *testing.T) { + t.Setenv(headerPropagationStyleExtract, "datadog,b3,tracecontext") + tracer := newTracer() + defer tracer.Stop() + ctx, err := tracer.Extract(TextMapCarrier{ + traceparentHeader: "00-00000000000000000000000000000001-0000000000000001-01", + DefaultTraceIDHeader: "1", + DefaultParentIDHeader: "22", + DefaultPriorityHeader: "2", + b3SingleHeader: "1-333", + }) + assert.NoError(t, err) + + sctx, _ := ctx.(*spanContext) + assert := assert.New(t) + assert.Equal(traceIDFrom64Bits(1), sctx.traceID) + assert.Equal(uint64(0x1), sctx.spanID) + p, _ := sctx.SamplingPriority() + assert.Equal(2, p) +} + func TestW3CExtractsBaggage(t *testing.T) { tracer := newTracer() defer tracer.Stop() diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 363bb24f31..3ecb4785c1 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -534,13 +534,6 @@ func (t *tracer) StartSpan(operationName string, options ...ddtrace.StartSpanOpt } } - // a remote context that was lacking the p: tracestate key - // the zero value indicates to the backend this could be the reparented - // as the root span of a trace - if context.isRemote && context.reparentID == "" { - span.setMeta(keyReparentID, "0000000000000000") - } - if context.reparentID != "" { span.setMeta(keyReparentID, context.reparentID) } From 9126b4fa24ea095c2a4bfca26ff9da8ea1000600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 30 May 2024 09:16:43 +0200 Subject: [PATCH 04/14] ddtrace/tracer: increment droppedP0Traces iff it's not going to be sent (#2713) --- ddtrace/tracer/tracer.go | 6 +++--- ddtrace/tracer/tracer_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 3ecb4785c1..c40a8293bf 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -427,11 +427,11 @@ func (t *tracer) sampleChunk(c *chunk) { atomic.AddUint32(&t.partialTraces, 1) } } - if len(kept) == 0 { - atomic.AddUint32(&t.droppedP0Traces, 1) - } atomic.AddUint32(&t.droppedP0Spans, uint32(len(c.spans)-len(kept))) if !c.willSend { + if len(kept) == 0 { + atomic.AddUint32(&t.droppedP0Traces, 1) + } c.spans = kept } } diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 5d95ea5295..cec6b84cdb 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -337,6 +337,11 @@ func TestSamplingDecision(t *testing.T) { t.Run("sampled", func(t *testing.T) { tracer, _, _, stop := startTestTracer(t) + defer func() { + // Must check these after tracer is stopped to avoid flakiness + assert.Equal(t, uint32(0), tracer.droppedP0Traces) + assert.Equal(t, uint32(2), tracer.droppedP0Spans) + }() defer stop() tracer.prioritySampling.defaultRate = 0 tracer.config.serviceName = "test_service" @@ -353,6 +358,11 @@ func TestSamplingDecision(t *testing.T) { // Even if DropP0s is enabled, spans should always be kept unless // client-side stats are also enabled. tracer, _, _, stop := startTestTracer(t) + defer func() { + // Must check these after tracer is stopped to avoid flakiness + assert.Equal(t, uint32(0), tracer.droppedP0Traces) + assert.Equal(t, uint32(2), tracer.droppedP0Spans) + }() defer stop() tracer.config.agent.DropP0s = true tracer.prioritySampling.defaultRate = 0 @@ -368,6 +378,11 @@ func TestSamplingDecision(t *testing.T) { t.Run("dropped_stats", func(t *testing.T) { tracer, _, _, stop := startTestTracer(t) + defer func() { + // Must check these after tracer is stopped to avoid flakiness + assert.Equal(t, uint32(1), tracer.droppedP0Traces) + assert.Equal(t, uint32(2), tracer.droppedP0Spans) + }() defer stop() tracer.config.featureFlags = make(map[string]struct{}) tracer.config.featureFlags["discovery"] = struct{}{} @@ -386,6 +401,11 @@ func TestSamplingDecision(t *testing.T) { t.Run("events_sampled", func(t *testing.T) { tracer, _, _, stop := startTestTracer(t) + defer func() { + // Must check these after tracer is stopped to avoid flakiness + assert.Equal(t, uint32(0), tracer.droppedP0Traces) + assert.Equal(t, uint32(2), tracer.droppedP0Spans) + }() defer stop() tracer.config.agent.DropP0s = true tracer.prioritySampling.defaultRate = 0 @@ -514,6 +534,11 @@ func TestSamplingDecision(t *testing.T) { // Rules are available. Trace sample rate equals 1. Span sample rate equals 1. // The trace should be kept. No single spans extracted. tracer, _, _, stop := startTestTracer(t) + defer func() { + // Must check these after tracer is stopped to avoid flakiness + assert.Equal(t, uint32(0), tracer.droppedP0Traces) + assert.Equal(t, uint32(0), tracer.droppedP0Spans) + }() defer stop() tracer.config.agent.DropP0s = true tracer.config.featureFlags = make(map[string]struct{}) @@ -573,6 +598,7 @@ func TestSamplingDecision(t *testing.T) { } assert.Equal(t, 50, singleSpans) assert.InDelta(t, 0.8, float64(keptSpans)/float64(len(spans)), 0.19) + assert.Equal(t, uint32(0), tracer.droppedP0Traces) }) t.Run("single_spans_without_max_per_second:rate_1.0", func(t *testing.T) { @@ -607,6 +633,7 @@ func TestSamplingDecision(t *testing.T) { } assert.Equal(t, 1000, keptSpans+singleSpans) assert.InDelta(t, 0.8, float64(keptSpans)/float64(1000), 0.15) + assert.Equal(t, uint32(0), tracer.droppedP0Traces) }) t.Run("single_spans_without_max_per_second:rate_0.5", func(t *testing.T) { @@ -643,6 +670,7 @@ func TestSamplingDecision(t *testing.T) { } assert.InDelta(t, 0.5, float64(singleSpans)/(float64(900-keptChildren)), 0.15) assert.InDelta(t, 0.8, float64(keptTotal)/1000, 0.15) + assert.Equal(t, uint32(0), tracer.droppedP0Traces) }) } From 4e981205ceca9e4d7e20556cca4e80d64ff81442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 30 May 2024 09:39:07 +0200 Subject: [PATCH 05/14] ddtrace/tracer: support default origin on dynamic config to support Active Tracing telemetry spec (#2623) --- ddtrace/tracer/dynamic_config.go | 9 ++- ddtrace/tracer/option.go | 20 +++-- ddtrace/tracer/option_test.go | 3 + ddtrace/tracer/remote_config_test.go | 77 ++++++++++++------- ddtrace/tracer/telemetry.go | 2 +- ddtrace/tracer/telemetry_test.go | 1 + ddtrace/tracer/tracer.go | 8 ++ internal/appsec/config/config.go | 2 +- internal/appsec/config/config_test.go | 5 +- internal/appsec/telemetry.go | 10 +-- internal/telemetry/client.go | 4 +- internal/telemetry/message.go | 45 ++++++++++- internal/telemetry/telemetry_test.go | 14 ++-- .../telemetry/telemetrytest/telemetrytest.go | 2 +- 14 files changed, 142 insertions(+), 60 deletions(-) diff --git a/ddtrace/tracer/dynamic_config.go b/ddtrace/tracer/dynamic_config.go index 919b4f7229..db48be5a73 100644 --- a/ddtrace/tracer/dynamic_config.go +++ b/ddtrace/tracer/dynamic_config.go @@ -19,7 +19,7 @@ type dynamicConfig[T any] struct { current T // holds the current configuration value startup T // holds the startup configuration value cfgName string // holds the name of the configuration, has to be compatible with telemetry.Configuration.Name - cfgOrigin string // holds the origin of the current configuration value (currently only supports remote_config, empty otherwise) + cfgOrigin telemetry.Origin // holds the origin of the current configuration value (currently only supports remote_config, empty otherwise) apply func(T) bool // executes any config-specific operations to propagate the update properly, returns whether the update was applied equal func(x, y T) bool // compares two configuration values, this is used to avoid unnecessary config and telemetry updates } @@ -42,7 +42,7 @@ func (dc *dynamicConfig[T]) get() T { } // update applies a new configuration value -func (dc *dynamicConfig[T]) update(val T, origin string) bool { +func (dc *dynamicConfig[T]) update(val T, origin telemetry.Origin) bool { dc.Lock() defer dc.Unlock() if dc.equal(dc.current, val) { @@ -61,7 +61,8 @@ func (dc *dynamicConfig[T]) reset() bool { return false } dc.current = dc.startup - dc.cfgOrigin = "" + // TODO: set the origin to the startup value's origin + dc.cfgOrigin = telemetry.OriginDefault return dc.apply(dc.startup) } @@ -69,7 +70,7 @@ func (dc *dynamicConfig[T]) reset() bool { // Returns whether the configuration value has been updated or not func (dc *dynamicConfig[T]) handleRC(val *T) bool { if val != nil { - return dc.update(*val, "remote_config") + return dc.update(*val, telemetry.OriginRemoteConfig) } return dc.reset() } diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index d73308f038..30352a9ba0 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -29,6 +29,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/namingschema" "gopkg.in/DataDog/dd-trace-go.v1/internal/normalizer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" "gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof" "gopkg.in/DataDog/dd-trace-go.v1/internal/version" @@ -336,7 +337,9 @@ func newConfig(opts ...StartOption) *config { } c.headerAsTags = newDynamicConfig("trace_header_tags", nil, setHeaderTags, equalSlice[string]) if v := os.Getenv("DD_TRACE_HEADER_TAGS"); v != "" { - WithHeaderTags(strings.Split(v, ","))(c) + c.headerAsTags.update(strings.Split(v, ","), telemetry.OriginEnvVar) + // Required to ensure that the startup header tags are set on reset. + c.headerAsTags.startup = c.headerAsTags.current } if v := os.Getenv("DD_TAGS"); v != "" { tags := internal.ParseTagString(v) @@ -344,6 +347,8 @@ func newConfig(opts ...StartOption) *config { for key, val := range tags { WithGlobalTag(key, val)(c) } + // TODO: should we track the origin of these tags individually? + c.globalTags.cfgOrigin = telemetry.OriginEnvVar } if _, ok := os.LookupEnv("AWS_LAMBDA_FUNCTION_NAME"); ok { // AWS_LAMBDA_FUNCTION_NAME being set indicates that we're running in an AWS Lambda environment. @@ -354,6 +359,9 @@ func newConfig(opts ...StartOption) *config { c.runtimeMetrics = internal.BoolEnv("DD_RUNTIME_METRICS_ENABLED", false) c.debug = internal.BoolEnv("DD_TRACE_DEBUG", false) c.enabled = newDynamicConfig("tracing_enabled", internal.BoolEnv("DD_TRACE_ENABLED", true), func(b bool) bool { return true }, equal[bool]) + if _, ok := os.LookupEnv("DD_TRACE_ENABLED"); ok { + c.enabled.cfgOrigin = telemetry.OriginEnvVar + } c.profilerEndpoints = internal.BoolEnv(traceprof.EndpointEnvVar, true) c.profilerHotspots = internal.BoolEnv(traceprof.CodeHotspotsEnvVar, true) c.enableHostnameDetection = internal.BoolEnv("DD_CLIENT_HOSTNAME_ENABLED", true) @@ -509,7 +517,8 @@ func newConfig(opts ...StartOption) *config { } // Re-initialize the globalTags config with the value constructed from the environment and start options // This allows persisting the initial value of globalTags for future resets and updates. - c.initGlobalTags(c.globalTags.get()) + globalTagsOrigin := c.globalTags.cfgOrigin + c.initGlobalTags(c.globalTags.get(), globalTagsOrigin) return c } @@ -898,7 +907,7 @@ func WithPeerServiceMapping(from, to string) StartOption { func WithGlobalTag(k string, v interface{}) StartOption { return func(c *config) { if c.globalTags.get() == nil { - c.initGlobalTags(map[string]interface{}{}) + c.initGlobalTags(map[string]interface{}{}, telemetry.OriginDefault) } c.globalTags.Lock() defer c.globalTags.Unlock() @@ -907,13 +916,14 @@ func WithGlobalTag(k string, v interface{}) StartOption { } // initGlobalTags initializes the globalTags config with the provided init value -func (c *config) initGlobalTags(init map[string]interface{}) { +func (c *config) initGlobalTags(init map[string]interface{}, origin telemetry.Origin) { apply := func(map[string]interface{}) bool { // always set the runtime ID on updates c.globalTags.current[ext.RuntimeID] = globalconfig.RuntimeID() return true } - c.globalTags = newDynamicConfig[map[string]interface{}]("trace_tags", init, apply, equalMap[string]) + c.globalTags = newDynamicConfig("trace_tags", init, apply, equalMap[string]) + c.globalTags.cfgOrigin = origin } // WithSampler sets the given sampler to be used with the tracer. By default diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 71eaac7587..55d7a65487 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -29,6 +29,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" "gopkg.in/DataDog/dd-trace-go.v1/internal/namingschema" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" "gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof" "gopkg.in/DataDog/dd-trace-go.v1/internal/version" @@ -548,6 +549,7 @@ func TestTracerOptionsDefaults(t *testing.T) { defer tracer.Stop() c := tracer.config assert.True(t, c.enabled.current) + assert.Equal(t, c.enabled.cfgOrigin, telemetry.OriginDefault) }) t.Run("override", func(t *testing.T) { @@ -556,6 +558,7 @@ func TestTracerOptionsDefaults(t *testing.T) { defer tracer.Stop() c := tracer.config assert.False(t, c.enabled.current) + assert.Equal(t, c.enabled.cfgOrigin, telemetry.OriginEnvVar) }) }) diff --git a/ddtrace/tracer/remote_config_test.go b/ddtrace/tracer/remote_config_test.go index e779daa856..3e5e4d881e 100644 --- a/ddtrace/tracer/remote_config_test.go +++ b/ddtrace/tracer/remote_config_test.go @@ -30,6 +30,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + // Apply RC. Assert _dd.rule_psr shows the RC sampling rate (0.5) is applied input := remoteconfig.ProductUpdate{ "path": []byte( @@ -47,7 +49,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient.AssertCalled( t, "ConfigChange", - []telemetry.Configuration{{Name: "trace_sample_rate", Value: 0.5, Origin: "remote_config"}}, + []telemetry.Configuration{{Name: "trace_sample_rate", Value: 0.5, Origin: telemetry.OriginRemoteConfig}}, ) //Apply RC with sampling rules. Assert _dd.rule_psr shows the corresponding rule matched rate. @@ -100,6 +102,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginEnvVar, tracer.config.traceSampleRate.cfgOrigin) + // Apply RC. Assert _dd.rule_psr shows the RC sampling rate (0.2) is applied input := remoteconfig.ProductUpdate{ "path": []byte( @@ -117,7 +121,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient.AssertCalled( t, "ConfigChange", - []telemetry.Configuration{{Name: "trace_sample_rate", Value: 0.2, Origin: "remote_config"}}, + []telemetry.Configuration{{Name: "trace_sample_rate", Value: 0.2, Origin: telemetry.OriginRemoteConfig}}, ) // Unset RC. Assert _dd.rule_psr shows the previous sampling rate (0.1) is applied @@ -135,7 +139,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient.AssertCalled( t, "ConfigChange", - []telemetry.Configuration{{Name: "trace_sample_rate", Value: 0.1, Origin: ""}}, + []telemetry.Configuration{{Name: "trace_sample_rate", Value: 0.1, Origin: telemetry.OriginDefault}}, ) }) @@ -152,6 +156,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + s := tracer.StartSpan("web.request").(*span) s.Finish() require.Equal(t, 0.1, s.Metrics[keyRulesSamplerAppliedRate]) @@ -188,11 +194,11 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient.AssertNumberOfCalls(t, "ConfigChange", 1) telemetryClient.AssertCalled(t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_sample_rate", Value: 0.5, Origin: "remote_config"}, + {Name: "trace_sample_rate", Value: 0.5, Origin: telemetry.OriginRemoteConfig}, { Name: "trace_sample_rules", Value: `[{"service":"my-service","name":"web.request","resource":"abc","sample_rate":1,"provenance":"customer"}]`, - Origin: "remote_config", + Origin: telemetry.OriginRemoteConfig, }}) }) @@ -265,16 +271,16 @@ func TestOnRemoteConfigUpdate(t *testing.T) { } telemetryClient.AssertNumberOfCalls(t, "ConfigChange", 2) telemetryClient.AssertCalled(t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_sample_rate", Value: 0.5, Origin: "remote_config"}, + {Name: "trace_sample_rate", Value: 0.5, Origin: telemetry.OriginRemoteConfig}, {Name: "trace_sample_rules", - Value: `[{"service":"my-service","name":"web.request","resource":"abc","sample_rate":1,"provenance":"customer"} {"service":"my-service","name":"web.request","resource":"*","sample_rate":0.3,"provenance":"dynamic"}]`, Origin: "remote_config"}, + Value: `[{"service":"my-service","name":"web.request","resource":"abc","sample_rate":1,"provenance":"customer"} {"service":"my-service","name":"web.request","resource":"*","sample_rate":0.3,"provenance":"dynamic"}]`, Origin: telemetry.OriginRemoteConfig}, }) telemetryClient.AssertCalled(t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_sample_rate", Value: nil, Origin: ""}, + {Name: "trace_sample_rate", Value: nil, Origin: telemetry.OriginDefault}, { Name: "trace_sample_rules", Value: `[{"service":"my-service","name":"web.request","resource":"*","sample_rate":0.1,"type":"1"}]`, - Origin: "", + Origin: telemetry.OriginDefault, }, }) }) @@ -315,11 +321,11 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient.AssertNumberOfCalls(t, "ConfigChange", 1) telemetryClient.AssertCalled(t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_sample_rate", Value: 0.5, Origin: "remote_config"}, + {Name: "trace_sample_rate", Value: 0.5, Origin: telemetry.OriginRemoteConfig}, { Name: "trace_sample_rules", Value: `[{"service":"my-service","name":"web.request","resource":"*","sample_rate":1,"tags":{"tag-a":"tv-a??"},"provenance":"customer"}]`, - Origin: "remote_config", + Origin: telemetry.OriginRemoteConfig, }, }) }) @@ -332,6 +338,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + // Apply RC. Assert global config shows the RC header tag is applied input := remoteconfig.ProductUpdate{ "path": []byte( @@ -348,7 +356,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name", Origin: "remote_config"}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name", Origin: telemetry.OriginRemoteConfig}, }, ) @@ -365,7 +373,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient.AssertCalled( t, "ConfigChange", - []telemetry.Configuration{{Name: "trace_header_tags", Value: "", Origin: ""}}, + []telemetry.Configuration{{Name: "trace_header_tags", Value: "", Origin: telemetry.OriginDefault}}, ) }) @@ -373,8 +381,10 @@ func TestOnRemoteConfigUpdate(t *testing.T) { telemetryClient := new(telemetrytest.MockClient) defer telemetry.MockGlobalClient(telemetryClient)() - Start(WithService("my-service"), WithEnv("my-env")) - defer Stop() + tr, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) + defer stop() + + require.Equal(t, telemetry.OriginDefault, tr.config.traceSampleRate.cfgOrigin) input := remoteconfig.ProductUpdate{ "path": []byte( @@ -382,8 +392,6 @@ func TestOnRemoteConfigUpdate(t *testing.T) { ), } - tr, ok := internal.GetGlobalTracer().(*tracer) - require.Equal(t, true, ok) applyStatus := tr.onRemoteConfigUpdate(input) require.Equal(t, state.ApplyStateAcknowledged, applyStatus["path"].State) require.Equal(t, false, tr.config.enabled.current) @@ -451,7 +459,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-from-rc", Origin: "remote_config"}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-from-rc", Origin: telemetry.OriginRemoteConfig}, }, ) @@ -470,7 +478,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-from-env", Origin: ""}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-from-env", Origin: telemetry.OriginDefault}, }, ) }, @@ -508,7 +516,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-from-rc", Origin: "remote_config"}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-from-rc", Origin: telemetry.OriginRemoteConfig}, }, ) @@ -527,7 +535,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-in-code", Origin: ""}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-name-in-code", Origin: telemetry.OriginDefault}, }, ) }, @@ -540,6 +548,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + input := remoteconfig.ProductUpdate{ "path": []byte( `{"lib_config": {"tracing_sampling_rate": "string value", "service_target": {"service": "my-service", "env": "my-env"}}}`, @@ -560,6 +570,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithServiceName("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + input := remoteconfig.ProductUpdate{ "path": []byte(`{"lib_config": {}, "service_target": {"service": "other-service", "env": "my-env"}}`), } @@ -578,6 +590,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithServiceName("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + input := remoteconfig.ProductUpdate{ "path": []byte(`{"lib_config": {}, "service_target": {"service": "my-service", "env": "other-env"}}`), } @@ -602,6 +616,9 @@ func TestOnRemoteConfigUpdate(t *testing.T) { ) defer stop() + require.Equal(t, telemetry.OriginDefault, tracer.config.traceSampleRate.cfgOrigin) + require.Equal(t, telemetry.OriginEnvVar, tracer.config.globalTags.cfgOrigin) + // Apply RC. Assert global tags have the RC tags key3:val3,key4:val4 applied + runtime ID input := remoteconfig.ProductUpdate{ "path": []byte( @@ -626,7 +643,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_tags", Value: "key3:val3,key4:val4," + runtimeIDTag, Origin: "remote_config"}, + {Name: "trace_tags", Value: "key3:val3,key4:val4," + runtimeIDTag, Origin: telemetry.OriginRemoteConfig}, }, ) @@ -651,7 +668,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_tags", Value: "key0:val0,key1:val1,key2:val2," + runtimeIDTag, Origin: ""}, + {Name: "trace_tags", Value: "key0:val0,key1:val1,key2:val2," + runtimeIDTag, Origin: telemetry.OriginDefault}, }, ) }) @@ -667,6 +684,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithService("my-service"), WithEnv("my-env")) defer stop() + require.Equal(t, telemetry.OriginEnvVar, tracer.config.traceSampleRate.cfgOrigin) + // Apply RC. Assert configuration is updated to the RC values. input := remoteconfig.ProductUpdate{ "path": []byte( @@ -684,12 +703,12 @@ func TestOnRemoteConfigUpdate(t *testing.T) { // Telemetry telemetryClient.AssertNumberOfCalls(t, "ConfigChange", 1) telemetryClient.AssertCalled(t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_sample_rate", Value: 0.2, Origin: "remote_config"}, - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-from-rc", Origin: "remote_config"}, + {Name: "trace_sample_rate", Value: 0.2, Origin: telemetry.OriginRemoteConfig}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-from-rc", Origin: telemetry.OriginRemoteConfig}, { Name: "trace_tags", Value: "ddtag:from-rc," + ext.RuntimeID + ":" + globalconfig.RuntimeID(), - Origin: "remote_config", + Origin: telemetry.OriginRemoteConfig, }, }) @@ -706,9 +725,9 @@ func TestOnRemoteConfigUpdate(t *testing.T) { // Telemetry telemetryClient.AssertNumberOfCalls(t, "ConfigChange", 2) telemetryClient.AssertCalled(t, "ConfigChange", []telemetry.Configuration{ - {Name: "trace_sample_rate", Value: 0.1, Origin: ""}, - {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-from-env", Origin: ""}, - {Name: "trace_tags", Value: "ddtag:from-env," + ext.RuntimeID + ":" + globalconfig.RuntimeID(), Origin: ""}, + {Name: "trace_sample_rate", Value: 0.1, Origin: telemetry.OriginDefault}, + {Name: "trace_header_tags", Value: "X-Test-Header:my-tag-from-env", Origin: telemetry.OriginDefault}, + {Name: "trace_tags", Value: "ddtag:from-env," + ext.RuntimeID + ":" + globalconfig.RuntimeID(), Origin: telemetry.OriginDefault}, }) }) diff --git a/ddtrace/tracer/telemetry.go b/ddtrace/tracer/telemetry.go index 3c9b635fb7..1f112ff830 100644 --- a/ddtrace/tracer/telemetry.go +++ b/ddtrace/tracer/telemetry.go @@ -54,7 +54,7 @@ func startTelemetry(c *config) { {Name: "trace_span_attribute_schema", Value: c.spanAttributeSchemaVersion}, {Name: "trace_peer_service_defaults_enabled", Value: c.peerServiceDefaultsEnabled}, {Name: "orchestrion_enabled", Value: c.orchestrionCfg.Enabled}, - {Name: "trace_enabled", Value: c.enabled.current}, + {Name: "trace_enabled", Value: c.enabled.current, Origin: c.enabled.cfgOrigin}, c.traceSampleRate.toTelemetry(), c.headerAsTags.toTelemetry(), c.globalTags.toTelemetry(), diff --git a/ddtrace/tracer/telemetry_test.go b/ddtrace/tracer/telemetry_test.go index f64c5e4f80..09e10cf638 100644 --- a/ddtrace/tracer/telemetry_test.go +++ b/ddtrace/tracer/telemetry_test.go @@ -47,6 +47,7 @@ func TestTelemetryEnabled(t *testing.T) { telemetry.Check(t, telemetryClient.Configuration, "env", "test-env") telemetry.Check(t, telemetryClient.Configuration, "runtime_metrics_enabled", true) telemetry.Check(t, telemetryClient.Configuration, "stats_computation_enabled", false) + telemetry.Check(t, telemetryClient.Configuration, "trace_enabled", true) telemetry.Check(t, telemetryClient.Configuration, "trace_span_attribute_schema", 0) telemetry.Check(t, telemetryClient.Configuration, "trace_peer_service_defaults_enabled", true) telemetry.Check(t, telemetryClient.Configuration, "trace_peer_service_mapping", "key:val") diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index c40a8293bf..93e07ec407 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -8,6 +8,7 @@ package tracer import ( gocontext "context" "encoding/binary" + "math" "os" "runtime/pprof" rt "runtime/trace" @@ -252,6 +253,13 @@ func newUnstartedTracer(opts ...StartOption) *tracer { globalRate := globalSampleRate() rulesSampler := newRulesSampler(c.traceRules, c.spanRules, globalRate) c.traceSampleRate = newDynamicConfig("trace_sample_rate", globalRate, rulesSampler.traces.setGlobalSampleRate, equal[float64]) + // If globalSampleRate returns NaN, it means the environment variable was not set or valid. + // We could always set the origin to "env_var" inconditionally, but then it wouldn't be possible + // to distinguish between the case where the environment variable was not set and the case where + // it default to NaN. + if !math.IsNaN(globalRate) { + c.traceSampleRate.cfgOrigin = telemetry.OriginEnvVar + } c.traceSampleRules = newDynamicConfig("trace_sample_rules", c.traceRules, rulesSampler.traces.setTraceSampleRules, EqualsFalseNegative) var dataStreamsProcessor *datastreams.Processor diff --git a/internal/appsec/config/config.go b/internal/appsec/config/config.go index d7cf538881..33ca45e56b 100644 --- a/internal/appsec/config/config.go +++ b/internal/appsec/config/config.go @@ -37,7 +37,7 @@ func registerSCAAppConfigTelemetry(client telemetry.Client) { return } if defined { - client.RegisterAppConfig(EnvSCAEnabled, val, "env_var") + client.RegisterAppConfig(EnvSCAEnabled, val, telemetry.OriginEnvVar) } } diff --git a/internal/appsec/config/config_test.go b/internal/appsec/config/config_test.go index 19dec18669..77d29e36df 100644 --- a/internal/appsec/config/config_test.go +++ b/internal/appsec/config/config_test.go @@ -8,6 +8,7 @@ package config import ( "testing" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry/telemetrytest" ) @@ -49,12 +50,12 @@ func TestSCAEnabled(t *testing.T) { } telemetryClient := new(telemetrytest.MockClient) - telemetryClient.On("RegisterAppConfig", EnvSCAEnabled, tc.expectedValue, "env_var").Return() + telemetryClient.On("RegisterAppConfig", EnvSCAEnabled, tc.expectedValue, telemetry.OriginEnvVar).Return() registerSCAAppConfigTelemetry(telemetryClient) if tc.telemetryExpected { - telemetryClient.AssertCalled(t, "RegisterAppConfig", EnvSCAEnabled, tc.expectedValue, "env_var") + telemetryClient.AssertCalled(t, "RegisterAppConfig", EnvSCAEnabled, tc.expectedValue, telemetry.OriginEnvVar) telemetryClient.AssertNumberOfCalls(t, "RegisterAppConfig", 1) } else { telemetryClient.AssertNumberOfCalls(t, "RegisterAppConfig", 0) diff --git a/internal/appsec/telemetry.go b/internal/appsec/telemetry.go index aeac5827d2..2b07117bd8 100644 --- a/internal/appsec/telemetry.go +++ b/internal/appsec/telemetry.go @@ -26,10 +26,10 @@ var ( wafSupported, _ = waf.SupportsTarget() wafHealthy, _ = waf.Health() staticConfigs = []telemetry.Configuration{ - {Name: "goos", Value: runtime.GOOS, Origin: "code"}, - {Name: "goarch", Value: runtime.GOARCH, Origin: "code"}, - {Name: "waf_supports_target", Value: wafSupported, Origin: "code"}, - {Name: "waf_healthy", Value: wafHealthy, Origin: "code"}, + {Name: "goos", Value: runtime.GOOS, Origin: telemetry.OriginCode}, + {Name: "goarch", Value: runtime.GOARCH, Origin: telemetry.OriginCode}, + {Name: "waf_supports_target", Value: wafSupported, Origin: telemetry.OriginCode}, + {Name: "waf_healthy", Value: wafHealthy, Origin: telemetry.OriginCode}, } ) @@ -62,7 +62,7 @@ func (a *appsecTelemetry) addEnvConfig(name string, value any) { if a == nil { return } - a.configs = append(a.configs, telemetry.Configuration{Name: name, Value: value, Origin: "env_var"}) + a.configs = append(a.configs, telemetry.Configuration{Name: name, Value: value, Origin: telemetry.OriginEnvVar}) } // setEnabled makes AppSec as having effectively been enabled. diff --git a/internal/telemetry/client.go b/internal/telemetry/client.go index 1613c5f911..1b01ced31f 100644 --- a/internal/telemetry/client.go +++ b/internal/telemetry/client.go @@ -30,7 +30,7 @@ import ( // Client buffers and sends telemetry messages to Datadog (possibly through an // agent). type Client interface { - RegisterAppConfig(name string, val interface{}, origin string) + RegisterAppConfig(name string, val interface{}, origin Origin) ProductChange(namespace Namespace, enabled bool, configuration []Configuration) ConfigChange(configuration []Configuration) Record(namespace Namespace, metric MetricKind, name string, value float64, tags []string, common bool) @@ -158,7 +158,7 @@ func log(msg string, args ...interface{}) { // RegisterAppConfig allows to register a globally-defined application configuration. // This configuration will be sent when the telemetry client is started and over related configuration updates. -func (c *client) RegisterAppConfig(name string, value interface{}, origin string) { +func (c *client) RegisterAppConfig(name string, value interface{}, origin Origin) { c.globalAppConfig = append(c.globalAppConfig, Configuration{ Name: name, Value: value, diff --git a/internal/telemetry/message.go b/internal/telemetry/message.go index 3348187596..27a72768d9 100644 --- a/internal/telemetry/message.go +++ b/internal/telemetry/message.go @@ -5,7 +5,11 @@ package telemetry -import "net/http" +import ( + "bytes" + "fmt" + "net/http" +) // Request captures all necessary information for a telemetry event submission type Request struct { @@ -132,13 +136,48 @@ type ConfigurationChange struct { RemoteConfig *RemoteConfig `json:"remote_config,omitempty"` } +type Origin int + +const ( + OriginDefault Origin = iota + OriginCode + OriginDDConfig + OriginEnvVar + OriginRemoteConfig +) + +func (o Origin) String() string { + switch o { + case OriginDefault: + return "default" + case OriginCode: + return "code" + case OriginDDConfig: + return "dd_config" + case OriginEnvVar: + return "env_var" + case OriginRemoteConfig: + return "remote_config" + default: + return fmt.Sprintf("unknown origin %d", o) + } +} + +func (o Origin) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + b.WriteString(`"`) + b.WriteString(o.String()) + b.WriteString(`"`) + return b.Bytes(), nil +} + // Configuration is a library-specific configuration value // that should be initialized through StringConfig, IntConfig, FloatConfig, or BoolConfig type Configuration struct { Name string `json:"name"` Value interface{} `json:"value"` - // origin is the source of the config. It is one of {env_var, code, dd_config, remote_config} - Origin string `json:"origin"` + // origin is the source of the config. It is one of {default, env_var, code, dd_config, remote_config}. + Origin Origin `json:"origin"` Error Error `json:"error"` IsOverriden bool `json:"is_overridden"` } diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index e7c917100b..2666a91c44 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -142,27 +142,27 @@ func TestProductChange(t *testing.T) { // Test that globally registered app config is sent in telemetry requests including the configuration state. func TestRegisterAppConfig(t *testing.T) { client := new(client) - client.RegisterAppConfig("key1", "val1", "origin1") + client.RegisterAppConfig("key1", "val1", OriginDefault) // Test that globally registered app config is sent in app-started payloads - client.start([]Configuration{{Name: "key2", Value: "val2", Origin: "origin2"}}, NamespaceTracers, false) + client.start([]Configuration{{Name: "key2", Value: "val2", Origin: OriginDDConfig}}, NamespaceTracers, false) req := client.requests[0].Body require.Equal(t, RequestTypeAppStarted, req.RequestType) appStarted := req.Payload.(*AppStarted) cfg := appStarted.Configuration require.Len(t, cfg, 2) - require.Contains(t, cfg, Configuration{Name: "key1", Value: "val1", Origin: "origin1"}) - require.Contains(t, cfg, Configuration{Name: "key2", Value: "val2", Origin: "origin2"}) + require.Contains(t, cfg, Configuration{Name: "key1", Value: "val1", Origin: OriginDefault}) + require.Contains(t, cfg, Configuration{Name: "key2", Value: "val2", Origin: OriginDDConfig}) // Test that globally registered app config is sent in app-client-configuration-change payloads - client.ProductChange(NamespaceTracers, true, []Configuration{{Name: "key3", Value: "val3", Origin: "origin3"}}) + client.ProductChange(NamespaceTracers, true, []Configuration{{Name: "key3", Value: "val3", Origin: OriginCode}}) req = client.requests[2].Body require.Equal(t, RequestTypeAppClientConfigurationChange, req.RequestType) appConfigChange := req.Payload.(*ConfigurationChange) cfg = appConfigChange.Configuration require.Len(t, cfg, 2) - require.Contains(t, cfg, Configuration{Name: "key1", Value: "val1", Origin: "origin1"}) - require.Contains(t, cfg, Configuration{Name: "key3", Value: "val3", Origin: "origin3"}) + require.Contains(t, cfg, Configuration{Name: "key1", Value: "val1", Origin: OriginDefault}) + require.Contains(t, cfg, Configuration{Name: "key3", Value: "val3", Origin: OriginCode}) } diff --git a/internal/telemetry/telemetrytest/telemetrytest.go b/internal/telemetry/telemetrytest/telemetrytest.go index 9a0df1b78c..0c8b22bd32 100644 --- a/internal/telemetry/telemetrytest/telemetrytest.go +++ b/internal/telemetry/telemetrytest/telemetrytest.go @@ -27,7 +27,7 @@ type MockClient struct { Metrics map[telemetry.Namespace]map[string]float64 } -func (c *MockClient) RegisterAppConfig(name string, val interface{}, origin string) { +func (c *MockClient) RegisterAppConfig(name string, val interface{}, origin telemetry.Origin) { _ = c.Called(name, val, origin) } From dd54cadc786cfc0db127a2b3fccd393bf24f29dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Thu, 30 May 2024 09:55:05 +0200 Subject: [PATCH 06/14] contrib/net/http: Support go1.22 ServeMux patterns (#2716) --- contrib/net/http/http.go | 17 ++++++++++++- contrib/net/http/http_test.go | 46 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/contrib/net/http/http.go b/contrib/net/http/http.go index b95bbffe41..5aedb032a4 100644 --- a/contrib/net/http/http.go +++ b/contrib/net/http/http.go @@ -8,6 +8,7 @@ package http // import "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" import ( "net/http" + "strings" "gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" @@ -49,7 +50,8 @@ func (mux *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } // get the resource associated to this request - _, route := mux.Handler(r) + _, pattern := mux.Handler(r) + route := patternRoute(pattern) resource := mux.cfg.resourceNamer(r) if resource == "" { resource = r.Method + " " + route @@ -65,6 +67,19 @@ func (mux *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { }) } +// patternRoute returns the route part of a go1.22 style ServeMux pattern. I.e. +// it returns "/foo" for the pattern "/foo" as well as the pattern "GET /foo". +func patternRoute(s string) string { + // Support go1.22 serve mux patterns: [METHOD ][HOST]/[PATH] + // Consider any text before a space or tab to be the method of the pattern. + // See net/http.parsePattern and the link below for more information. + // https://pkg.go.dev/net/http#hdr-Patterns + if i := strings.IndexAny(s, " \t"); i > 0 && len(s) >= i+1 { + return strings.TrimLeft(s[i+1:], " \t") + } + return s +} + // WrapHandler wraps an http.Handler with tracing using the given service and resource. // If the WithResourceNamer option is provided as part of opts, it will take precedence over the resource argument. func WrapHandler(h http.Handler, service, resource string, opts ...Option) http.Handler { diff --git a/contrib/net/http/http_test.go b/contrib/net/http/http_test.go index 07bcf221fa..18e92ea443 100644 --- a/contrib/net/http/http_test.go +++ b/contrib/net/http/http_test.go @@ -290,6 +290,52 @@ func TestServeMuxUsesResourceNamer(t *testing.T) { assert.Equal("net/http", s.Tag(ext.Component)) } +func TestServeMuxGo122Patterns(t *testing.T) { + mt := mocktracer.Start() + defer mt.Stop() + + // A mux with go1.21 patterns ("/bar") and go1.22 patterns ("GET /foo") + mux := NewServeMux() + mux.HandleFunc("/bar", func(w http.ResponseWriter, r *http.Request) {}) + mux.HandleFunc("GET /foo", func(w http.ResponseWriter, r *http.Request) {}) + + // Try to hit both routes + barW := httptest.NewRecorder() + mux.ServeHTTP(barW, httptest.NewRequest("GET", "/bar", nil)) + fooW := httptest.NewRecorder() + mux.ServeHTTP(fooW, httptest.NewRequest("GET", "/foo", nil)) + + // Assert the number of spans + assert := assert.New(t) + spans := mt.FinishedSpans() + assert.Equal(2, len(spans)) + + // Check the /bar span + barSpan := spans[0] + assert.Equal(http.StatusOK, barW.Code) + assert.Equal("/bar", barSpan.Tag(ext.HTTPRoute)) + assert.Equal("GET /bar", barSpan.Tag(ext.ResourceName)) + + // Check the /foo span + fooSpan := spans[1] + if fooW.Code == http.StatusOK { + assert.Equal("/foo", fooSpan.Tag(ext.HTTPRoute)) + assert.Equal("GET /foo", fooSpan.Tag(ext.ResourceName)) + } else { + // Until our go.mod version is go1.22 or greater, the mux will not + // understand the "GET /foo" pattern, causing the request to be handled + // by the 404 handler. Let's assert what we can, and mark the test as + // skipped to highlight the issue. + assert.Equal(http.StatusNotFound, fooW.Code) + assert.Equal(nil, fooSpan.Tag(ext.HTTPRoute)) + // Using "GET " as a resource name doesn't seem ideal, but that's how + // the mux instrumentation deals with 404s right now. + assert.Equal("GET ", fooSpan.Tag(ext.ResourceName)) + t.Skip("run `go mod edit -go=1.22` to run the full test") + } + +} + func TestWrapHandlerWithResourceNameNoRace(_ *testing.T) { mt := mocktracer.Start() defer mt.Stop() From b137f2ca2b7c121dffff00d0f3253f33f158b4e6 Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Thu, 30 May 2024 10:52:21 -0400 Subject: [PATCH 07/14] profiler: remove TestProfilerPassthrough and BenchmarkDoRequest (#2702) The test does not verify anything useful. It modifies several private variables/fields to change the behavior of the profiler, and makes assertions about the modified behavior. If we change the internal implementation of the profiler, this test could break without identifying an actual bug. Tests should, where possible, only make assertions about the observable behavior (what the profiler sends to the agent) based on the public API (what a customer will actually use). TestAllUploaded does what this test is meant to do, but without perturbing the implementation. BenchmarkDoRequest also modifies implementation details, and more importantly measures something that is in reality almost entirely I/O-bound in an unrealistic way. The real overhead of the profiler comes from elsewhere: delta profile computation, which we already measure, and the overhead of the profilers themselves, which are better studied on real workloads. --- profiler/profiler_test.go | 35 ----------------------------------- profiler/upload_test.go | 31 ------------------------------- 2 files changed, 66 deletions(-) diff --git a/profiler/profiler_test.go b/profiler/profiler_test.go index e9d56444c2..5e94eac82e 100644 --- a/profiler/profiler_test.go +++ b/profiler/profiler_test.go @@ -236,41 +236,6 @@ func TestSetProfileFraction(t *testing.T) { }) } -func TestProfilerPassthrough(t *testing.T) { - if testing.Short() { - return - } - beforeExecutionTraceEnabledDefault := executionTraceEnabledDefault - executionTraceEnabledDefault = false - defer func() { executionTraceEnabledDefault = beforeExecutionTraceEnabledDefault }() - - out := make(chan batch) - p, err := newProfiler() - require.NoError(t, err) - p.cfg.period = 200 * time.Millisecond - p.cfg.cpuDuration = 1 * time.Millisecond - p.uploadFunc = func(bat batch) error { - out <- bat - return nil - } - p.run() - defer p.stop() - var bat batch - select { - case bat = <-out: - // TODO (knusbaum) this timeout is long because we were seeing timeouts at 500ms. - // it would be nice to have a time-independent way to test this - case <-time.After(1000 * time.Millisecond): - t.Fatal("time expired") - } - - assert := assert.New(t) - // should contain cpu.pprof, delta-heap.pprof - assert.Equal(2, len(bat.profiles)) - assert.NotEmpty(bat.profiles[0].data) - assert.NotEmpty(bat.profiles[1].data) -} - func unstartedProfiler(opts ...Option) (*profiler, error) { p, err := newProfiler(opts...) if err != nil { diff --git a/profiler/upload_test.go b/profiler/upload_test.go index acaa471011..1e56eb0e4a 100644 --- a/profiler/upload_test.go +++ b/profiler/upload_test.go @@ -7,7 +7,6 @@ package profiler import ( "fmt" - "io" "net" "net/http" "net/http/httptest" @@ -187,36 +186,6 @@ func TestEntityIDHeader(t *testing.T) { assert.Equal(t, entityID, profile.headers.Get("Datadog-Entity-Id")) } -func BenchmarkDoRequest(b *testing.B) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, err := io.ReadAll(req.Body) - if err != nil { - b.Fatal(err) - } - req.Body.Close() - w.WriteHeader(200) - })) - defer srv.Close() - prof := profile{ - name: "heap", - data: []byte("my-heap-profile"), - } - bat := batch{ - start: time.Now().Add(-10 * time.Second), - end: time.Now(), - host: "my-host", - profiles: []*profile{&prof}, - } - p, err := unstartedProfiler() - require.NoError(b, err) - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - p.doRequest(bat) - } -} - func TestGitMetadata(t *testing.T) { maininternal.ResetGitMetadataTags() defer maininternal.ResetGitMetadataTags() From e0b2c6dd65c64b1a441fe043805241e4d7a7a3e9 Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Thu, 30 May 2024 11:03:58 -0400 Subject: [PATCH 08/14] profiler: remove TestTryUpload (#2706) This is yet another test that hooks into internal implementation details of the profiler. It's been mostly superseded by TestAllUploaded. The bits that are unique to this test (asserting some of the default tags and a few details of the event payload) can be folded into tests that aren't (as) dependent on implementation details. --- profiler/profiler_test.go | 15 ++++++ profiler/upload_test.go | 103 +++++++------------------------------- 2 files changed, 33 insertions(+), 85 deletions(-) diff --git a/profiler/profiler_test.go b/profiler/profiler_test.go index 5e94eac82e..09d2d2a1cb 100644 --- a/profiler/profiler_test.go +++ b/profiler/profiler_test.go @@ -25,9 +25,11 @@ import ( "time" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" "gopkg.in/DataDog/dd-trace-go.v1/internal/httpmem" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof" + "gopkg.in/DataDog/dd-trace-go.v1/internal/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -381,6 +383,11 @@ func TestAllUploaded(t *testing.T) { assert.ElementsMatch(t, customLabelKeys[:customProfileLabelLimit], profile.event.CustomAttributes) assert.Contains(t, profile.tags, fmt.Sprintf("profile_seq:%d", seq)) + + assert.Equal(t, profile.event.Version, "4") + assert.Equal(t, profile.event.Family, "go") + assert.NotNil(t, profile.event.Start) + assert.NotNil(t, profile.event.End) } validateProfile(<-profiles, 0) @@ -402,6 +409,14 @@ func TestCorrectTags(t *testing.T) { "foo:bar", "service:xyz", "host:example", + "runtime:go", + fmt.Sprintf("process_id:%d", os.Getpid()), + fmt.Sprintf("profiler_version:%s", version.Tag), + fmt.Sprintf("runtime_version:%s", strings.TrimPrefix(runtime.Version(), "go")), + fmt.Sprintf("runtime_compiler:%s", runtime.Compiler), + fmt.Sprintf("runtime_arch:%s", runtime.GOARCH), + fmt.Sprintf("runtime_os:%s", runtime.GOOS), + fmt.Sprintf("runtime-id:%s", globalconfig.RuntimeID()), } for i := 0; i < 20; i++ { // We check the tags we get several times to try to have a diff --git a/profiler/upload_test.go b/profiler/upload_test.go index 1e56eb0e4a..2e26a5bd70 100644 --- a/profiler/upload_test.go +++ b/profiler/upload_test.go @@ -6,19 +6,14 @@ package profiler import ( - "fmt" "net" "net/http" "net/http/httptest" - "os" "runtime" - "strings" "testing" "time" maininternal "gopkg.in/DataDog/dd-trace-go.v1/internal" - "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" - "gopkg.in/DataDog/dd-trace-go.v1/internal/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -41,57 +36,6 @@ var testBatch = batch{ }, } -func TestTryUpload(t *testing.T) { - // Force an empty containerid and entityID on this test. - defer func(cid, eid string) { containerID = cid; entityID = eid }(containerID, entityID) - containerID = "" - entityID = "" - - profiles := make(chan profileMeta, 1) - server := httptest.NewServer(&mockBackend{t: t, profiles: profiles}) - defer server.Close() - p, err := unstartedProfiler( - WithAgentAddr(server.Listener.Addr().String()), - WithService("my-service"), - WithEnv("my-env"), - WithTags("tag1:1", "tag2:2"), - ) - require.NoError(t, err) - err = p.doRequest(testBatch) - require.NoError(t, err) - profile := <-profiles - - assert := assert.New(t) - assert.Empty(profile.headers.Get("Datadog-Container-ID")) - assert.Empty(profile.headers.Get("Datadog-Entity-ID")) - assert.Subset(profile.tags, []string{ - "host:my-host", - "runtime:go", - "service:my-service", - "env:my-env", - "profile_seq:23", - "tag1:1", - "tag2:2", - fmt.Sprintf("process_id:%d", os.Getpid()), - fmt.Sprintf("profiler_version:%s", version.Tag), - fmt.Sprintf("runtime_version:%s", strings.TrimPrefix(runtime.Version(), "go")), - fmt.Sprintf("runtime_compiler:%s", runtime.Compiler), - fmt.Sprintf("runtime_arch:%s", runtime.GOARCH), - fmt.Sprintf("runtime_os:%s", runtime.GOOS), - fmt.Sprintf("runtime-id:%s", globalconfig.RuntimeID()), - }) - assert.Equal(profile.event.Version, "4") - assert.Equal(profile.event.Family, "go") - assert.NotNil(profile.event.Start) - assert.NotNil(profile.event.End) - for k, v := range map[string][]byte{ - "cpu.pprof": []byte("my-cpu-profile"), - "heap.pprof": []byte("my-heap-profile"), - } { - assert.Equal(v, profile.attachments[k]) - } -} - func TestTryUploadUDS(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Unix domain sockets are non-functional on windows.") @@ -155,35 +99,24 @@ func TestOldAgent(t *testing.T) { assert.Equal(t, errOldAgent, err) } -func TestContainerIDHeader(t *testing.T) { - // Force a non-empty containerid on this test. - defer func(cid string) { containerID = cid }(containerID) - containerID = "fakeContainerID" - - profile := doOneShortProfileUpload(t) - assert.Equal(t, containerID, profile.headers.Get("Datadog-Container-Id")) -} - -func TestEntityIDHeader(t *testing.T) { - // Force a non-empty entityID on this test. - defer func(eid string) { entityID = eid }(entityID) - entityID = "fakeEntityID" - - profiles := make(chan profileMeta, 1) - server := httptest.NewServer(&mockBackend{t: t, profiles: profiles}) - defer server.Close() - p, err := unstartedProfiler( - WithAgentAddr(server.Listener.Addr().String()), - WithService("my-service"), - WithEnv("my-env"), - WithTags("tag1:1", "tag2:2"), - ) - require.NoError(t, err) - err = p.doRequest(testBatch) - require.NoError(t, err) - - profile := <-profiles - assert.Equal(t, entityID, profile.headers.Get("Datadog-Entity-Id")) +func TestEntityContainerIDHeaders(t *testing.T) { + t.Run("set", func(t *testing.T) { + defer func(cid, eid string) { containerID = cid; entityID = eid }(containerID, entityID) + entityID = "fakeEntityID" + containerID = "fakeContainerID" + profile := doOneShortProfileUpload(t) + assert.Equal(t, containerID, profile.headers.Get("Datadog-Container-Id")) + assert.Equal(t, entityID, profile.headers.Get("Datadog-Entity-Id")) + }) + t.Run("unset", func(t *testing.T) { + // Force an empty containerid and entityID on this test. + defer func(cid, eid string) { containerID = cid; entityID = eid }(containerID, entityID) + entityID = "" + containerID = "" + profile := doOneShortProfileUpload(t) + assert.Empty(t, profile.headers.Get("Datadog-Container-ID")) + assert.Empty(t, profile.headers.Get("Datadog-Entity-ID")) + }) } func TestGitMetadata(t *testing.T) { From a43e1665af446ddef33bc2d6647881a6a0d01591 Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Fri, 31 May 2024 10:11:09 -0400 Subject: [PATCH 09/14] ddtrace/tracer: only finish execution trace task, restore pprof labels once (#2708) It's possible for users to call Finish multiple times on a span. We should only record the span finishing in the execution tracer and via pprof labels one time, though. Otherwise we're 1) wasting space in the trace and 2) possibly overriding pprof labels with incorrect values. Move the task ending and label setting inside span.finish, after we check whether the span is already finished. To parse execution traces for testing execution trace-related functionality, we'll need to use golang.org/x/exp/trace. As new versions of the format come out, we'll need to upgrade that dependency to keep tests working. The golang.org/x/exp module is prone to breaking changes, so we should try not to inflict them on our users. Thus, this commit creates a separate module where execution trace testing logic can live, so we can freely upgrade golang.org/x/exp. Future commits will move existing test logic, which uses gotraceui's parser, to this new module. --- .github/workflows/unit-integration-tests.yml | 4 +- ddtrace/tracer/span.go | 19 +-- internal/exectracetest/exectrace_go120.go | 8 + internal/exectracetest/exectrace_test.go | 141 ++++++++++++++++++ internal/exectracetest/go.mod | 40 +++++ internal/exectracetest/go.sum | 146 +++++++++++++++++++ 6 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 internal/exectracetest/exectrace_go120.go create mode 100644 internal/exectracetest/exectrace_test.go create mode 100644 internal/exectracetest/go.mod create mode 100644 internal/exectracetest/go.sum diff --git a/.github/workflows/unit-integration-tests.yml b/.github/workflows/unit-integration-tests.yml index 3d88429d45..e8db780839 100644 --- a/.github/workflows/unit-integration-tests.yml +++ b/.github/workflows/unit-integration-tests.yml @@ -311,6 +311,8 @@ jobs: mkdir -p $TEST_RESULTS PACKAGE_NAMES=$(go list ./... | grep -v /contrib/) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic + cd ./internal/exectracetest + gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report-exectrace.xml -- -v -race -coverprofile=coverage.txt -covermode=atomic - name: Upload the results to Datadog CI App if: always() @@ -318,7 +320,7 @@ jobs: uses: ./.github/actions/dd-ci-upload with: dd-api-key: ${{ secrets.DD_CI_API_KEY }} - files: ${{ env.TEST_RESULTS }}/gotestsum-report.xml + files: ${{ env.TEST_RESULTS }}/gotestsum-report.xml ${{ env.TEST_RESULTS }}/gotestsum-report-exectrace.xml tags: go:${{ inputs.go-version }}},arch:${{ runner.arch }},os:${{ runner.os }},distribution:${{ runner.distribution }} - name: Upload Coverage if: always() diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index 7d5ebf3a52..ee201c4f9c 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -478,9 +478,7 @@ func (s *span) Finish(opts ...ddtrace.FinishOption) { s.Unlock() } } - if s.taskEnd != nil { - s.taskEnd() - } + if s.goExecTraced && rt.IsEnabled() { // Only tag spans as traced if they both started & ended with // execution tracing enabled. This is technically not sufficient @@ -505,12 +503,6 @@ func (s *span) Finish(opts ...ddtrace.FinishOption) { } s.finish(t) - - if s.pprofCtxRestore != nil { - // Restore the labels of the parent span so any CPU samples after this - // point are attributed correctly. - pprof.SetGoroutineLabels(s.pprofCtxRestore) - } } // SetOperationName sets or changes the operation name. @@ -543,6 +535,9 @@ func (s *span) finish(finishTime int64) { if s.Duration < 0 { s.Duration = 0 } + if s.taskEnd != nil { + s.taskEnd() + } keep := true if t, ok := internal.GetGlobalTracer().(*tracer); ok { @@ -583,6 +578,12 @@ func (s *span) finish(finishTime int64) { s, s.Name, s.Resource, s.Meta, s.Metrics) } s.context.finish() + + if s.pprofCtxRestore != nil { + // Restore the labels of the parent span so any CPU samples after this + // point are attributed correctly. + pprof.SetGoroutineLabels(s.pprofCtxRestore) + } } // newAggregableSpan creates a new summary for the span s, within an application diff --git a/internal/exectracetest/exectrace_go120.go b/internal/exectracetest/exectrace_go120.go new file mode 100644 index 0000000000..17703613cb --- /dev/null +++ b/internal/exectracetest/exectrace_go120.go @@ -0,0 +1,8 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package exectracetest + +// Placeholder -- the latestgolang.org/x/exp does not support go1.20 diff --git a/internal/exectracetest/exectrace_test.go b/internal/exectracetest/exectrace_test.go new file mode 100644 index 0000000000..ffeb280d66 --- /dev/null +++ b/internal/exectracetest/exectrace_test.go @@ -0,0 +1,141 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +//go:build go1.21 + +// exectracetest tests execution tracer-related functionality. +// The official execution trace parser lives in golang.org/x/exp, +// which we generally should avoid upgrading as it is prone +// to breaking changes which could affect our customers. +// So, this package lives in a separate module in order to +// freely upgrade golang.org/x/exp/trace as the trace format changes. +package exectracetest + +import ( + "bytes" + "context" + "io" + "regexp" + "runtime/pprof" + "runtime/trace" + "testing" + "time" + + "github.com/google/pprof/profile" + exptrace "golang.org/x/exp/trace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +type discardLogger struct{} + +func (discardLogger) Log(msg string) {} + +// collectTestData runs f under the CPU profiler and execution tracer +func collectTestData(t *testing.T, f func()) (*profile.Profile, []exptrace.Event) { + cpuProf := new(bytes.Buffer) + execTrace := new(bytes.Buffer) + if pprof.StartCPUProfile(cpuProf) != nil { + t.Skip("CPU profiler already running") + } + defer pprof.StopCPUProfile() // okay to double-stop + if trace.Start(execTrace) != nil { + t.Skip("execution tracer already running") + } + defer trace.Stop() // okay to double-stop + + f() + + trace.Stop() + pprof.StopCPUProfile() + + pprof, err := profile.Parse(cpuProf) + if err != nil { + t.Fatalf("parsing profile: %s", err) + } + reader, err := exptrace.NewReader(execTrace) + if err != nil { + t.Fatalf("reading execution trace: %s", err) + } + var traceEvents []exptrace.Event + for { + ev, err := reader.ReadEvent() + if err == io.EOF { + break + } else if err != nil { + t.Fatalf("reading event: %s", err) + } + traceEvents = append(traceEvents, ev) + } + return pprof, traceEvents +} + +func waste(d time.Duration) { + now := time.Now() + for time.Since(now) < d { + } +} + +func TestSpanDoubleFinish(t *testing.T) { + generate := func(d time.Duration) { + tracer.Start(tracer.WithLogger(discardLogger{})) + defer tracer.Stop() + foo, ctx := tracer.StartSpanFromContext(context.Background(), "foo") + bar, _ := tracer.StartSpanFromContext(ctx, "bar") + bar.Finish() + foo.Finish() + bar.Finish() + // If we don't handle double finish right, we will see CPU profile samples + // for waste tagged with the trace/span IDs from foo + waste(d) + } + + var ( + pprof *profile.Profile + execTrace []exptrace.Event + retries int + ) + const maxRetries = 5 + for duration := 30 * time.Millisecond; retries < maxRetries; retries++ { + pprof, execTrace = collectTestData(t, func() { + generate(duration) + }) + focus, _, _, _ := pprof.FilterSamplesByName(regexp.MustCompile("waste"), nil, nil, nil) + if !focus || len(execTrace) == 0 { + // Retry with longer run to reduce flake likelihood + duration *= 2 + continue + } + break + } + if retries == maxRetries { + t.Fatalf("could not collect sufficient data after %d retries", maxRetries) + } + + // Check CPU profile: we should have un-set the labels + // for the goroutine by the time waste5 runs + for _, sample := range pprof.Sample { + if labels := sample.Label; len(labels) > 0 { + t.Errorf("unexpected labels for sample: %+v", labels) + } + } + + // Check execution trace: we should not emit a second task end event + // even though we finish the "bar" span twice + taskEvents := make(map[exptrace.TaskID][]exptrace.Event) + for _, ev := range execTrace { + switch ev.Kind() { + case exptrace.EventTaskBegin, exptrace.EventTaskEnd: + id := ev.Task().ID + taskEvents[id] = append(taskEvents[id], ev) + } + } + for id, events := range taskEvents { + if len(events) > 2 { + t.Errorf("extraneous events for task %d: %v", id, events) + } + } +} + +// TODO: move database/sql tests here? likely requires copying over contrib/sql/internal.MockDriver diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod new file mode 100644 index 0000000000..f357481015 --- /dev/null +++ b/internal/exectracetest/go.mod @@ -0,0 +1,40 @@ +module gopkg.in/DataDog/dd-trace-go.v1/internal/exectracetest + +go 1.20 + +require ( + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + gopkg.in/DataDog/dd-trace-go.v1 v1.64.0 +) + +require ( + github.com/DataDog/appsec-internal-go v1.5.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect + github.com/DataDog/datadog-go/v5 v5.3.0 // indirect + github.com/DataDog/go-libddwaf/v3 v3.2.0 // indirect + github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect + github.com/DataDog/sketches-go v1.4.5 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.6.0-alpha.5 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect + github.com/tinylib/msgp v1.1.8 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.21.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/protobuf v1.32.0 // indirect +) + +// use local version of dd-trace-go +replace gopkg.in/DataDog/dd-trace-go.v1 => ../.. diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum new file mode 100644 index 0000000000..3effef625d --- /dev/null +++ b/internal/exectracetest/go.sum @@ -0,0 +1,146 @@ +github.com/DataDog/appsec-internal-go v1.5.0 h1:8kS5zSx5T49uZ8dZTdT19QVAvC/B8ByyZdhQKYQWHno= +github.com/DataDog/appsec-internal-go v1.5.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ= +github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8= +github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q= +github.com/DataDog/go-libddwaf/v3 v3.2.0 h1:arvhB3A+TCQXT0eg4J9ksdIhuOO5b1OpHdu0EX8WHuc= +github.com/DataDog/go-libddwaf/v3 v3.2.0/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY= +github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I= +github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/sketches-go v1.4.5 h1:ki7VfeNz7IcNafq7yI/j5U/YCkO3LJiMDtXz9OMQbyE= +github.com/DataDog/sketches-go v1.4.5/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg= +github.com/ebitengine/purego v0.6.0-alpha.5 h1:EYID3JOAdmQ4SNZYJHu9V6IqOeRQDBYxqKAg9PyoHFY= +github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= +github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= +github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/gotraceui v0.2.0 h1:dmNsfQ9Vl3GwbiVD7Z8d/osC6WtGGrasyrC2suc4ZIQ= From 92095e9b9313df5e15d381a76150681facbaaaec Mon Sep 17 00:00:00 2001 From: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> Date: Fri, 31 May 2024 11:12:43 -0400 Subject: [PATCH 10/14] ddtrace/tracer: add support for OTEL env vars (#2715) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Munir Abdinur Co-authored-by: Dario Castañé --- ddtrace/tracer/option.go | 29 +++- ddtrace/tracer/option_test.go | 139 ++++++++++++++++- ddtrace/tracer/otel_dd_mappings.go | 198 ++++++++++++++++++++++++ ddtrace/tracer/otel_dd_mappings_test.go | 49 ++++++ ddtrace/tracer/rules_sampler.go | 20 --- ddtrace/tracer/sampler_test.go | 48 ++++-- ddtrace/tracer/textmap.go | 2 +- ddtrace/tracer/textmap_test.go | 72 +++++++++ ddtrace/tracer/tracer.go | 7 +- ddtrace/tracer/tracer_test.go | 37 ++++- internal/env.go | 21 ++- internal/utils.go | 6 + 12 files changed, 562 insertions(+), 66 deletions(-) create mode 100644 ddtrace/tracer/otel_dd_mappings.go create mode 100644 ddtrace/tracer/otel_dd_mappings_test.go diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 30352a9ba0..68de8390ef 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -271,6 +271,9 @@ type config struct { // dynamicInstrumentationEnabled controls if the target application can be modified by Dynamic Instrumentation or not. // Value from DD_DYNAMIC_INSTRUMENTATION_ENABLED, default false. dynamicInstrumentationEnabled bool + + // globalSampleRate holds sample rate read from environment variables. + globalSampleRate float64 } // orchestrionConfig contains Orchestrion configuration. @@ -302,8 +305,20 @@ const partialFlushMinSpansDefault = 1000 func newConfig(opts ...StartOption) *config { c := new(config) c.sampler = NewAllSampler() + defaultRate, err := strconv.ParseFloat(getDDorOtelConfig("sampleRate"), 64) + if err != nil { + log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err) + defaultRate = math.NaN() + } else if defaultRate < 0.0 || defaultRate > 1.0 { + log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", defaultRate) + defaultRate = math.NaN() + } + c.globalSampleRate = defaultRate c.httpClientTimeout = time.Second * 10 // 10 seconds + if v := os.Getenv("OTEL_LOGS_EXPORTER"); v != "" { + log.Warn("OTEL_LOGS_EXPORTER is not supported") + } if internal.BoolEnv("DD_TRACE_ANALYTICS_ENABLED", false) { globalconfig.SetAnalyticsRate(1.0) } @@ -325,7 +340,7 @@ func newConfig(opts ...StartOption) *config { return r == ',' || r == ' ' })...)(c) } - if v := os.Getenv("DD_SERVICE"); v != "" { + if v := getDDorOtelConfig("service"); v != "" { c.serviceName = v globalconfig.SetServiceName(v) } @@ -333,7 +348,7 @@ func newConfig(opts ...StartOption) *config { c.version = ver } if v := os.Getenv("DD_SERVICE_MAPPING"); v != "" { - internal.ForEachStringTag(v, func(key, val string) { WithServiceMapping(key, val)(c) }) + internal.ForEachStringTag(v, internal.DDTagsDelimiter, func(key, val string) { WithServiceMapping(key, val)(c) }) } c.headerAsTags = newDynamicConfig("trace_header_tags", nil, setHeaderTags, equalSlice[string]) if v := os.Getenv("DD_TRACE_HEADER_TAGS"); v != "" { @@ -341,7 +356,7 @@ func newConfig(opts ...StartOption) *config { // Required to ensure that the startup header tags are set on reset. c.headerAsTags.startup = c.headerAsTags.current } - if v := os.Getenv("DD_TAGS"); v != "" { + if v := getDDorOtelConfig("resourceAttributes"); v != "" { tags := internal.ParseTagString(v) internal.CleanGitMetadataTags(tags) for key, val := range tags { @@ -356,9 +371,9 @@ func newConfig(opts ...StartOption) *config { c.logToStdout = true } c.logStartup = internal.BoolEnv("DD_TRACE_STARTUP_LOGS", true) - c.runtimeMetrics = internal.BoolEnv("DD_RUNTIME_METRICS_ENABLED", false) - c.debug = internal.BoolEnv("DD_TRACE_DEBUG", false) - c.enabled = newDynamicConfig("tracing_enabled", internal.BoolEnv("DD_TRACE_ENABLED", true), func(b bool) bool { return true }, equal[bool]) + c.runtimeMetrics = internal.BoolVal(getDDorOtelConfig("metrics"), false) + c.debug = internal.BoolVal(getDDorOtelConfig("debugMode"), false) + c.enabled = newDynamicConfig("tracing_enabled", internal.BoolVal(getDDorOtelConfig("enabled"), true), func(b bool) bool { return true }, equal[bool]) if _, ok := os.LookupEnv("DD_TRACE_ENABLED"); ok { c.enabled.cfgOrigin = telemetry.OriginEnvVar } @@ -406,7 +421,7 @@ func newConfig(opts ...StartOption) *config { } c.peerServiceMappings = make(map[string]string) if v := os.Getenv("DD_TRACE_PEER_SERVICE_MAPPING"); v != "" { - internal.ForEachStringTag(v, func(key, val string) { c.peerServiceMappings[key] = val }) + internal.ForEachStringTag(v, internal.DDTagsDelimiter, func(key, val string) { c.peerServiceMappings[key] = val }) } for _, fn := range opts { diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 55d7a65487..51efde3363 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -399,6 +399,7 @@ func TestTracerOptionsDefaults(t *testing.T) { assert.Equal(x.Timeout, y.Timeout) compareHTTPClients(t, x, y) assert.True(getFuncName(x.Transport.(*http.Transport).DialContext) == getFuncName(defaultDialer.DialContext)) + assert.False(c.debug) }) t.Run("http-client", func(t *testing.T) { @@ -443,6 +444,45 @@ func TestTracerOptionsDefaults(t *testing.T) { }) }) + t.Run("debug", func(t *testing.T) { + t.Run("option", func(t *testing.T) { + tracer := newTracer(WithDebugMode(true)) + defer tracer.Stop() + c := tracer.config + assert.True(t, c.debug) + }) + t.Run("env", func(t *testing.T) { + t.Setenv("DD_TRACE_DEBUG", "true") + c := newConfig() + assert.True(t, c.debug) + }) + t.Run("otel-env-debug", func(t *testing.T) { + t.Setenv("OTEL_LOG_LEVEL", "debug") + c := newConfig() + assert.True(t, c.debug) + }) + t.Run("otel-env-notdebug", func(t *testing.T) { + // any value other than debug, does nothing + t.Setenv("OTEL_LOG_LEVEL", "notdebug") + c := newConfig() + assert.False(t, c.debug) + }) + t.Run("override-chain", func(t *testing.T) { + assert := assert.New(t) + // option override otel + t.Setenv("OTEL_LOG_LEVEL", "debug") + c := newConfig(WithDebugMode(false)) + assert.False(c.debug) + // env override otel + t.Setenv("DD_TRACE_DEBUG", "false") + c = newConfig() + assert.False(c.debug) + // option override env + c = newConfig(WithDebugMode(true)) + assert.True(c.debug) + }) + }) + t.Run("dogstatsd", func(t *testing.T) { t.Run("default", func(t *testing.T) { tracer := newTracer(WithAgentTimeout(2)) @@ -884,6 +924,18 @@ func TestServiceName(t *testing.T) { assert.Equal("api-intake", globalconfig.ServiceName()) }) + t.Run("otel-env", func(t *testing.T) { + defer func() { + globalconfig.SetServiceName("") + }() + t.Setenv("OTEL_SERVICE_NAME", "api-intake") + assert := assert.New(t) + c := newConfig() + + assert.Equal("api-intake", c.serviceName) + assert.Equal("api-intake", globalconfig.ServiceName()) + }) + t.Run("WithGlobalTag", func(t *testing.T) { defer globalconfig.SetServiceName("") assert := assert.New(t) @@ -892,6 +944,18 @@ func TestServiceName(t *testing.T) { assert.Equal("api-intake", globalconfig.ServiceName()) }) + t.Run("OTEL_RESOURCE_ATTRIBUTES", func(t *testing.T) { + defer func() { + globalconfig.SetServiceName("") + }() + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=api-intake") + assert := assert.New(t) + c := newConfig() + + assert.Equal("api-intake", c.serviceName) + assert.Equal("api-intake", globalconfig.ServiceName()) + }) + t.Run("DD_TAGS", func(t *testing.T) { defer globalconfig.SetServiceName("") t.Setenv("DD_TAGS", "service:api-intake") @@ -903,12 +967,21 @@ func TestServiceName(t *testing.T) { }) t.Run("override-chain", func(t *testing.T) { + defer func() { + globalconfig.SetServiceName("") + }() assert := assert.New(t) globalconfig.SetServiceName("") c := newConfig() assert.Equal(c.serviceName, filepath.Base(os.Args[0])) assert.Equal("", globalconfig.ServiceName()) + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=testService6") + globalconfig.SetServiceName("") + c = newConfig() + assert.Equal(c.serviceName, "testService6") + assert.Equal("testService6", globalconfig.ServiceName()) + t.Setenv("DD_TAGS", "service:testService") globalconfig.SetServiceName("") c = newConfig() @@ -920,17 +993,22 @@ func TestServiceName(t *testing.T) { assert.Equal(c.serviceName, "testService2") assert.Equal("testService2", globalconfig.ServiceName()) - t.Setenv("DD_SERVICE", "testService3") + t.Setenv("OTEL_SERVICE_NAME", "testService3") globalconfig.SetServiceName("") c = newConfig(WithGlobalTag("service", "testService2")) assert.Equal(c.serviceName, "testService3") assert.Equal("testService3", globalconfig.ServiceName()) + t.Setenv("DD_SERVICE", "testService4") globalconfig.SetServiceName("") - c = newConfig(WithGlobalTag("service", "testService2"), WithService("testService4")) + c = newConfig(WithGlobalTag("service", "testService2")) assert.Equal(c.serviceName, "testService4") assert.Equal("testService4", globalconfig.ServiceName()) - defer globalconfig.SetServiceName("") + + globalconfig.SetServiceName("") + c = newConfig(WithGlobalTag("service", "testService2"), WithService("testService5")) + assert.Equal(c.serviceName, "testService5") + assert.Equal("testService5", globalconfig.ServiceName()) }) } @@ -947,6 +1025,17 @@ func TestStartWithLink(t *testing.T) { assert.Equal(span.SpanLinks[1].SpanID, uint64(4)) } +func TestOtelResourceAtttributes(t *testing.T) { + t.Run("max 10", func(t *testing.T) { + assert := assert.New(t) + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "tag1=val1,tag2=val2,tag3=val3,tag4=val4,tag5=val5,tag6=val6,tag7=val7,tag8=val8,tag9=val9,tag10=val10,tag11=val11,tag12=val12") + c := newConfig() + globalTags := c.globalTags.get() + // runtime-id tag is added automatically, so we expect runtime-id + our first 10 tags + assert.Len(globalTags, 11) + }) +} + func TestTagSeparators(t *testing.T) { assert := assert.New(t) @@ -1066,6 +1155,14 @@ func TestVersionConfig(t *testing.T) { assert.Equal("1.2.3", c.version) }) + t.Run("OTEL_RESOURCE_ATTRIBUTES", func(t *testing.T) { + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "service.version=1.2.3") + assert := assert.New(t) + c := newConfig() + + assert.Equal("1.2.3", c.version) + }) + t.Run("DD_TAGS", func(t *testing.T) { t.Setenv("DD_TAGS", "version:1.2.3") assert := assert.New(t) @@ -1079,6 +1176,10 @@ func TestVersionConfig(t *testing.T) { c := newConfig() assert.Equal(c.version, "") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "service.version=1.1.0") + c = newConfig() + assert.Equal("1.1.0", c.version) + t.Setenv("DD_TAGS", "version:1.1.1") c = newConfig() assert.Equal("1.1.1", c.version) @@ -1118,6 +1219,14 @@ func TestEnvConfig(t *testing.T) { assert.Equal("testing", c.env) }) + t.Run("OTEL_RESOURCE_ATTRIBUTES", func(t *testing.T) { + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=testing") + assert := assert.New(t) + c := newConfig() + + assert.Equal("testing", c.env) + }) + t.Run("DD_TAGS", func(t *testing.T) { t.Setenv("DD_TAGS", "env:testing") assert := assert.New(t) @@ -1131,6 +1240,10 @@ func TestEnvConfig(t *testing.T) { c := newConfig() assert.Equal(c.env, "") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=testing0") + c = newConfig() + assert.Equal("testing0", c.env) + t.Setenv("DD_TAGS", "env:testing1") c = newConfig() assert.Equal("testing1", c.env) @@ -1205,18 +1318,30 @@ func TestWithTraceEnabled(t *testing.T) { assert.False(c.enabled.current) }) - t.Run("env", func(t *testing.T) { + t.Run("otel-env", func(t *testing.T) { assert := assert.New(t) - t.Setenv("DD_TRACE_ENABLED", "false") + t.Setenv("OTEL_TRACES_EXPORTER", "none") c := newConfig() assert.False(c.enabled.current) }) - t.Run("env-override", func(t *testing.T) { + t.Run("dd-env", func(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_ENABLED", "false") - c := newConfig(WithTraceEnabled(true)) + c := newConfig() + assert.False(c.enabled.current) + }) + + t.Run("override-chain", func(t *testing.T) { + assert := assert.New(t) + // dd env overrides otel env + t.Setenv("OTEL_TRACES_EXPORTER", "none") + t.Setenv("DD_TRACE_ENABLED", "true") + c := newConfig() assert.True(c.enabled.current) + // tracer option overrides dd env + c = newConfig(WithTraceEnabled(false)) + assert.False(c.enabled.current) }) } diff --git a/ddtrace/tracer/otel_dd_mappings.go b/ddtrace/tracer/otel_dd_mappings.go new file mode 100644 index 0000000000..9af9b9478d --- /dev/null +++ b/ddtrace/tracer/otel_dd_mappings.go @@ -0,0 +1,198 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. +package tracer + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/DataDog/dd-trace-go.v1/internal" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" +) + +// otelDDEnv contains env vars from both dd (DD) and ot (OTEL) that map to the same tracer configuration +// remapper contains functionality to remap OTEL values to DD values +type otelDDEnv struct { + dd string + ot string + remapper func(string) (string, error) +} + +var otelDDConfigs = map[string]*otelDDEnv{ + "service": { + dd: "DD_SERVICE", + ot: "OTEL_SERVICE_NAME", + remapper: mapService, + }, + "metrics": { + dd: "DD_RUNTIME_METRICS_ENABLED", + ot: "OTEL_METRICS_EXPORTER", + remapper: mapMetrics, + }, + "debugMode": { + dd: "DD_TRACE_DEBUG", + ot: "OTEL_LOG_LEVEL", + remapper: mapLogLevel, + }, + "enabled": { + dd: "DD_TRACE_ENABLED", + ot: "OTEL_TRACES_EXPORTER", + remapper: mapEnabled, + }, + "sampleRate": { + dd: "DD_TRACE_SAMPLE_RATE", + ot: "OTEL_TRACES_SAMPLER", + remapper: mapSampleRate, + }, + "propagationStyle": { + dd: "DD_TRACE_PROPAGATION_STYLE", + ot: "OTEL_PROPAGATORS", + remapper: mapPropagationStyle, + }, + "resourceAttributes": { + dd: "DD_TAGS", + ot: "OTEL_RESOURCE_ATTRIBUTES", + remapper: mapDDTags, + }, +} + +var ddTagsMapping = map[string]string{ + "service.name": "service", + "deployment.environment": "env", + "service.version": "version", +} + +var unsupportedSamplerMapping = map[string]string{ + "always_on": "parentbased_always_on", + "always_off": "parentbased_always_off", + "traceidratio": "parentbased_traceidratio", +} + +var propagationMapping = map[string]string{ + "tracecontext": "tracecontext", + "b3": "b3 single header", + "b3multi": "b3multi", + "datadog": "datadog", + "none": "none", +} + +// getDDorOtelConfig determines whether the provided otelDDOpt will be set via DD or OTEL env vars, and returns the value +func getDDorOtelConfig(configName string) string { + config, ok := otelDDConfigs[configName] + if !ok { + panic(fmt.Sprintf("Programming Error: %v not found in supported configurations", configName)) + } + + val := os.Getenv(config.dd) + if otVal := os.Getenv(config.ot); otVal != "" { + if val != "" { + log.Warn("Both %v and %v are set, using %v=%v", config.ot, config.dd, config.dd, val) + telemetry.GlobalClient.Count(telemetry.NamespaceTracers, "otel.env.hiding", 1.0, []string{config.dd, config.ot}, true) + } else { + v, err := config.remapper(otVal) + if err != nil { + log.Warn(err.Error()) + telemetry.GlobalClient.Count(telemetry.NamespaceTracers, "otel.env.invalid", 1.0, []string{config.dd, config.ot}, true) + } + val = v + } + } + return val +} + +// mapDDTags maps OTEL_RESOURCE_ATTRIBUTES to DD_TAGS +func mapDDTags(ot string) (string, error) { + ddTags := make([]string, 0) + internal.ForEachStringTag(ot, internal.OtelTagsDelimeter, func(key, val string) { + // replace otel delimiter with dd delimiter and normalize tag names + if ddkey, ok := ddTagsMapping[key]; ok { + // map reserved otel tag names to dd tag names + ddTags = append([]string{ddkey + internal.DDTagsDelimiter + val}, ddTags...) + } else { + ddTags = append(ddTags, key+internal.DDTagsDelimiter+val) + } + }) + + if len(ddTags) > 10 { + log.Warn("The following resource attributes have been dropped: %v. Only the first 10 resource attributes will be applied: %v", ddTags[10:], ddTags[:10]) + ddTags = ddTags[:10] + } + + return strings.Join(ddTags, ","), nil +} + +// mapService maps OTEL_SERVICE_NAME to DD_SERVICE +func mapService(ot string) (string, error) { + return ot, nil +} + +// mapMetrics maps OTEL_METRICS_EXPORTER to DD_RUNTIME_METRICS_ENABLED +func mapMetrics(ot string) (string, error) { + ot = strings.TrimSpace(strings.ToLower(ot)) + if ot == "none" { + return "false", nil + } + return "", fmt.Errorf("The following configuration is not supported: OTEL_METRICS_EXPORTER=%v", ot) +} + +// mapLogLevel maps OTEL_LOG_LEVEL to DD_TRACE_DEBUG +func mapLogLevel(ot string) (string, error) { + if strings.TrimSpace(strings.ToLower(ot)) == "debug" { + return "true", nil + } + return "", fmt.Errorf("The following configuration is not supported: OTEL_LOG_LEVEL=%v", ot) +} + +// mapEnabled maps OTEL_TRACES_EXPORTER to DD_TRACE_ENABLED +func mapEnabled(ot string) (string, error) { + if strings.TrimSpace(strings.ToLower(ot)) == "none" { + return "false", nil + } + return "", fmt.Errorf("The following configuration is not supported: OTEL_METRICS_EXPORTER=%v", ot) +} + +// mapSampleRate maps OTEL_TRACES_SAMPLER to DD_TRACE_SAMPLE_RATE +func otelTraceIDRatio() string { + if v := os.Getenv("OTEL_TRACES_SAMPLER_ARG"); v != "" { + return v + } + return "1.0" +} + +// mapSampleRate maps OTEL_TRACES_SAMPLER to DD_TRACE_SAMPLE_RATE +func mapSampleRate(ot string) (string, error) { + ot = strings.TrimSpace(strings.ToLower(ot)) + if v, ok := unsupportedSamplerMapping[ot]; ok { + log.Warn("The following configuration is not supported: OTEL_TRACES_SAMPLER=%v. %v will be used", ot, v) + ot = v + } + + var samplerMapping = map[string]string{ + "parentbased_always_on": "1.0", + "parentbased_always_off": "0.0", + "parentbased_traceidratio": otelTraceIDRatio(), + } + if v, ok := samplerMapping[ot]; ok { + return v, nil + } + return "", fmt.Errorf("unknown sampling configuration %v", ot) +} + +// mapPropagationStyle maps OTEL_PROPAGATORS to DD_TRACE_PROPAGATION_STYLE +func mapPropagationStyle(ot string) (string, error) { + ot = strings.TrimSpace(strings.ToLower(ot)) + supportedStyles := make([]string, 0) + for _, otStyle := range strings.Split(ot, ",") { + otStyle = strings.TrimSpace(otStyle) + if _, ok := propagationMapping[otStyle]; ok { + supportedStyles = append(supportedStyles, propagationMapping[otStyle]) + } else { + log.Warn("Invalid configuration: %v is not supported. This propagation style will be ignored.", otStyle) + } + } + return strings.Join(supportedStyles, ","), nil +} diff --git a/ddtrace/tracer/otel_dd_mappings_test.go b/ddtrace/tracer/otel_dd_mappings_test.go new file mode 100644 index 0000000000..8cbaa3318f --- /dev/null +++ b/ddtrace/tracer/otel_dd_mappings_test.go @@ -0,0 +1,49 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry/telemetrytest" +) + +func TestAssessSource(t *testing.T) { + t.Run("invalid", func(t *testing.T) { + assert.Panics(t, func() { getDDorOtelConfig("invalid") }, "invalid config should panic") + }) + + t.Run("dd", func(t *testing.T) { + t.Setenv("DD_SERVICE", "abc") + v := getDDorOtelConfig("service") + assert.Equal(t, "abc", v) + }) + t.Run("ot", func(t *testing.T) { + t.Setenv("OTEL_SERVICE_NAME", "abc") + v := getDDorOtelConfig("service") + assert.Equal(t, "abc", v) + }) + t.Run("both", func(t *testing.T) { + telemetryClient := new(telemetrytest.MockClient) + defer telemetry.MockGlobalClient(telemetryClient)() + // DD_SERVICE prevails + t.Setenv("DD_SERVICE", "abc") + t.Setenv("OTEL_SERVICE_NAME", "123") + v := getDDorOtelConfig("service") + assert.Equal(t, "abc", v) + telemetryClient.AssertCalled(t, "Count", telemetry.NamespaceTracers, "otel.env.hiding", 1.0, []string{"DD_SERVICE", "OTEL_SERVICE_NAME"}, true) + }) + t.Run("invalid-ot", func(t *testing.T) { + telemetryClient := new(telemetrytest.MockClient) + defer telemetry.MockGlobalClient(telemetryClient)() + t.Setenv("OTEL_LOG_LEVEL", "nonesense") + v := getDDorOtelConfig("debugMode") + assert.Equal(t, "", v) + telemetryClient.AssertCalled(t, "Count", telemetry.NamespaceTracers, "otel.env.invalid", 1.0, []string{"DD_TRACE_DEBUG", "OTEL_LOG_LEVEL"}, true) + }) +} diff --git a/ddtrace/tracer/rules_sampler.go b/ddtrace/tracer/rules_sampler.go index 6786c18473..a90bbb586d 100644 --- a/ddtrace/tracer/rules_sampler.go +++ b/ddtrace/tracer/rules_sampler.go @@ -380,26 +380,6 @@ func newTraceRulesSampler(rules []SamplingRule, traceSampleRate float64) *traceR } } -// globalSampleRate returns the sampling rate found in the DD_TRACE_SAMPLE_RATE environment variable. -// If it is invalid or not within the 0-1 range, NaN is returned. -func globalSampleRate() float64 { - defaultRate := math.NaN() - v := os.Getenv("DD_TRACE_SAMPLE_RATE") - if v == "" { - return defaultRate - } - r, err := strconv.ParseFloat(v, 64) - if err != nil { - log.Warn("ignoring DD_TRACE_SAMPLE_RATE: error: %v", err) - return defaultRate - } - if r >= 0.0 && r <= 1.0 { - return r - } - log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", r) - return defaultRate -} - func (rs *traceRulesSampler) enabled() bool { rs.m.RLock() defer rs.m.RUnlock() diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 4b8ab3fe2d..3af7cd19bd 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -187,7 +187,7 @@ func TestRateSamplerSetting(t *testing.T) { } func TestRuleEnvVars(t *testing.T) { - t.Run("sample-rate", func(t *testing.T) { + t.Run("dd-sample-rate", func(t *testing.T) { assert := assert.New(t) for _, tt := range []struct { in string @@ -201,7 +201,7 @@ func TestRuleEnvVars(t *testing.T) { {in: "1point0", out: math.NaN()}, // default if invalid value } { t.Setenv("DD_TRACE_SAMPLE_RATE", tt.in) - res := globalSampleRate() + res := newConfig().globalSampleRate if math.IsNaN(tt.out) { assert.True(math.IsNaN(res)) } else { @@ -210,6 +210,27 @@ func TestRuleEnvVars(t *testing.T) { } }) + t.Run("otel-sample-rate", func(t *testing.T) { + for _, tt := range []struct { + config string + rate float64 + }{ + {config: "parentbased_always_on", rate: 1.0}, + {config: "parentbased_always_off", rate: 0.0}, + {config: "parentbased_traceidratio", rate: 0.5}, + {config: "always_on", rate: 1.0}, + {config: "always_off", rate: 0.0}, + {config: "traceidratio", rate: 0.75}, + } { + t.Run(tt.config, func(t *testing.T) { + assert := assert.New(t) + t.Setenv("OTEL_TRACES_SAMPLER", tt.config) + t.Setenv("OTEL_TRACES_SAMPLER_ARG", fmt.Sprintf("%f", tt.rate)) + assert.Equal(tt.rate, newConfig().globalSampleRate) + }) + } + }) + t.Run("rate-limit", func(t *testing.T) { assert := assert.New(t) for _, tt := range []struct { @@ -456,7 +477,7 @@ func TestRulesSampler(t *testing.T) { } t.Run("no-rules", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(nil, nil, globalSampleRate()) + rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -521,7 +542,7 @@ func TestRulesSampler(t *testing.T) { assert.Nil(t, err) assert := assert.New(t) - rs := newRulesSampler(rules, nil, globalSampleRate()) + rs := newRulesSampler(rules, nil, newConfig().globalSampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.spanRsc, tt.spanTags) @@ -545,7 +566,7 @@ func TestRulesSampler(t *testing.T) { for _, v := range traceRules { t.Run("", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(v, nil, globalSampleRate()) + rs := newRulesSampler(v, nil, newConfig().globalSampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -571,7 +592,7 @@ func TestRulesSampler(t *testing.T) { for _, v := range traceRules { t.Run("", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(v, nil, globalSampleRate()) + rs := newRulesSampler(v, nil, newConfig().globalSampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -617,7 +638,7 @@ func TestRulesSampler(t *testing.T) { _, rules, err := samplingRulesFromEnv() assert.Nil(t, err) assert := assert.New(t) - rs := newRulesSampler(nil, rules, globalSampleRate()) + rs := newRulesSampler(nil, rules, newConfig().globalSampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30"}) @@ -740,7 +761,7 @@ func TestRulesSampler(t *testing.T) { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { assert := assert.New(t) c := newConfig(WithSamplingRules(tt.rules)) - rs := newRulesSampler(nil, c.spanRules, globalSampleRate()) + rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30", "tag": 20.1, @@ -808,7 +829,8 @@ func TestRulesSampler(t *testing.T) { _, rules, _ := samplingRulesFromEnv() assert := assert.New(t) - rs := newRulesSampler(nil, rules, globalSampleRate()) + sampleRate := newConfig().globalSampleRate + rs := newRulesSampler(nil, rules, sampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.resName, map[string]interface{}{"hostname": "hn-30"}) result := rs.SampleSpan(span) @@ -918,7 +940,7 @@ func TestRulesSampler(t *testing.T) { t.Run("", func(t *testing.T) { assert := assert.New(t) c := newConfig(WithSamplingRules(tt.rules)) - rs := newRulesSampler(nil, c.spanRules, globalSampleRate()) + rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30", "tag": 20.1, @@ -947,7 +969,7 @@ func TestRulesSampler(t *testing.T) { t.Run("", func(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) - rs := newRulesSampler(nil, rules, globalSampleRate()) + rs := newRulesSampler(nil, rules, newConfig().globalSampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -1228,7 +1250,7 @@ func TestRulesSamplerInternals(t *testing.T) { t.Run("full-rate", func(t *testing.T) { assert := assert.New(t) now := time.Now() - rs := newRulesSampler(nil, nil, globalSampleRate()) + rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) // set samplingLimiter to specific state rs.traces.limiter.prevTime = now.Add(-1 * time.Second) rs.traces.limiter.allowed = 1 @@ -1243,7 +1265,7 @@ func TestRulesSamplerInternals(t *testing.T) { t.Run("limited-rate", func(t *testing.T) { assert := assert.New(t) now := time.Now() - rs := newRulesSampler(nil, nil, globalSampleRate()) + rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) // force sampling limiter to 1.0 spans/sec rs.traces.limiter.limiter = rate.NewLimiter(rate.Limit(1.0), 1) rs.traces.limiter.prevTime = now.Add(-1 * time.Second) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 5f9dc5a8fa..25d77c6e1c 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -204,7 +204,7 @@ func getPropagators(cfg *PropagatorConfig, ps string) ([]Propagator, string) { defaultPsName += ",b3" } if ps == "" { - if prop := os.Getenv(headerPropagationStyle); prop != "" { + if prop := getDDorOtelConfig("propagationStyle"); prop != "" { ps = prop // use the generic DD_TRACE_PROPAGATION_STYLE if set } else { return defaultPs, defaultPsName // no env set, so use default from configuration diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index 6306cd12e6..7685d8f725 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -29,6 +29,8 @@ import ( "github.com/stretchr/testify/require" ) +const otelHeaderPropagationStyle = "OTEL_PROPAGATORS" + func traceIDFrom64Bits(i uint64) traceID { t := traceID{} t.SetLower(i) @@ -618,6 +620,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleInject: "b3"}, {headerPropagationStyleInjectDeprecated: "b3,none" /* none should have no affect */}, {headerPropagationStyle: "b3"}, + {otelHeaderPropagationStyle: "b3multi"}, {headerPropagationStyleInject: "b3multi", headerPropagationStyleInjectDeprecated: "none" /* none should have no affect */}, {headerPropagationStyleInject: "b3multi", headerPropagationStyle: "none" /* none should have no affect */}, } @@ -689,6 +692,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleExtract: "b3"}, {headerPropagationStyleExtractDeprecated: "b3"}, {headerPropagationStyle: "b3,none" /* none should have no affect */}, + {otelHeaderPropagationStyle: "b3multi"}, {headerPropagationStyleExtract: "b3multi", headerPropagationStyleExtractDeprecated: "none" /* none should have no affect */}, {headerPropagationStyleExtract: "b3multi", headerPropagationStyle: "none" /* none should have no affect */}, } @@ -755,6 +759,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleExtract: "b3"}, {headerPropagationStyleExtractDeprecated: "b3"}, {headerPropagationStyle: "b3,none" /* none should have no affect */}, + {otelHeaderPropagationStyle: "b3multi"}, {headerPropagationStyleExtract: "b3multi", headerPropagationStyleExtractDeprecated: "none" /* none should have no affect */}, {headerPropagationStyleExtract: "b3multi", headerPropagationStyle: "none" /* none should have no affect */}, } @@ -789,6 +794,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleExtract: "B3 single header"}, {headerPropagationStyleExtractDeprecated: "B3 single header"}, {headerPropagationStyle: "B3 single header,none" /* none should have no affect */}, + {otelHeaderPropagationStyle: "b3"}, } for _, testEnv := range testEnvs { for k, v := range testEnv { @@ -888,6 +894,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleInject: "datadog"}, {headerPropagationStyleInjectDeprecated: "datadog,none" /* none should have no affect */}, {headerPropagationStyle: "datadog"}, + {otelHeaderPropagationStyle: "datadog"}, {headerPropagationStyleInject: "datadog", headerPropagationStyleInjectDeprecated: "none" /* none should have no affect */}, {headerPropagationStyleInject: "datadog", headerPropagationStyle: "none" /* none should have no affect */}, } @@ -949,6 +956,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleExtractDeprecated: "Datadog,b3multi"}, {headerPropagationStyle: "Datadog,b3"}, {headerPropagationStyle: "none,Datadog,b3" /* none should have no affect */}, + {otelHeaderPropagationStyle: "Datadog,b3multi"}, } for _, testEnv := range testEnvs { for k, v := range testEnv { @@ -1024,6 +1032,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleInjectDeprecated: "datadog", headerPropagationStyleExtractDeprecated: "datadog"}, {headerPropagationStyleInject: "datadog", headerPropagationStyle: "datadog"}, {headerPropagationStyle: "datadog"}, + {otelHeaderPropagationStyle: "datadog"}, } for _, testEnv := range testEnvs { for k, v := range testEnv { @@ -1091,6 +1100,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleExtract: "traceContext"}, {headerPropagationStyleExtractDeprecated: "traceContext,none" /* none should have no affect */}, {headerPropagationStyle: "traceContext"}, + {otelHeaderPropagationStyle: "traceContext"}, {headerPropagationStyleExtract: "traceContext", headerPropagationStyleExtractDeprecated: "none" /* none should have no affect */}, {headerPropagationStyleExtract: "traceContext", headerPropagationStyle: "none" /* none should have no affect */}, } @@ -1340,6 +1350,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleExtract: "traceContext"}, {headerPropagationStyleExtractDeprecated: "traceContext,none" /* none should have no affect */}, {headerPropagationStyle: "traceContext"}, + {otelHeaderPropagationStyle: "traceContext"}, } for _, testEnv := range testEnvs { for k, v := range testEnv { @@ -1415,6 +1426,7 @@ func TestEnvVars(t *testing.T) { {headerPropagationStyleInjectDeprecated: "tracecontext", headerPropagationStyleExtractDeprecated: "tracecontext"}, {headerPropagationStyleInject: "datadog,tracecontext", headerPropagationStyle: "datadog,tracecontext"}, {headerPropagationStyle: "datadog,tracecontext"}, + {otelHeaderPropagationStyle: "datadog,traceContext"}, } for _, testEnv := range testEnvs { for k, v := range testEnv { @@ -2043,6 +2055,27 @@ func TestNonePropagator(t *testing.T) { _, err = tracer.Extract(headers) assert.Equal(err, ErrSpanContextNotFound) }) + t.Run("", func(t *testing.T) { + t.Setenv(otelHeaderPropagationStyle, "NoNe") + tracer := newTracer() + defer tracer.Stop() + root := tracer.StartSpan("web.request").(*span) + root.SetTag(ext.SamplingPriority, -1) + root.SetBaggageItem("item", "x") + ctx, ok := root.Context().(*spanContext) + ctx.traceID = traceIDFrom64Bits(1) + ctx.spanID = 1 + headers := TextMapCarrier(map[string]string{}) + err := tracer.Inject(ctx, headers) + + assert := assert.New(t) + assert.True(ok) + assert.Nil(err) + assert.Len(headers, 0) + + _, err = tracer.Extract(headers) + assert.Equal(err, ErrSpanContextNotFound) + }) t.Run("", func(t *testing.T) { //"DD_TRACE_PROPAGATION_STYLE_EXTRACT": "NoNe", // "DD_TRACE_PROPAGATION_STYLE_INJECT": "none", @@ -2074,6 +2107,45 @@ func assertTraceTags(t *testing.T, expected, actual string) { assert.ElementsMatch(t, strings.Split(expected, ","), strings.Split(actual, ",")) } +func TestOtelPropagator(t *testing.T) { + tests := []struct { + env string + result string + }{ + { + env: "tracecontext, b3", + result: "tracecontext,b3 single header", + }, + { + env: "b3multi , jaegar , datadog ", + result: "b3multi,datadog", + }, + { + env: "none", + result: "", + }, + { + env: "nonesense", + result: "datadog,tracecontext", + }, + { + env: "jaegar", + result: "datadog,tracecontext", + }, + } + for _, test := range tests { + t.Setenv(otelHeaderPropagationStyle, test.env) + t.Run(fmt.Sprintf("inject with %v=%v", otelHeaderPropagationStyle, test.env), func(t *testing.T) { + assert := assert.New(t) + c := newConfig() + cp, ok := c.propagator.(*chainedPropagator) + assert.True(ok) + assert.Equal(test.result, cp.injectorNames) + assert.Equal(test.result, cp.extractorsNames) + }) + } +} + func BenchmarkInjectDatadog(b *testing.B) { b.Setenv(headerPropagationStyleInject, "datadog") tracer := newTracer() diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 93e07ec407..3f4a8f0f5f 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -250,14 +250,13 @@ func newUnstartedTracer(opts ...StartOption) *tracer { if spans != nil { c.spanRules = spans } - globalRate := globalSampleRate() - rulesSampler := newRulesSampler(c.traceRules, c.spanRules, globalRate) - c.traceSampleRate = newDynamicConfig("trace_sample_rate", globalRate, rulesSampler.traces.setGlobalSampleRate, equal[float64]) + rulesSampler := newRulesSampler(c.traceRules, c.spanRules, c.globalSampleRate) + c.traceSampleRate = newDynamicConfig("trace_sample_rate", c.globalSampleRate, rulesSampler.traces.setGlobalSampleRate, equal[float64]) // If globalSampleRate returns NaN, it means the environment variable was not set or valid. // We could always set the origin to "env_var" inconditionally, but then it wouldn't be possible // to distinguish between the case where the environment variable was not set and the case where // it default to NaN. - if !math.IsNaN(globalRate) { + if !math.IsNaN(c.globalSampleRate) { c.traceSampleRate.cfgOrigin = telemetry.OriginEnvVar } c.traceSampleRules = newDynamicConfig("trace_sample_rules", c.traceRules, diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index cec6b84cdb..392f152545 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -192,7 +192,7 @@ func TestTracerStart(t *testing.T) { internal.Testing = false }) - t.Run("tracing_not_enabled", func(t *testing.T) { + t.Run("dd_tracing_not_enabled", func(t *testing.T) { t.Setenv("DD_TRACE_ENABLED", "false") Start() defer Stop() @@ -204,6 +204,18 @@ func TestTracerStart(t *testing.T) { } }) + t.Run("otel_tracing_not_enabled", func(t *testing.T) { + t.Setenv("OTEL_TRACES_EXPORTER", "none") + Start() + defer Stop() + if _, ok := internal.GetGlobalTracer().(*tracer); ok { + t.Fail() + } + if _, ok := internal.GetGlobalTracer().(*internal.NoopTracer); !ok { + t.Fail() + } + }) + t.Run("deadlock/api", func(t *testing.T) { Stop() Stop() @@ -683,7 +695,7 @@ func TestTracerRuntimeMetrics(t *testing.T) { assert.Contains(t, tp.Logs()[0], "DEBUG: Runtime metrics enabled") }) - t.Run("env", func(t *testing.T) { + t.Run("dd-env", func(t *testing.T) { t.Setenv("DD_RUNTIME_METRICS_ENABLED", "true") tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) @@ -692,13 +704,22 @@ func TestTracerRuntimeMetrics(t *testing.T) { assert.Contains(t, tp.Logs()[0], "DEBUG: Runtime metrics enabled") }) - t.Run("overrideEnv", func(t *testing.T) { + t.Run("otel-env", func(t *testing.T) { + t.Setenv("OTEL_METRICS_EXPORTER", "none") + c := newConfig() + assert.False(t, c.runtimeMetrics) + }) + + t.Run("override-chain", func(t *testing.T) { + // dd env overrides otel env + t.Setenv("OTEL_METRICS_EXPORTER", "none") + t.Setenv("DD_RUNTIME_METRICS_ENABLED", "true") + c := newConfig() + assert.True(t, c.runtimeMetrics) + // tracer option overrides dd env t.Setenv("DD_RUNTIME_METRICS_ENABLED", "false") - tp := new(log.RecordLogger) - tp.Ignore("appsec: ", telemetry.LogPrefix) - tracer := newTracer(WithRuntimeMetrics(), WithLogger(tp), WithDebugMode(true)) - defer tracer.Stop() - assert.Contains(t, tp.Logs()[0], "DEBUG: Runtime metrics enabled") + c = newConfig(WithRuntimeMetrics()) + assert.True(t, c.runtimeMetrics) }) } diff --git a/internal/env.go b/internal/env.go index ae377ecbc3..62704b7792 100644 --- a/internal/env.go +++ b/internal/env.go @@ -59,10 +59,10 @@ func DurationEnv(key string, def time.Duration) time.Duration { return v } -// ForEachStringTag runs fn on every key:val pair encountered in str. -// str may contain multiple key:val pairs separated by either space -// or comma (but not a mixture of both). -func ForEachStringTag(str string, fn func(key string, val string)) { +// ForEachStringTag runs fn on every key val pair encountered in str. +// str may contain multiple key val pairs separated by either space +// or comma (but not a mixture of both), and each key val pair is separated by a delimiter. +func ForEachStringTag(str string, delimiter string, fn func(key string, val string)) { sep := " " if strings.Index(str, ",") > -1 { // falling back to comma as separator @@ -73,7 +73,7 @@ func ForEachStringTag(str string, fn func(key string, val string)) { if tag == "" { continue } - kv := strings.SplitN(tag, ":", 2) + kv := strings.SplitN(tag, delimiter, 2) key := strings.TrimSpace(kv[0]) if key == "" { continue @@ -89,7 +89,7 @@ func ForEachStringTag(str string, fn func(key string, val string)) { // ParseTagString returns tags parsed from string as map func ParseTagString(str string) map[string]string { res := make(map[string]string) - ForEachStringTag(str, func(key, val string) { res[key] = val }) + ForEachStringTag(str, DDTagsDelimiter, func(key, val string) { res[key] = val }) return res } @@ -107,3 +107,12 @@ func FloatEnv(key string, def float64) float64 { } return v } + +// BoolVal returns the parsed boolean value of string val, or def if not parseable +func BoolVal(val string, def bool) bool { + v, err := strconv.ParseBool(val) + if err != nil { + return def + } + return v +} diff --git a/internal/utils.go b/internal/utils.go index 98ca6b050a..bc9a12b4af 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -10,6 +10,12 @@ import ( "sync/atomic" ) +// OtelTagsDelimeter is the separator between key-val pairs for OTEL env vars +const OtelTagsDelimeter = "=" + +// DDTagsDelimiter is the separator between key-val pairs for DD env vars +const DDTagsDelimiter = ":" + // LockMap uses an RWMutex to synchronize map access to allow for concurrent access. // This should not be used for cases with heavy write load and performance concerns. type LockMap struct { From c593a5fed8379368247267b4191cdd6c92a3cda0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Fri, 31 May 2024 17:59:13 +0200 Subject: [PATCH 11/14] ddtrace/tracer: report trace and span sampling rules separately (#2718) --- ddtrace/tracer/log.go | 6 ++-- ddtrace/tracer/log_test.go | 10 +++---- ddtrace/tracer/remote_config.go | 13 ++++---- ddtrace/tracer/remote_config_test.go | 5 ++-- ddtrace/tracer/rules_sampler.go | 5 ---- ddtrace/tracer/sampler_test.go | 44 ++++++++++++++-------------- ddtrace/tracer/telemetry.go | 1 + ddtrace/tracer/telemetry_test.go | 36 ++++++++++++++++++++--- 8 files changed, 73 insertions(+), 47 deletions(-) diff --git a/ddtrace/tracer/log.go b/ddtrace/tracer/log.go index bf4b46b6d5..77139f0f43 100644 --- a/ddtrace/tracer/log.go +++ b/ddtrace/tracer/log.go @@ -37,7 +37,8 @@ type startupInfo struct { AnalyticsEnabled bool `json:"analytics_enabled"` // True if there is a global analytics rate set SampleRate string `json:"sample_rate"` // The default sampling rate for the rules sampler SampleRateLimit string `json:"sample_rate_limit"` // The rate limit configured with the rules sampler - SamplingRules []SamplingRule `json:"sampling_rules"` // Rules used by the rules sampler + TraceSamplingRules []SamplingRule `json:"trace_sampling_rules"` // Trace rules used by the rules sampler + SpanSamplingRules []SamplingRule `json:"span_sampling_rules"` // Span rules used by the rules sampler SamplingRulesError string `json:"sampling_rules_error"` // Any errors that occurred while parsing sampling rules ServiceMappings map[string]string `json:"service_mappings"` // Service Mappings Tags map[string]string `json:"tags"` // Global tags @@ -103,7 +104,8 @@ func logStartup(t *tracer) { AnalyticsEnabled: !math.IsNaN(globalconfig.AnalyticsRate()), SampleRate: fmt.Sprintf("%f", t.rulesSampling.traces.globalRate), SampleRateLimit: "disabled", - SamplingRules: append(t.config.traceRules, t.config.spanRules...), + TraceSamplingRules: t.config.traceRules, + SpanSamplingRules: t.config.spanRules, ServiceMappings: t.config.serviceMappings, Tags: tags, RuntimeMetricsEnabled: t.config.runtimeMetrics, diff --git a/ddtrace/tracer/log_test.go b/ddtrace/tracer/log_test.go index 7b3403868d..a26d902459 100644 --- a/ddtrace/tracer/log_test.go +++ b/ddtrace/tracer/log_test.go @@ -31,7 +31,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"health_metrics_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"DataStreams":((true)|(false)),"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","trace_sampling_rules":null,"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"health_metrics_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"DataStreams":((true)|(false)),"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[1]) }) t.Run("configured", func(t *testing.T) { @@ -63,7 +63,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"100","sampling_rules":\[{"service":"mysql","sample_rate":0\.75,"type":"1"}\],"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"health_metrics_enabled":true,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"DataStreams":false,"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":true,"metadata":{"version":"v1"}},"feature_flags":\["discovery"\]}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"100","trace_sampling_rules":\[{"service":"mysql","sample_rate":0\.75}\],"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"health_metrics_enabled":true,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"DataStreams":false,"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":true,"metadata":{"version":"v1"}},"feature_flags":\["discovery"\]}`, tp.Logs()[1]) }) t.Run("limit", func(t *testing.T) { @@ -93,7 +93,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"1000.001","sampling_rules":\[{"service":"mysql","sample_rate":0\.75,"type":"1"}\],"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"health_metrics_enabled":true,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"DataStreams":false,"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"configuredEnv","service":"configured.service","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":true,"analytics_enabled":true,"sample_rate":"0\.123000","sample_rate_limit":"1000.001","trace_sampling_rules":\[{"service":"mysql","sample_rate":0\.75}\],"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":{"initial_service":"new_service"},"tags":{"runtime-id":"[^"]*","tag":"value","tag2":"NaN"},"runtime_metrics_enabled":true,"health_metrics_enabled":true,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"2.3.4","architecture":"[^"]*","global_service":"configured.service","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"DataStreams":false,"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[1]) }) t.Run("errors", func(t *testing.T) { @@ -107,7 +107,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) require.Len(t, tp.Logs(), 2) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"100","sampling_rules":\[{"service":"some\.service","sample_rate":0\.234,"type":"1"}\],"sampling_rules_error":"\\n\\tat index 1: ignoring rule {Service:other.service Rate:2}: rate is out of \[0\.0, 1\.0] range","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"health_metrics_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"DataStreams":((true)|(false)),"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[1]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"Post .*","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"100","trace_sampling_rules":\[{"service":"some\.service","sample_rate":0\.234}\],"span_sampling_rules":null,"sampling_rules_error":"\\n\\tat index 1: ignoring rule {Service:other.service Rate:2}: rate is out of \[0\.0, 1\.0] range","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"health_metrics_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"false","appsec":((true)|(false)),"agent_features":{"DropP0s":((true)|(false)),"Stats":((true)|(false)),"DataStreams":((true)|(false)),"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[1]) }) t.Run("lambda", func(t *testing.T) { @@ -120,7 +120,7 @@ func TestStartupLog(t *testing.T) { tp.Ignore("appsec: ", telemetry.LogPrefix) logStartup(tracer) assert.Len(tp.Logs(), 1) - assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"health_metrics_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"true","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"DataStreams":false,"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[0]) + assert.Regexp(logPrefixRegexp+` INFO: DATADOG TRACER CONFIGURATION {"date":"[^"]*","os_name":"[^"]*","os_version":"[^"]*","version":"[^"]*","lang":"Go","lang_version":"[^"]*","env":"","service":"tracer\.test(\.exe)?","agent_url":"http://localhost:9/v0.4/traces","agent_error":"","debug":false,"analytics_enabled":false,"sample_rate":"NaN","sample_rate_limit":"disabled","trace_sampling_rules":null,"span_sampling_rules":null,"sampling_rules_error":"","service_mappings":null,"tags":{"runtime-id":"[^"]*"},"runtime_metrics_enabled":false,"health_metrics_enabled":false,"profiler_code_hotspots_enabled":((false)|(true)),"profiler_endpoints_enabled":((false)|(true)),"dd_version":"","architecture":"[^"]*","global_service":"","lambda_mode":"true","appsec":((true)|(false)),"agent_features":{"DropP0s":false,"Stats":false,"DataStreams":false,"StatsdPort":0},"integrations":{.*},"partial_flush_enabled":false,"partial_flush_min_spans":1000,"orchestrion":{"enabled":false},"feature_flags":\[\]}`, tp.Logs()[0]) }) t.Run("integrations", func(t *testing.T) { diff --git a/ddtrace/tracer/remote_config.go b/ddtrace/tracer/remote_config.go index 5ac04fa136..bc317d6b85 100644 --- a/ddtrace/tracer/remote_config.go +++ b/ddtrace/tracer/remote_config.go @@ -33,11 +33,11 @@ type target struct { } type libConfig struct { - Enabled *bool `json:"tracing_enabled,omitempty"` - SamplingRate *float64 `json:"tracing_sampling_rate,omitempty"` - SamplingRules *[]rcSamplingRule `json:"tracing_sampling_rules,omitempty"` - HeaderTags *headerTags `json:"tracing_header_tags,omitempty"` - Tags *tags `json:"tracing_tags,omitempty"` + Enabled *bool `json:"tracing_enabled,omitempty"` + SamplingRate *float64 `json:"tracing_sampling_rate,omitempty"` + TraceSamplingRules *[]rcSamplingRule `json:"tracing_sampling_rules,omitempty"` + HeaderTags *headerTags `json:"tracing_header_tags,omitempty"` + Tags *tags `json:"tracing_tags,omitempty"` } type rcTag struct { @@ -93,7 +93,6 @@ func convertRemoteSamplingRules(rules *[]rcSamplingRule) *[]SamplingRule { Provenance: rule.Provenance, globRule: &jsonRule{Name: rule.Name, Service: rule.Service, Resource: rule.Resource}, } - convertedRules = append(convertedRules, x) } } @@ -220,7 +219,7 @@ func (t *tracer) onRemoteConfigUpdate(u remoteconfig.ProductUpdate) map[string]s if updated { telemConfigs = append(telemConfigs, t.config.traceSampleRate.toTelemetry()) } - updated = t.config.traceSampleRules.handleRC(convertRemoteSamplingRules(c.LibConfig.SamplingRules)) + updated = t.config.traceSampleRules.handleRC(convertRemoteSamplingRules(c.LibConfig.TraceSamplingRules)) if updated { telemConfigs = append(telemConfigs, t.config.traceSampleRules.toTelemetry()) } diff --git a/ddtrace/tracer/remote_config_test.go b/ddtrace/tracer/remote_config_test.go index 3e5e4d881e..d64c2666fe 100644 --- a/ddtrace/tracer/remote_config_test.go +++ b/ddtrace/tracer/remote_config_test.go @@ -199,7 +199,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { Name: "trace_sample_rules", Value: `[{"service":"my-service","name":"web.request","resource":"abc","sample_rate":1,"provenance":"customer"}]`, Origin: telemetry.OriginRemoteConfig, - }}) + }, + }) }) t.Run("DD_TRACE_SAMPLING_RULES=0.1 and RC rule rate=1.0 and revert", func(t *testing.T) { @@ -279,7 +280,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { {Name: "trace_sample_rate", Value: nil, Origin: telemetry.OriginDefault}, { Name: "trace_sample_rules", - Value: `[{"service":"my-service","name":"web.request","resource":"*","sample_rate":0.1,"type":"1"}]`, + Value: `[{"service":"my-service","name":"web.request","resource":"*","sample_rate":0.1}]`, Origin: telemetry.OriginDefault, }, }) diff --git a/ddtrace/tracer/rules_sampler.go b/ddtrace/tracer/rules_sampler.go index a90bbb586d..d0b61f8454 100644 --- a/ddtrace/tracer/rules_sampler.go +++ b/ddtrace/tracer/rules_sampler.go @@ -854,7 +854,6 @@ func (sr SamplingRule) MarshalJSON() ([]byte, error) { Resource string `json:"resource,omitempty"` Rate float64 `json:"sample_rate"` Tags map[string]string `json:"tags,omitempty"` - Type *string `json:"type,omitempty"` MaxPerSecond *float64 `json:"max_per_second,omitempty"` Provenance string `json:"provenance,omitempty"` }{} @@ -884,10 +883,6 @@ func (sr SamplingRule) MarshalJSON() ([]byte, error) { s.MaxPerSecond = &sr.MaxPerSecond } s.Rate = sr.Rate - if v := sr.ruleType.String(); v != "" { - t := fmt.Sprintf("%d", sr.ruleType) - s.Type = &t - } if sr.Provenance != Local { s.Provenance = sr.Provenance.String() } diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 3af7cd19bd..b28a040037 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -1505,15 +1505,15 @@ func TestSamplingRuleMarshall(t *testing.T) { in SamplingRule out string }{ - {ServiceRule("srv.*", 0), `{"service":"srv.*","sample_rate":0,"type":"1"}`}, - {NameServiceRule("ops.*", "srv.*", 0), `{"service":"srv.*","name":"ops.*","sample_rate":0,"type":"1"}`}, - {NameServiceRule("ops.*", "srv.*", 0.55), `{"service":"srv.*","name":"ops.*","sample_rate":0.55,"type":"1"}`}, - {TagsResourceRule(nil, "http_get", "", "", 0.55), `{"resource":"http_get","sample_rate":0.55,"type":"1"}`}, - {TagsResourceRule(map[string]string{"host": "hn-*"}, "http_get", "", "", 0.35), `{"resource":"http_get","sample_rate":0.35,"tags":{"host":"hn-*"},"type":"1"}`}, - {SpanNameServiceRule("ops.*", "srv.*", 0.55), `{"service":"srv.*","name":"ops.*","sample_rate":0.55,"type":"2"}`}, - {SpanNameServiceMPSRule("ops.*", "srv.*", 0.55, 1000), `{"service":"srv.*","name":"ops.*","sample_rate":0.55,"type":"2","max_per_second":1000}`}, - {TagsResourceRule(nil, "//bar", "", "", 1), `{"resource":"//bar","sample_rate":1,"type":"1"}`}, - {TagsResourceRule(map[string]string{"tag_key": "tag_value.*"}, "//bar", "", "", 1), `{"resource":"//bar","sample_rate":1,"tags":{"tag_key":"tag_value.*"},"type":"1"}`}, + {ServiceRule("srv.*", 0), `{"service":"srv.*","sample_rate":0}`}, + {NameServiceRule("ops.*", "srv.*", 0), `{"service":"srv.*","name":"ops.*","sample_rate":0}`}, + {NameServiceRule("ops.*", "srv.*", 0.55), `{"service":"srv.*","name":"ops.*","sample_rate":0.55}`}, + {TagsResourceRule(nil, "http_get", "", "", 0.55), `{"resource":"http_get","sample_rate":0.55}`}, + {TagsResourceRule(map[string]string{"host": "hn-*"}, "http_get", "", "", 0.35), `{"resource":"http_get","sample_rate":0.35,"tags":{"host":"hn-*"}}`}, + {SpanNameServiceRule("ops.*", "srv.*", 0.55), `{"service":"srv.*","name":"ops.*","sample_rate":0.55}`}, + {SpanNameServiceMPSRule("ops.*", "srv.*", 0.55, 1000), `{"service":"srv.*","name":"ops.*","sample_rate":0.55,"max_per_second":1000}`}, + {TagsResourceRule(nil, "//bar", "", "", 1), `{"resource":"//bar","sample_rate":1}`}, + {TagsResourceRule(map[string]string{"tag_key": "tag_value.*"}, "//bar", "", "", 1), `{"resource":"//bar","sample_rate":1,"tags":{"tag_key":"tag_value.*"}}`}, } { m, err := tt.in.MarshalJSON() assert.Nil(t, err) @@ -1529,22 +1529,22 @@ func TestSamplingRuleMarshallGlob(t *testing.T) { marshal string }{ // pattern with * - {"test*", "test", regexp.MustCompile("(?i)^test.*$"), `{"service":"test*","sample_rate":1,"type":"1"}`}, - {"*test", "a-test", regexp.MustCompile("(?i)^.*test$"), `{"service":"*test","sample_rate":1,"type":"1"}`}, - {"a*case", "acase", regexp.MustCompile("(?i)^a.*case$"), `{"service":"a*case","sample_rate":1,"type":"1"}`}, + {"test*", "test", regexp.MustCompile("(?i)^test.*$"), `{"service":"test*","sample_rate":1}`}, + {"*test", "a-test", regexp.MustCompile("(?i)^.*test$"), `{"service":"*test","sample_rate":1}`}, + {"a*case", "acase", regexp.MustCompile("(?i)^a.*case$"), `{"service":"a*case","sample_rate":1}`}, // pattern regexp.MustCompile(), ``, with ? - {"a?case", "a-case", regexp.MustCompile("(?i)^a.case$"), `{"service":"a?case","sample_rate":1,"type":"1"}`}, - {"a?test?case", "a-test-case", regexp.MustCompile("(?i)^a.test.case$"), `{"service":"a?test?case","sample_rate":1,"type":"1"}`}, + {"a?case", "a-case", regexp.MustCompile("(?i)^a.case$"), `{"service":"a?case","sample_rate":1}`}, + {"a?test?case", "a-test-case", regexp.MustCompile("(?i)^a.test.case$"), `{"service":"a?test?case","sample_rate":1}`}, //// pattern with ? regexp.MustCompile(), ``, and * - {"?test*", "atest", regexp.MustCompile("(?i)^.test.*$"), `{"service":"?test*","sample_rate":1,"type":"1"}`}, - {"test*case", "testcase", regexp.MustCompile("(?i)^test.*case$"), `{"service":"test*case","sample_rate":1,"type":"1"}`}, - {"a?test*", "a-test-case", regexp.MustCompile("(?i)^a.test.*$"), `{"service":"a?test*","sample_rate":1,"type":"1"}`}, - {"a*test?", "a-test-", regexp.MustCompile("(?i)^a.*test.$"), `{"service":"a*test?","sample_rate":1,"type":"1"}`}, - {"a*test?case", "a--test-case", regexp.MustCompile("(?i)^a.*test.case$"), `{"service":"a*test?case","sample_rate":1,"type":"1"}`}, - {"a?test*case", "a-testing--case", regexp.MustCompile("(?i)^a.test.*case$"), `{"service":"a?test*case","sample_rate":1,"type":"1"}`}, + {"?test*", "atest", regexp.MustCompile("(?i)^.test.*$"), `{"service":"?test*","sample_rate":1}`}, + {"test*case", "testcase", regexp.MustCompile("(?i)^test.*case$"), `{"service":"test*case","sample_rate":1}`}, + {"a?test*", "a-test-case", regexp.MustCompile("(?i)^a.test.*$"), `{"service":"a?test*","sample_rate":1}`}, + {"a*test?", "a-test-", regexp.MustCompile("(?i)^a.*test.$"), `{"service":"a*test?","sample_rate":1}`}, + {"a*test?case", "a--test-case", regexp.MustCompile("(?i)^a.*test.case$"), `{"service":"a*test?case","sample_rate":1}`}, + {"a?test*case", "a-testing--case", regexp.MustCompile("(?i)^a.test.*case$"), `{"service":"a?test*case","sample_rate":1}`}, //// valid non-glob regex regexp.MustCompile(), ``, pattern - {"*/*", `a/123`, regexp.MustCompile("(?i)^.*/.*$"), `{"service":"*/*","sample_rate":1,"type":"1"}`}, - {`*\/*`, `a\/123`, regexp.MustCompile("(?i)^.*/.*$"), `{"service":"*/*","sample_rate":1,"type":"1"}`}, + {"*/*", `a/123`, regexp.MustCompile("(?i)^.*/.*$"), `{"service":"*/*","sample_rate":1}`}, + {`*\/*`, `a\/123`, regexp.MustCompile("(?i)^.*/.*$"), `{"service":"*/*","sample_rate":1}`}, } { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { // the goal of this test is diff --git a/ddtrace/tracer/telemetry.go b/ddtrace/tracer/telemetry.go index 1f112ff830..7fec111383 100644 --- a/ddtrace/tracer/telemetry.go +++ b/ddtrace/tracer/telemetry.go @@ -59,6 +59,7 @@ func startTelemetry(c *config) { c.headerAsTags.toTelemetry(), c.globalTags.toTelemetry(), c.traceSampleRules.toTelemetry(), + telemetry.Sanitize(telemetry.Configuration{Name: "span_sample_rules", Value: c.spanRules}), } var peerServiceMapping []string for key, value := range c.peerServiceMappings { diff --git a/ddtrace/tracer/telemetry_test.go b/ddtrace/tracer/telemetry_test.go index 09e10cf638..9c3755b652 100644 --- a/ddtrace/tracer/telemetry_test.go +++ b/ddtrace/tracer/telemetry_test.go @@ -55,7 +55,8 @@ func TestTelemetryEnabled(t *testing.T) { telemetry.Check(t, telemetryClient.Configuration, "trace_sample_rate", nil) // default value is NaN which is sanitized to nil telemetry.Check(t, telemetryClient.Configuration, "trace_header_tags", "key:val,key2:val2") telemetry.Check(t, telemetryClient.Configuration, "trace_sample_rules", - `[{"service":"test-serv","name":"op-name","resource":"resource-*","sample_rate":0.1,"tags":{"tag-a":"tv-a??"},"type":"1"}]`) + `[{"service":"test-serv","name":"op-name","resource":"resource-*","sample_rate":0.1,"tags":{"tag-a":"tv-a??"}}]`) + telemetry.Check(t, telemetryClient.Configuration, "span_sample_rules", "[]") if metrics, ok := telemetryClient.Metrics[telemetry.NamespaceGeneral]; ok { if initTime, ok := metrics["init_time"]; ok { assert.True(t, initTime > 0) @@ -67,8 +68,7 @@ func TestTelemetryEnabled(t *testing.T) { }) t.Run("telemetry customer or dynamic rules", func(t *testing.T) { - rule := TagsResourceRule(map[string]string{"tag-a": "tv-a??"}, - "resource-*", "op-name", "test-serv", 0.1) + rule := TagsResourceRule(map[string]string{"tag-a": "tv-a??"}, "resource-*", "op-name", "test-serv", 0.1) for _, prov := range provenances { if prov == Local { @@ -86,15 +86,42 @@ func TestTelemetryEnabled(t *testing.T) { assert.True(t, telemetryClient.Started) telemetry.Check(t, telemetryClient.Configuration, "trace_sample_rules", - fmt.Sprintf(`[{"service":"test-serv","name":"op-name","resource":"resource-*","sample_rate":0.1,"tags":{"tag-a":"tv-a??"},"type":"1","provenance":"%s"}]`, prov.String())) + fmt.Sprintf(`[{"service":"test-serv","name":"op-name","resource":"resource-*","sample_rate":0.1,"tags":{"tag-a":"tv-a??"},"provenance":"%s"}]`, prov.String())) } }) + t.Run("telemetry local rules", func(t *testing.T) { + rules := []SamplingRule{ + TagsResourceRule(map[string]string{"tag-a": "tv-a??"}, "resource-*", "op-name", "test-serv", 0.1), + // Span rules can have only local provenance for now. + SpanNameServiceRule("op-name", "test-serv", 0.1), + } + + for i := range rules { + rules[i].Provenance = Local + } + + telemetryClient := new(telemetrytest.MockClient) + defer telemetry.MockGlobalClient(telemetryClient)() + Start(WithService("test-serv"), + WithSamplingRules(rules), + ) + defer globalconfig.SetServiceName("") + defer Stop() + + assert.True(t, telemetryClient.Started) + telemetry.Check(t, telemetryClient.Configuration, "trace_sample_rules", + `[{"service":"test-serv","name":"op-name","resource":"resource-*","sample_rate":0.1,"tags":{"tag-a":"tv-a??"}}]`) + telemetry.Check(t, telemetryClient.Configuration, "span_sample_rules", + `[{"service":"test-serv","name":"op-name","sample_rate":0.1}]`) + }) + t.Run("tracer start with empty rules", func(t *testing.T) { telemetryClient := new(telemetrytest.MockClient) defer telemetry.MockGlobalClient(telemetryClient)() t.Setenv("DD_TRACE_SAMPLING_RULES", "") + t.Setenv("DD_SPAN_SAMPLING_RULES", "") Start() defer globalconfig.SetServiceName("") defer Stop() @@ -105,6 +132,7 @@ func TestTelemetryEnabled(t *testing.T) { cfgs = append(cfgs, telemetry.Sanitize(c)) } telemetry.Check(t, cfgs, "trace_sample_rules", "[]") + telemetry.Check(t, cfgs, "span_sample_rules", "[]") }) t.Run("profiler start, tracer start", func(t *testing.T) { From 1b338dbe9645ee476cedf4eb29add6fc51f13b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Mon, 3 Jun 2024 11:33:01 +0200 Subject: [PATCH 12/14] go.mod: upgrade google.golang.org/protobuf to v1.33.0 (#2699) --- .../google.golang.org/grpc/fixtures_test.pb.go | 6 +++--- .../grpc/fixtures_test_grpc.pb.go | 4 ++-- contrib/google.golang.org/grpc/gen_proto.sh | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/apps/go.mod | 2 +- internal/apps/go.sum | 4 ++-- internal/exectracetest/go.mod | 2 +- internal/exectracetest/go.sum | 4 ++-- internal/traceprof/testapp/test_app.pb.go | 9 ++++----- internal/traceprof/testapp/test_app_grpc.pb.go | 18 +++++++++++++++--- 11 files changed, 34 insertions(+), 23 deletions(-) diff --git a/contrib/google.golang.org/grpc/fixtures_test.pb.go b/contrib/google.golang.org/grpc/fixtures_test.pb.go index d513208660..0ffa6edf3d 100644 --- a/contrib/google.golang.org/grpc/fixtures_test.pb.go +++ b/contrib/google.golang.org/grpc/fixtures_test.pb.go @@ -1,12 +1,12 @@ // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2023 Datadog, Inc. +// Copyright 2024 Datadog, Inc. // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc-gen-go v1.33.0 +// protoc v5.26.1 // source: fixtures_test.proto package grpc diff --git a/contrib/google.golang.org/grpc/fixtures_test_grpc.pb.go b/contrib/google.golang.org/grpc/fixtures_test_grpc.pb.go index 1ceb511fdc..95509e9025 100644 --- a/contrib/google.golang.org/grpc/fixtures_test_grpc.pb.go +++ b/contrib/google.golang.org/grpc/fixtures_test_grpc.pb.go @@ -1,12 +1,12 @@ // Unless explicitly stated otherwise all files in this repository are licensed // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2023 Datadog, Inc. +// Copyright 2024 Datadog, Inc. // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.4 +// - protoc v5.26.1 // source: fixtures_test.proto package grpc diff --git a/contrib/google.golang.org/grpc/gen_proto.sh b/contrib/google.golang.org/grpc/gen_proto.sh index 3affdf0783..42e611a8eb 100755 --- a/contrib/google.golang.org/grpc/gen_proto.sh +++ b/contrib/google.golang.org/grpc/gen_proto.sh @@ -7,7 +7,7 @@ COPYRIGHT_HEADER="// Unless explicitly stated otherwise all files in this reposi // Copyright ${YEAR} Datadog, Inc. " -go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 +go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.33.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 protoc fixtures_test.proto \ diff --git a/go.mod b/go.mod index 1c6daf57f5..0e1259f4ff 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 google.golang.org/api v0.128.0 google.golang.org/grpc v1.57.1 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/jinzhu/gorm.v1 v1.9.2 gopkg.in/olivere/elastic.v3 v3.0.75 gopkg.in/olivere/elastic.v5 v5.0.84 diff --git a/go.sum b/go.sum index 058eeea647..d669fd8906 100644 --- a/go.sum +++ b/go.sum @@ -2959,8 +2959,8 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 940661bf2a..9b4d0f10f2 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -50,7 +50,7 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) // use local version of dd-trace-go diff --git a/internal/apps/go.sum b/internal/apps/go.sum index 23c97dba0b..55ea6e5eed 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -178,8 +178,8 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNq google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod index f357481015..8a48dd3a42 100644 --- a/internal/exectracetest/go.mod +++ b/internal/exectracetest/go.mod @@ -33,7 +33,7 @@ require ( golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.21.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) // use local version of dd-trace-go diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index 3effef625d..b53091660e 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -136,8 +136,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/traceprof/testapp/test_app.pb.go b/internal/traceprof/testapp/test_app.pb.go index 06ba2cc07e..439bfe12fe 100644 --- a/internal/traceprof/testapp/test_app.pb.go +++ b/internal/traceprof/testapp/test_app.pb.go @@ -5,18 +5,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.33.0 +// protoc v5.26.1 // source: internal/traceprof/testapp/test_app.proto package testapp import ( - reflect "reflect" - sync "sync" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) const ( diff --git a/internal/traceprof/testapp/test_app_grpc.pb.go b/internal/traceprof/testapp/test_app_grpc.pb.go index 5b0cdce070..f2b6bcedc4 100644 --- a/internal/traceprof/testapp/test_app_grpc.pb.go +++ b/internal/traceprof/testapp/test_app_grpc.pb.go @@ -1,10 +1,18 @@ +// +//Run this generate Go code after changing things below: +// +//protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative internal/traceprof/testapp/test_app.proto + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v5.26.1 +// source: internal/traceprof/testapp/test_app.proto package testapp import ( context "context" - grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -15,6 +23,10 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TestApp_Work_FullMethodName = "/testapp.TestApp/Work" +) + // TestAppClient is the client API for TestApp service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -32,7 +44,7 @@ func NewTestAppClient(cc grpc.ClientConnInterface) TestAppClient { func (c *testAppClient) Work(ctx context.Context, in *WorkReq, opts ...grpc.CallOption) (*WorkRes, error) { out := new(WorkRes) - err := c.cc.Invoke(ctx, "/testapp.TestApp/Work", in, out, opts...) + err := c.cc.Invoke(ctx, TestApp_Work_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -77,7 +89,7 @@ func _TestApp_Work_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/testapp.TestApp/Work", + FullMethod: TestApp_Work_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TestAppServer).Work(ctx, req.(*WorkReq)) From 9298b8d33b557ccda3048fafc15edd0f2871fa26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Mon, 3 Jun 2024 16:26:24 +0200 Subject: [PATCH 13/14] go.mod: upgrade richardartoul/molecule (#2724) --- go.mod | 2 +- go.sum | 4 ++-- internal/apps/go.mod | 2 +- internal/apps/go.sum | 6 ++---- internal/exectracetest/go.sum | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0e1259f4ff..37ea06e17a 100644 --- a/go.mod +++ b/go.mod @@ -75,7 +75,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/opentracing/opentracing-go v1.2.0 github.com/redis/go-redis/v9 v9.1.0 - github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 + github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 github.com/segmentio/kafka-go v0.4.42 github.com/sirupsen/logrus v1.9.3 github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index d669fd8906..2f0c0289fa 100644 --- a/go.sum +++ b/go.sum @@ -1879,8 +1879,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqn github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= -github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 9b4d0f10f2..0c24d7944b 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -42,7 +42,7 @@ require ( github.com/google/uuid v1.5.0 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect + github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.8.4 diff --git a/internal/apps/go.sum b/internal/apps/go.sum index 55ea6e5eed..12ef104219 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -39,7 +39,6 @@ github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUk github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -83,8 +82,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= -github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= @@ -177,7 +176,6 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index b53091660e..f9168824e3 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -53,7 +53,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= From 6126fb6b6013a51716a38f43a4b6e3f992f04094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mazeau?= Date: Mon, 3 Jun 2024 16:50:27 +0200 Subject: [PATCH 14/14] appsec: support for SSRF Exploit Prevention (#2707) Co-authored-by Eliott Bouhana Co-authored-by: Julio Guerra --- appsec/events/block.go | 33 ++++ contrib/google.golang.org/grpc/appsec.go | 19 ++- contrib/labstack/echo.v4/appsec.go | 4 +- contrib/net/http/roundtripper.go | 13 +- contrib/net/http/roundtripper_test.go | 78 +++++++++ go.mod | 2 +- go.sum | 4 +- internal/apps/go.mod | 2 +- internal/apps/go.sum | 4 +- internal/appsec/appsec.go | 8 + internal/appsec/config/config.go | 4 +- .../appsec/emitter/grpcsec/types/types.go | 25 --- internal/appsec/emitter/httpsec/http.go | 21 +-- .../appsec/emitter/httpsec/roundtripper.go | 55 ++++++ .../appsec/emitter/httpsec/types/types.go | 32 ++-- internal/appsec/emitter/sharedsec/actions.go | 6 +- internal/appsec/emitter/sharedsec/shared.go | 3 +- .../appsec/listener/graphqlsec/graphql.go | 15 +- internal/appsec/listener/grpcsec/grpc.go | 42 +++-- internal/appsec/listener/httpsec/http.go | 27 +-- .../appsec/listener/httpsec/roundtripper.go | 32 ++++ internal/appsec/listener/sharedsec/shared.go | 54 +++--- internal/appsec/remoteconfig.go | 10 ++ internal/appsec/testdata/rasp.json | 158 ++++++++++++++++++ internal/appsec/waf_test.go | 4 +- internal/exectracetest/go.mod | 2 +- internal/exectracetest/go.sum | 4 +- internal/remoteconfig/remoteconfig.go | 2 + internal/stacktrace/event.go | 14 +- internal/stacktrace/event_test.go | 25 ++- 30 files changed, 553 insertions(+), 149 deletions(-) create mode 100644 appsec/events/block.go create mode 100644 internal/appsec/emitter/httpsec/roundtripper.go create mode 100644 internal/appsec/listener/httpsec/roundtripper.go create mode 100644 internal/appsec/testdata/rasp.json diff --git a/appsec/events/block.go b/appsec/events/block.go new file mode 100644 index 0000000000..8db51b8268 --- /dev/null +++ b/appsec/events/block.go @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2022 Datadog, Inc. + +// Package events provides security event types that appsec can return in function calls it monitors when blocking them. +// It allows finer-grained integrations of appsec into your Go errors' management logic. +package events + +import "errors" + +var _ error = (*BlockingSecurityEvent)(nil) + +var securityError = &BlockingSecurityEvent{} + +// BlockingSecurityEvent is the error type returned by function calls blocked by appsec. +// Even though appsec takes care of responding automatically to the blocked requests, it +// is your duty to abort the request handlers that are calling functions blocked by appsec. +// For instance, if a gRPC handler performs a SQL query blocked by appsec, the SQL query +// function call gets blocked and aborted by returning an error of type SecurityBlockingEvent. +// This allows you to safely abort your request handlers, and to be able to leverage errors.As if +// necessary in your Go error management logic to be able to tell if the error is a blocking security +// event or not (eg. to avoid retrying an HTTP client request). +type BlockingSecurityEvent struct{} + +func (*BlockingSecurityEvent) Error() string { + return "request blocked by WAF" +} + +// IsSecurityError returns true if the error is a security event. +func IsSecurityError(err error) bool { + return errors.Is(err, securityError) +} diff --git a/contrib/google.golang.org/grpc/appsec.go b/contrib/google.golang.org/grpc/appsec.go index 227beb136d..108d900838 100644 --- a/contrib/google.golang.org/grpc/appsec.go +++ b/contrib/google.golang.org/grpc/appsec.go @@ -62,11 +62,13 @@ func appsecUnaryHandlerMiddleware(method string, span ddtrace.Span, handler grpc return nil, err } defer grpcsec.StartReceiveOperation(types.ReceiveOperationArgs{}, op).Finish(types.ReceiveOperationRes{Message: req}) - rv, err := handler(ctx, req) - if e, ok := err.(*types.MonitoringError); ok { - err = status.Error(codes.Code(e.GRPCStatus()), e.Error()) + + rv, downstreamErr := handler(ctx, req) + if blocked { + return nil, err } - return rv, err + + return rv, downstreamErr } } @@ -113,11 +115,12 @@ func appsecStreamHandlerMiddleware(method string, span ddtrace.Span, handler grp return err } - err = handler(srv, stream) - if e, ok := err.(*types.MonitoringError); ok { - err = status.Error(codes.Code(e.GRPCStatus()), e.Error()) + downstreamErr := handler(srv, stream) + if blocked { + return err } - return err + + return downstreamErr } } diff --git a/contrib/labstack/echo.v4/appsec.go b/contrib/labstack/echo.v4/appsec.go index 2f820551cc..9cd849cc20 100644 --- a/contrib/labstack/echo.v4/appsec.go +++ b/contrib/labstack/echo.v4/appsec.go @@ -8,9 +8,9 @@ package echo import ( "net/http" + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/httpsec" - "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/httpsec/types" "github.com/labstack/echo/v4" ) @@ -27,7 +27,7 @@ func withAppSec(next echo.HandlerFunc, span tracer.Span) echo.HandlerFunc { err = next(c) // If the error is a monitoring one, it means appsec actions will take care of writing the response // and handling the error. Don't call the echo error handler in this case - if _, ok := err.(*types.MonitoringError); !ok && err != nil { + if _, ok := err.(*events.BlockingSecurityEvent); !ok && err != nil { c.Error(err) } }) diff --git a/contrib/net/http/roundtripper.go b/contrib/net/http/roundtripper.go index 20ba543ddd..bd71daa801 100644 --- a/contrib/net/http/roundtripper.go +++ b/contrib/net/http/roundtripper.go @@ -7,6 +7,7 @@ package http import ( "fmt" + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" "math" "net/http" "os" @@ -15,6 +16,8 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/httpsec" ) type roundTripper struct { @@ -57,7 +60,7 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err er if rt.cfg.after != nil { rt.cfg.after(res, span) } - if rt.cfg.errCheck == nil || rt.cfg.errCheck(err) { + if !events.IsSecurityError(err) && (rt.cfg.errCheck == nil || rt.cfg.errCheck(err)) { span.Finish(tracer.WithError(err)) } else { span.Finish() @@ -75,7 +78,15 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err er fmt.Fprintf(os.Stderr, "contrib/net/http.Roundtrip: failed to inject http headers: %v\n", err) } } + + if appsec.RASPEnabled() { + if err := httpsec.ProtectRoundTrip(ctx, r2.URL.String()); err != nil { + return nil, err + } + } + res, err = rt.base.RoundTrip(r2) + if err != nil { span.SetTag("http.errors", err.Error()) if rt.cfg.errCheck == nil || rt.cfg.errCheck(err) { diff --git a/contrib/net/http/roundtripper_test.go b/contrib/net/http/roundtripper_test.go index 5528799f19..dd7c9301be 100644 --- a/contrib/net/http/roundtripper_test.go +++ b/contrib/net/http/roundtripper_test.go @@ -16,11 +16,14 @@ import ( "testing" "time" + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" "gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/namingschematest" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/httpsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" "github.com/stretchr/testify/assert" @@ -619,3 +622,78 @@ func TestClientNamingSchema(t *testing.T) { t.Run("ServiceName", namingschematest.NewServiceNameTest(genSpans, wantServiceNameV0)) t.Run("SpanName", namingschematest.NewSpanNameTest(genSpans, assertOpV0, assertOpV1)) } + +type emptyRoundTripper struct{} + +func (rt *emptyRoundTripper) RoundTrip(_ *http.Request) (*http.Response, error) { + recorder := httptest.NewRecorder() + recorder.WriteHeader(200) + return recorder.Result(), nil +} + +func TestAppsec(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/rasp.json") + + client := WrapRoundTripper(&emptyRoundTripper{}) + + for _, enabled := range []bool{true, false} { + + t.Run(strconv.FormatBool(enabled), func(t *testing.T) { + t.Setenv("DD_APPSEC_RASP_ENABLED", strconv.FormatBool(enabled)) + + mt := mocktracer.Start() + defer mt.Stop() + + appsec.Start() + if !appsec.Enabled() { + t.Skip("appsec not enabled") + } + + defer appsec.Stop() + + w := httptest.NewRecorder() + r, err := http.NewRequest("GET", "?value=169.254.169.254", nil) + require.NoError(t, err) + + TraceAndServe(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req, err := http.NewRequest("GET", "http://169.254.169.254", nil) + require.NoError(t, err) + + resp, err := client.RoundTrip(req.WithContext(r.Context())) + + if enabled { + require.ErrorIs(t, err, &events.BlockingSecurityEvent{}) + } else { + require.NoError(t, err) + } + + if resp != nil { + defer resp.Body.Close() + } + }), w, r, &ServeConfig{ + Service: "service", + Resource: "resource", + }) + + spans := mt.FinishedSpans() + require.Len(t, spans, 2) // service entry serviceSpan & http request serviceSpan + serviceSpan := spans[1] + + if !enabled { + require.NotContains(t, serviceSpan.Tags(), "_dd.appsec.json") + require.NotContains(t, serviceSpan.Tags(), "_dd.stack") + return + } + + require.Contains(t, serviceSpan.Tags(), "_dd.appsec.json") + appsecJSON := serviceSpan.Tag("_dd.appsec.json") + require.Contains(t, appsecJSON, httpsec.ServerIoNetURLAddr) + + require.Contains(t, serviceSpan.Tags(), "_dd.stack") + + // This is a nested event so it should contain the child span id in the service entry span + // TODO(eliott.bouhana): uncomment this once we have the child span id in the service entry span + // require.Contains(t, appsecJSON, `"span_id":`+strconv.FormatUint(requestSpan.SpanID(), 10)) + }) + } +} diff --git a/go.mod b/go.mod index 37ea06e17a..badc5bb218 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( cloud.google.com/go/pubsub v1.33.0 github.com/99designs/gqlgen v0.17.36 - github.com/DataDog/appsec-internal-go v1.5.0 + github.com/DataDog/appsec-internal-go v1.6.0 github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 github.com/DataDog/datadog-go/v5 v5.3.0 diff --git a/go.sum b/go.sum index 2f0c0289fa..ca92373874 100644 --- a/go.sum +++ b/go.sum @@ -624,8 +624,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9s github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/appsec-internal-go v1.5.0 h1:8kS5zSx5T49uZ8dZTdT19QVAvC/B8ByyZdhQKYQWHno= -github.com/DataDog/appsec-internal-go v1.5.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= +github.com/DataDog/appsec-internal-go v1.6.0 h1:QHvPOv/O0s2fSI/BraZJNpRDAtdlrRm5APJFZNBxjAw= +github.com/DataDog/appsec-internal-go v1.6.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 0c24d7944b..558f1fd6ce 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -8,7 +8,7 @@ require ( ) require ( - github.com/DataDog/appsec-internal-go v1.5.0 // indirect + github.com/DataDog/appsec-internal-go v1.6.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect diff --git a/internal/apps/go.sum b/internal/apps/go.sum index 12ef104219..e5ce401622 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -1,5 +1,5 @@ -github.com/DataDog/appsec-internal-go v1.5.0 h1:8kS5zSx5T49uZ8dZTdT19QVAvC/B8ByyZdhQKYQWHno= -github.com/DataDog/appsec-internal-go v1.5.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= +github.com/DataDog/appsec-internal-go v1.6.0 h1:QHvPOv/O0s2fSI/BraZJNpRDAtdlrRm5APJFZNBxjAw= +github.com/DataDog/appsec-internal-go v1.6.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= diff --git a/internal/appsec/appsec.go b/internal/appsec/appsec.go index 17d7846870..ae68d0d668 100644 --- a/internal/appsec/appsec.go +++ b/internal/appsec/appsec.go @@ -26,6 +26,13 @@ func Enabled() bool { return activeAppSec != nil && activeAppSec.started } +// RASPEnabled returns true when DD_APPSEC_RASP_ENABLED=true or is unset. Granted that AppSec is enabled. +func RASPEnabled() bool { + mu.RLock() + defer mu.RUnlock() + return activeAppSec != nil && activeAppSec.started && activeAppSec.cfg.RASP +} + // Start AppSec when enabled is enabled by both using the appsec build tag and // setting the environment variable DD_APPSEC_ENABLED to true. func Start(opts ...config.StartOption) { @@ -162,6 +169,7 @@ func (a *appsec) start(telemetry *appsecTelemetry) error { } a.enableRCBlocking() + a.enableRASP() a.started = true log.Info("appsec: up and running") diff --git a/internal/appsec/config/config.go b/internal/appsec/config/config.go index 33ca45e56b..e2a0b7736a 100644 --- a/internal/appsec/config/config.go +++ b/internal/appsec/config/config.go @@ -66,7 +66,8 @@ type Config struct { // APISec configuration APISec internal.APISecConfig // RC is the remote configuration client used to receive product configuration updates. Nil if RC is disabled (default) - RC *remoteconfig.ClientConfig + RC *remoteconfig.ClientConfig + RASP bool } // WithRCConfig sets the AppSec remote config client configuration to the specified cfg @@ -115,5 +116,6 @@ func NewConfig() (*Config, error) { TraceRateLimit: int64(internal.RateLimitFromEnv()), Obfuscator: internal.NewObfuscatorConfig(), APISec: internal.NewAPISecConfig(), + RASP: internal.RASPEnabled(), }, nil } diff --git a/internal/appsec/emitter/grpcsec/types/types.go b/internal/appsec/emitter/grpcsec/types/types.go index 6e94ef9495..449ce2fbc3 100644 --- a/internal/appsec/emitter/grpcsec/types/types.go +++ b/internal/appsec/emitter/grpcsec/types/types.go @@ -72,33 +72,8 @@ type ( // Corresponds to the address `grpc.server.request.message`. Message interface{} } - - // MonitoringError is used to vehicle a gRPC error that also embeds a request status code - MonitoringError struct { - msg string - status uint32 - } ) -// NewMonitoringError creates and returns a new gRPC monitoring error, wrapped under -// sharedesec.MonitoringError -func NewMonitoringError(msg string, code uint32) error { - return &MonitoringError{ - msg: msg, - status: code, - } -} - -// GRPCStatus returns the gRPC status code embedded in the error -func (e *MonitoringError) GRPCStatus() uint32 { - return e.status -} - -// Error implements the error interface -func (e *MonitoringError) Error() string { - return e.msg -} - // Finish the gRPC handler operation, along with the given results, and emit a // finish event up in the operation stack. func (op *HandlerOperation) Finish(res HandlerOperationRes) []any { diff --git a/internal/appsec/emitter/httpsec/http.go b/internal/appsec/emitter/httpsec/http.go index 78d51e5cb8..2f7ec76e49 100644 --- a/internal/appsec/emitter/httpsec/http.go +++ b/internal/appsec/emitter/httpsec/http.go @@ -12,6 +12,7 @@ package httpsec import ( "context" + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" // Blank import needed to use embed for the default blocked response payloads _ "embed" @@ -26,6 +27,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/trace" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/trace/httptrace" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + "gopkg.in/DataDog/dd-trace-go.v1/internal/stacktrace" "github.com/DataDog/appsec-internal-go/netip" ) @@ -34,7 +36,7 @@ import ( // This function should not be called when AppSec is disabled in order to // get preciser error logs. func MonitorParsedBody(ctx context.Context, body any) error { - parent := fromContext(ctx) + parent, _ := ctx.Value(listener.ContextKey{}).(*types.Operation) if parent == nil { log.Error("appsec: parsed http body monitoring ignored: could not find the http handler instrumentation metadata in the request context: the request handler is not being monitored by a middleware function or the provided context is not the expected request context") return nil @@ -48,7 +50,7 @@ func MonitorParsedBody(ctx context.Context, body any) error { func ExecuteSDKBodyOperation(parent dyngo.Operation, args types.SDKBodyOperationArgs) error { var err error op := &types.SDKBodyOperation{Operation: dyngo.NewOperation(parent)} - dyngo.OnData(op, func(e error) { + dyngo.OnData(op, func(e *events.BlockingSecurityEvent) { err = e }) dyngo.StartOperation(op, args) @@ -78,6 +80,7 @@ func WrapHandler(handler http.Handler, span ddtrace.Span, pathParams map[string] var bypassHandler http.Handler var blocking bool + var stackTrace *stacktrace.Event args := MakeHandlerOperationArgs(r, clientIP, pathParams) ctx, op := StartOperation(r.Context(), args, func(op *types.Operation) { dyngo.OnData(op, func(a *sharedsec.HTTPAction) { @@ -85,7 +88,7 @@ func WrapHandler(handler http.Handler, span ddtrace.Span, pathParams map[string] bypassHandler = a.Handler }) dyngo.OnData(op, func(a *sharedsec.StackTraceAction) { - // TODO: do something with the stacktrace + stackTrace = &a.Event }) }) r = r.WithContext(ctx) @@ -102,6 +105,11 @@ func WrapHandler(handler http.Handler, span ddtrace.Span, pathParams map[string] } } + // Add stacktraces to the span, if any + if stackTrace != nil { + stacktrace.AddToSpan(span, stackTrace) + } + if bypassHandler != nil { bypassHandler.ServeHTTP(w, r) } @@ -194,10 +202,3 @@ func StartOperation(ctx context.Context, args types.HandlerOperationArgs, setup dyngo.StartOperation(op, args) return newCtx, op } - -// fromContext returns the Operation object stored in the context, if any -func fromContext(ctx context.Context) *types.Operation { - // Avoid a runtime panic in case of type-assertion error by collecting the 2 return values - op, _ := ctx.Value(listener.ContextKey{}).(*types.Operation) - return op -} diff --git a/internal/appsec/emitter/httpsec/roundtripper.go b/internal/appsec/emitter/httpsec/roundtripper.go new file mode 100644 index 0000000000..99e248fe6c --- /dev/null +++ b/internal/appsec/emitter/httpsec/roundtripper.go @@ -0,0 +1,55 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package httpsec + +import ( + "context" + "sync" + + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/httpsec/types" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" +) + +var badInputContextOnce sync.Once + +func ProtectRoundTrip(ctx context.Context, url string) error { + opArgs := types.RoundTripOperationArgs{ + URL: url, + } + + parent, _ := ctx.Value(listener.ContextKey{}).(dyngo.Operation) + if parent == nil { // No parent operation => we can't monitor the request + badInputContextOnce.Do(func() { + log.Debug("appsec: outgoing http request monitoring ignored: could not find the handler " + + "instrumentation metadata in the request context: the request handler is not being monitored by a " + + "middleware function or the incoming request context has not be forwarded correctly to the roundtripper") + }) + return nil + } + + op := &types.RoundTripOperation{ + Operation: dyngo.NewOperation(parent), + } + + var err *events.BlockingSecurityEvent + // TODO: move the data listener as a setup function of httpsec.StartRoundTripperOperation(ars, ) + dyngo.OnData(op, func(e *events.BlockingSecurityEvent) { + err = e + }) + + dyngo.StartOperation(op, opArgs) + dyngo.FinishOperation(op, types.RoundTripOperationRes{}) + + if err != nil { + log.Debug("appsec: outgoing http request blocked by the WAF on URL: %s", url) + return err + } + + return nil +} diff --git a/internal/appsec/emitter/httpsec/types/types.go b/internal/appsec/emitter/httpsec/types/types.go index 04e481c124..2ea8648b7c 100644 --- a/internal/appsec/emitter/httpsec/types/types.go +++ b/internal/appsec/emitter/httpsec/types/types.go @@ -27,6 +27,10 @@ type ( SDKBodyOperation struct { dyngo.Operation } + + RoundTripOperation struct { + dyngo.Operation + } ) // Finish the HTTP handler operation, along with the given results and emits a @@ -66,16 +70,20 @@ type ( // SDKBodyOperationArgs is the SDK body operation arguments. SDKBodyOperationArgs struct { // Body corresponds to the address `server.request.body`. - Body interface{} + Body any } // SDKBodyOperationRes is the SDK body operation results. SDKBodyOperationRes struct{} - // MonitoringError is used to vehicle an HTTP error, usually resurfaced through Appsec SDKs. - MonitoringError struct { - msg string + // RoundTripOperationArgs is the round trip operation arguments. + RoundTripOperationArgs struct { + // URL corresponds to the address `server.io.net.url`. + URL string } + + // RoundTripOperationRes is the round trip operation results. + RoundTripOperationRes struct{} ) // Finish finishes the SDKBody operation and emits a finish event @@ -83,21 +91,11 @@ func (op *SDKBodyOperation) Finish() { dyngo.FinishOperation(op, SDKBodyOperationRes{}) } -// Error implements the Error interface -func (e *MonitoringError) Error() string { - return e.msg -} - -// NewMonitoringError creates and returns a new HTTP monitoring error, wrapped under -// sharedesec.MonitoringError -func NewMonitoringError(msg string) error { - return &MonitoringError{ - msg: msg, - } -} - func (SDKBodyOperationArgs) IsArgOf(*SDKBodyOperation) {} func (SDKBodyOperationRes) IsResultOf(*SDKBodyOperation) {} func (HandlerOperationArgs) IsArgOf(*Operation) {} func (HandlerOperationRes) IsResultOf(*Operation) {} + +func (RoundTripOperationArgs) IsArgOf(*RoundTripOperation) {} +func (RoundTripOperationRes) IsResultOf(*RoundTripOperation) {} diff --git a/internal/appsec/emitter/sharedsec/actions.go b/internal/appsec/emitter/sharedsec/actions.go index abbab10a0f..0a9c0cd3b8 100644 --- a/internal/appsec/emitter/sharedsec/actions.go +++ b/internal/appsec/emitter/sharedsec/actions.go @@ -7,11 +7,11 @@ package sharedsec import ( _ "embed" // Blank import - "errors" "net/http" "os" "strings" + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/stacktrace" @@ -103,7 +103,7 @@ func (a *StackTraceAction) EmitData(op dyngo.Operation) { dyngo.EmitData(op, a) func NewStackTraceAction(params map[string]any) Action { id, ok := params["stack_id"] if !ok { - log.Debug("appsec: could not read stack_id parameter for stack_trace action") + log.Debug("appsec: could not read stack_id parameter for generate_stack action") return nil } @@ -197,7 +197,7 @@ func newBlockRequestHandler(status int, ct string, payload []byte) http.Handler func newGRPCBlockHandler(status int) GRPCWrapper { return func(_ map[string][]string) (uint32, error) { - return uint32(status), errors.New("Request blocked") + return uint32(status), &events.BlockingSecurityEvent{} } } diff --git a/internal/appsec/emitter/sharedsec/shared.go b/internal/appsec/emitter/sharedsec/shared.go index 715afc45cd..440fe47839 100644 --- a/internal/appsec/emitter/sharedsec/shared.go +++ b/internal/appsec/emitter/sharedsec/shared.go @@ -7,6 +7,7 @@ package sharedsec import ( "context" + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" "reflect" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" @@ -39,7 +40,7 @@ var userIDOperationArgsType = reflect.TypeOf((*UserIDOperationArgs)(nil)).Elem() func ExecuteUserIDOperation(parent dyngo.Operation, args UserIDOperationArgs) error { var err error op := &UserIDOperation{Operation: dyngo.NewOperation(parent)} - dyngo.OnData(op, func(e error) { err = e }) + dyngo.OnData(op, func(e *events.BlockingSecurityEvent) { err = e }) dyngo.StartOperation(op, args) dyngo.FinishOperation(op, UserIDOperationRes{}) return err diff --git a/internal/appsec/listener/graphqlsec/graphql.go b/internal/appsec/listener/graphqlsec/graphql.go index 03f82cac50..9a1ef48b2c 100644 --- a/internal/appsec/listener/graphqlsec/graphql.go +++ b/internal/appsec/listener/graphqlsec/graphql.go @@ -8,17 +8,19 @@ package graphqlsec import ( "sync" - "github.com/DataDog/appsec-internal-go/limiter" - waf "github.com/DataDog/go-libddwaf/v3" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/graphqlsec/types" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/httpsec" shared "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/sharedsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/trace" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames" + + "github.com/DataDog/appsec-internal-go/limiter" + waf "github.com/DataDog/go-libddwaf/v3" ) // GraphQL rule addresses currently supported by the WAF @@ -28,7 +30,8 @@ const ( // List of GraphQL rule addresses currently supported by the WAF var supportedAddresses = listener.AddressSet{ - graphQLServerResolverAddr: {}, + graphQLServerResolverAddr: {}, + httpsec.ServerIoNetURLAddr: {}, } // Install registers the GraphQL WAF Event Listener on the given root operation. @@ -83,6 +86,10 @@ func (l *wafEventListener) onEvent(request *types.RequestOperation, _ types.Requ return } + if _, ok := l.addresses[httpsec.ServerIoNetURLAddr]; ok { + httpsec.RegisterRoundTripperListener(request, &request.SecurityEventsHolder, wafCtx, l.limiter) + } + // Add span tags notifying this trace is AppSec-enabled trace.SetAppSecEnabledTags(request) l.once.Do(func() { @@ -101,7 +108,7 @@ func (l *wafEventListener) onEvent(request *types.RequestOperation, _ types.Requ }, }, ) - shared.AddSecurityEvents(field, l.limiter, wafResult.Events) + shared.AddSecurityEvents(&field.SecurityEventsHolder, l.limiter, wafResult.Events) } dyngo.OnFinish(field, func(field *types.ResolveOperation, res types.ResolveOperationRes) { diff --git a/internal/appsec/listener/grpcsec/grpc.go b/internal/appsec/listener/grpcsec/grpc.go index a258410766..3dc4e16154 100644 --- a/internal/appsec/listener/grpcsec/grpc.go +++ b/internal/appsec/listener/grpcsec/grpc.go @@ -10,9 +10,6 @@ import ( "go.uber.org/atomic" - "github.com/DataDog/appsec-internal-go/limiter" - waf "github.com/DataDog/go-libddwaf/v3" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" @@ -23,6 +20,9 @@ import ( shared "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/sharedsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames" + + "github.com/DataDog/appsec-internal-go/limiter" + waf "github.com/DataDog/go-libddwaf/v3" ) // gRPC rule addresses currently supported by the WAF @@ -30,8 +30,6 @@ const ( GRPCServerMethodAddr = "grpc.server.method" GRPCServerRequestMessageAddr = "grpc.server.request.message" GRPCServerRequestMetadataAddr = "grpc.server.request.metadata" - HTTPClientIPAddr = httpsec.HTTPClientIPAddr - UserIDAddr = httpsec.UserIDAddr ) // List of gRPC rule addresses currently supported by the WAF @@ -39,8 +37,9 @@ var supportedAddresses = listener.AddressSet{ GRPCServerMethodAddr: {}, GRPCServerRequestMessageAddr: {}, GRPCServerRequestMetadataAddr: {}, - HTTPClientIPAddr: {}, - UserIDAddr: {}, + httpsec.HTTPClientIPAddr: {}, + httpsec.UserIDAddr: {}, + httpsec.ServerIoNetURLAddr: {}, } // Install registers the gRPC WAF Event Listener on the given root operation. @@ -106,26 +105,23 @@ func (l *wafEventListener) onEvent(op *types.HandlerOperation, handlerArgs types return } + if _, ok := l.addresses[httpsec.ServerIoNetURLAddr]; ok { + httpsec.RegisterRoundTripperListener(op, &op.SecurityEventsHolder, wafCtx, l.limiter) + } + // Listen to the UserID address if the WAF rules are using it - if l.isSecAddressListened(UserIDAddr) { + if l.isSecAddressListened(httpsec.UserIDAddr) { // UserIDOperation happens when appsec.SetUser() is called. We run the WAF and apply actions to // see if the associated user should be blocked. Since we don't control the execution flow in this case // (SetUser is SDK), we delegate the responsibility of interrupting the handler to the user. dyngo.On(op, func(userIDOp *sharedsec.UserIDOperation, args sharedsec.UserIDOperationArgs) { values := map[string]any{ - UserIDAddr: args.UserID, + httpsec.UserIDAddr: args.UserID, } wafResult := shared.RunWAF(wafCtx, waf.RunAddressData{Persistent: values}) if wafResult.HasActions() || wafResult.HasEvents() { - for aType, params := range wafResult.Actions { - for _, action := range shared.ActionsFromEntry(aType, params) { - if grpcAction, ok := action.(*sharedsec.GRPCAction); ok { - code, err := grpcAction.GRPCWrapper(map[string][]string{}) - dyngo.EmitData(userIDOp, types.NewMonitoringError(err.Error(), code)) - } - } - } - shared.AddSecurityEvents(op, l.limiter, wafResult.Events) + shared.ProcessActions(userIDOp, wafResult.Actions) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, wafResult.Events) log.Debug("appsec: WAF detected an authenticated user attack: %s", args.UserID) } }) @@ -136,14 +132,14 @@ func (l *wafEventListener) onEvent(op *types.HandlerOperation, handlerArgs types // Note that this address is passed asap for the passlist, which are created per grpc method values[GRPCServerMethodAddr] = handlerArgs.Method } - if l.isSecAddressListened(HTTPClientIPAddr) && handlerArgs.ClientIP.IsValid() { - values[HTTPClientIPAddr] = handlerArgs.ClientIP.String() + if l.isSecAddressListened(httpsec.HTTPClientIPAddr) && handlerArgs.ClientIP.IsValid() { + values[httpsec.HTTPClientIPAddr] = handlerArgs.ClientIP.String() } wafResult := shared.RunWAF(wafCtx, waf.RunAddressData{Persistent: values}) if wafResult.HasActions() || wafResult.HasEvents() { - interrupt := shared.ProcessActions(op, wafResult.Actions, nil) - shared.AddSecurityEvents(op, l.limiter, wafResult.Events) + interrupt := shared.ProcessActions(op, wafResult.Actions) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, wafResult.Events) log.Debug("appsec: WAF detected an attack before executing the request") if interrupt { wafCtx.Close() @@ -199,7 +195,7 @@ func (l *wafEventListener) onEvent(op *types.HandlerOperation, handlerArgs types op.SetTag(ext.ManualKeep, samplernames.AppSec) }) - shared.AddSecurityEvents(op, l.limiter, events) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, events) }) } diff --git a/internal/appsec/listener/httpsec/http.go b/internal/appsec/listener/httpsec/http.go index 12d0038d9e..1d9c750e69 100644 --- a/internal/appsec/listener/httpsec/http.go +++ b/internal/appsec/listener/httpsec/http.go @@ -10,8 +10,6 @@ import ( "math/rand" "sync" - "github.com/DataDog/appsec-internal-go/limiter" - waf "github.com/DataDog/go-libddwaf/v3" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" @@ -21,6 +19,9 @@ import ( shared "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/sharedsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames" + + "github.com/DataDog/appsec-internal-go/limiter" + waf "github.com/DataDog/go-libddwaf/v3" ) // HTTP rule addresses currently supported by the WAF @@ -36,6 +37,7 @@ const ( ServerResponseHeadersNoCookiesAddr = "server.response.headers.no_cookies" HTTPClientIPAddr = "http.client_ip" UserIDAddr = "usr.id" + ServerIoNetURLAddr = "server.io.net.url" ) // List of HTTP rule addresses currently supported by the WAF @@ -51,6 +53,7 @@ var supportedAddresses = listener.AddressSet{ ServerResponseHeadersNoCookiesAddr: {}, HTTPClientIPAddr: {}, UserIDAddr: {}, + ServerIoNetURLAddr: {}, } // Install registers the HTTP WAF Event Listener on the given root operation. @@ -70,6 +73,7 @@ type wafEventListener struct { once sync.Once } +// newWAFEventListener returns the WAF event listener to register in order to enable it. func newWafEventListener(wafHandle *waf.Handle, cfg *config.Config, limiter limiter.Limiter) *wafEventListener { if wafHandle == nil { log.Debug("appsec: no WAF Handle available, the HTTP WAF Event Listener will not be registered") @@ -91,7 +95,6 @@ func newWafEventListener(wafHandle *waf.Handle, cfg *config.Config, limiter limi } } -// NewWAFEventListener returns the WAF event listener to register in order to enable it. func (l *wafEventListener) onEvent(op *types.Operation, args types.HandlerOperationArgs) { wafCtx, err := l.wafHandle.NewContextWithBudget(l.config.WAFTimeout) if err != nil { @@ -105,6 +108,10 @@ func (l *wafEventListener) onEvent(op *types.Operation, args types.HandlerOperat return } + if _, ok := l.addresses[ServerIoNetURLAddr]; ok { + RegisterRoundTripperListener(op, &op.SecurityEventsHolder, wafCtx, l.limiter) + } + if _, ok := l.addresses[UserIDAddr]; ok { // OnUserIDOperationStart happens when appsec.SetUser() is called. We run the WAF and apply actions to // see if the associated user should be blocked. Since we don't control the execution flow in this case @@ -112,8 +119,8 @@ func (l *wafEventListener) onEvent(op *types.Operation, args types.HandlerOperat dyngo.On(op, func(operation *sharedsec.UserIDOperation, args sharedsec.UserIDOperationArgs) { wafResult := shared.RunWAF(wafCtx, waf.RunAddressData{Persistent: map[string]any{UserIDAddr: args.UserID}}) if wafResult.HasActions() || wafResult.HasEvents() { - shared.ProcessActions(operation, wafResult.Actions, types.NewMonitoringError("Request blocked")) - shared.AddSecurityEvents(op, l.limiter, wafResult.Events) + shared.ProcessActions(operation, wafResult.Actions) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, wafResult.Events) log.Debug("appsec: WAF detected a suspicious user: %s", args.UserID) } }) @@ -159,8 +166,8 @@ func (l *wafEventListener) onEvent(op *types.Operation, args types.HandlerOperat op.AddSerializableTag(tag, value) } if wafResult.HasActions() || wafResult.HasEvents() { - interrupt := shared.ProcessActions(op, wafResult.Actions, nil) - shared.AddSecurityEvents(op, l.limiter, wafResult.Events) + interrupt := shared.ProcessActions(op, wafResult.Actions) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, wafResult.Events) log.Debug("appsec: WAF detected an attack before executing the request") if interrupt { wafCtx.Close() @@ -175,8 +182,8 @@ func (l *wafEventListener) onEvent(op *types.Operation, args types.HandlerOperat op.AddSerializableTag(tag, value) } if wafResult.HasActions() || wafResult.HasEvents() { - shared.ProcessActions(sdkBodyOp, wafResult.Actions, types.NewMonitoringError("Request blocked")) - shared.AddSecurityEvents(op, l.limiter, wafResult.Events) + shared.ProcessActions(sdkBodyOp, wafResult.Actions) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, wafResult.Events) log.Debug("appsec: WAF detected a suspicious request body") } }) @@ -211,7 +218,7 @@ func (l *wafEventListener) onEvent(op *types.Operation, args types.HandlerOperat // Log the attacks if any if wafResult.HasEvents() { log.Debug("appsec: attack detected by the waf") - shared.AddSecurityEvents(op, l.limiter, wafResult.Events) + shared.AddSecurityEvents(&op.SecurityEventsHolder, l.limiter, wafResult.Events) } for tag, value := range wafResult.Derivatives { op.AddSerializableTag(tag, value) diff --git a/internal/appsec/listener/httpsec/roundtripper.go b/internal/appsec/listener/httpsec/roundtripper.go new file mode 100644 index 0000000000..e631013120 --- /dev/null +++ b/internal/appsec/listener/httpsec/roundtripper.go @@ -0,0 +1,32 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package httpsec + +import ( + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/httpsec/types" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/sharedsec" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/trace" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + + "github.com/DataDog/appsec-internal-go/limiter" + "github.com/DataDog/go-libddwaf/v3" +) + +// RegisterRoundTripperListener registers a listener on outgoing HTTP client requests to run the WAF. +func RegisterRoundTripperListener(op dyngo.Operation, events *trace.SecurityEventsHolder, wafCtx *waf.Context, limiter limiter.Limiter) { + dyngo.On(op, func(op *types.RoundTripOperation, args types.RoundTripOperationArgs) { + wafResult := sharedsec.RunWAF(wafCtx, waf.RunAddressData{Persistent: map[string]any{ServerIoNetURLAddr: args.URL}}) + if !wafResult.HasEvents() { + return + } + + log.Debug("appsec: WAF detected a suspicious outgoing request URL: %s", args.URL) + + sharedsec.ProcessActions(op, wafResult.Actions) + sharedsec.AddSecurityEvents(events, limiter, wafResult.Events) + }) +} diff --git a/internal/appsec/listener/sharedsec/shared.go b/internal/appsec/listener/sharedsec/shared.go index d4773dbb1d..39f4353d41 100644 --- a/internal/appsec/listener/sharedsec/shared.go +++ b/internal/appsec/listener/sharedsec/shared.go @@ -7,43 +7,44 @@ package sharedsec import ( "encoding/json" - "github.com/DataDog/appsec-internal-go/limiter" - waf "github.com/DataDog/go-libddwaf/v3" + "errors" + + "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/sharedsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/trace" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + + "github.com/DataDog/appsec-internal-go/limiter" + waf "github.com/DataDog/go-libddwaf/v3" + wafErrors "github.com/DataDog/go-libddwaf/v3/errors" +) + +const ( + eventRulesVersionTag = "_dd.appsec.event_rules.version" + eventRulesErrorsTag = "_dd.appsec.event_rules.errors" + eventRulesLoadedTag = "_dd.appsec.event_rules.loaded" + eventRulesFailedTag = "_dd.appsec.event_rules.error_count" + wafVersionTag = "_dd.appsec.waf.version" ) func RunWAF(wafCtx *waf.Context, values waf.RunAddressData) waf.Result { result, err := wafCtx.Run(values) - if err == waf.ErrTimeout { - log.Debug("appsec: waf timeout value of reached: %v", err) + if errors.Is(err, wafErrors.ErrTimeout) { + log.Debug("appsec: waf timeout value reached: %v", err) } else if err != nil { log.Error("appsec: unexpected waf error: %v", err) } return result } -type securityEventsAdder interface { - AddSecurityEvents(events []any) -} - // AddSecurityEvents is a helper function to add sec events to an operation taking into account the rate limiter. -func AddSecurityEvents(op securityEventsAdder, limiter limiter.Limiter, matches []any) { +func AddSecurityEvents(holder *trace.SecurityEventsHolder, limiter limiter.Limiter, matches []any) { if len(matches) > 0 && limiter.Allow() { - op.AddSecurityEvents(matches) + holder.AddSecurityEvents(matches) } } -const ( - eventRulesVersionTag = "_dd.appsec.event_rules.version" - eventRulesErrorsTag = "_dd.appsec.event_rules.errors" - eventRulesLoadedTag = "_dd.appsec.event_rules.loaded" - eventRulesFailedTag = "_dd.appsec.event_rules.error_count" - wafVersionTag = "_dd.appsec.waf.version" -) - // AddRulesMonitoringTags adds the tags related to security rules monitoring func AddRulesMonitoringTags(th trace.TagSetter, wafDiags *waf.Diagnostics) { rInfo := wafDiags.Rules @@ -78,8 +79,9 @@ func AddWAFMonitoringTags(th trace.TagSetter, rulesVersion string, stats map[str // ProcessActions sends the relevant actions to the operation's data listener. // It returns true if at least one of those actions require interrupting the request handler // When SDKError is not nil, this error is sent to the op with EmitData so that the invoked SDK can return it -func ProcessActions(op dyngo.Operation, actions map[string]any, SDKError error) (interrupt bool) { +func ProcessActions(op dyngo.Operation, actions map[string]any) (interrupt bool) { for aType, params := range actions { + log.Debug("appsec: processing %s action with params %v", aType, params) actionArray := ActionsFromEntry(aType, params) if actionArray == nil { log.Debug("cannot process %s action with params %v", aType, params) @@ -87,12 +89,16 @@ func ProcessActions(op dyngo.Operation, actions map[string]any, SDKError error) } for _, a := range actionArray { a.EmitData(op) - if a.Blocking() && SDKError != nil { // Send the error to be returned by the SDK - interrupt = true - dyngo.EmitData(op, SDKError) // Send error - } + interrupt = interrupt || a.Blocking() } } + + // If any of the actions are supposed to interrupt the request, emit a blocking event for the SDK operations + // to return an error. + if interrupt { + dyngo.EmitData(op, &events.BlockingSecurityEvent{}) + } + return interrupt } @@ -108,7 +114,7 @@ func ActionsFromEntry(actionType string, params any) []sharedsec.Action { return sharedsec.NewBlockAction(p) case "redirect_request": return []sharedsec.Action{sharedsec.NewRedirectAction(p)} - case "stack_trace": + case "generate_stack": return []sharedsec.Action{sharedsec.NewStackTraceAction(p)} default: diff --git a/internal/appsec/remoteconfig.go b/internal/appsec/remoteconfig.go index ecaa60a2a4..9dddf65a8d 100644 --- a/internal/appsec/remoteconfig.go +++ b/internal/appsec/remoteconfig.go @@ -402,6 +402,16 @@ func (a *appsec) enableRCBlocking() { } } +func (a *appsec) enableRASP() { + if !a.cfg.RASP { + return + } + if err := remoteconfig.RegisterCapability(remoteconfig.ASMRASPSSRF); err != nil { + log.Debug("appsec: Remote config: couldn't register RASP SSRF: %v", err) + } + // TODO: register other RASP capabilities when supported +} + func (a *appsec) disableRCBlocking() { if a.cfg.RC == nil { return diff --git a/internal/appsec/testdata/rasp.json b/internal/appsec/testdata/rasp.json new file mode 100644 index 0000000000..ecbaf0d16f --- /dev/null +++ b/internal/appsec/testdata/rasp.json @@ -0,0 +1,158 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.4.2" + }, + "rules": [ + { + "id": "rasp-930-100", + "name": "Local file inclusion exploit", + "tags": { + "type": "lfi", + "category": "vulnerability_trigger", + "cwe": "22", + "capec": "1000/255/153/126", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.fs.file" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "lfi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace", + "block" + ] + }, + { + "id": "rasp-934-100", + "name": "Server-side request forgery exploit", + "tags": { + "type": "ssrf", + "category": "vulnerability_trigger", + "cwe": "918", + "capec": "1000/225/115/664", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.net.url" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "ssrf_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace", + "block" + ] + }, + { + "id": "rasp-942-100", + "name": "SQL injection exploit", + "tags": { + "type": "sql_injection", + "category": "vulnerability_trigger", + "cwe": "89", + "capec": "1000/152/248/66", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.db.statement" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ], + "db_type": [ + { + "address": "server.db.system" + } + ] + }, + "operator": "sqli_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace", + "block" + ] + } + ], + "rules_data": [] +} diff --git a/internal/appsec/waf_test.go b/internal/appsec/waf_test.go index e3fbf3d755..c2e3d7c954 100644 --- a/internal/appsec/waf_test.go +++ b/internal/appsec/waf_test.go @@ -15,8 +15,6 @@ import ( "strings" "testing" - internal "github.com/DataDog/appsec-internal-go/appsec" - waf "github.com/DataDog/go-libddwaf/v3" pAppsec "gopkg.in/DataDog/dd-trace-go.v1/appsec" httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" @@ -24,6 +22,8 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener/httpsec" + internal "github.com/DataDog/appsec-internal-go/appsec" + waf "github.com/DataDog/go-libddwaf/v3" "github.com/stretchr/testify/require" ) diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod index 8a48dd3a42..4ab15fa92a 100644 --- a/internal/exectracetest/go.mod +++ b/internal/exectracetest/go.mod @@ -9,7 +9,7 @@ require ( ) require ( - github.com/DataDog/appsec-internal-go v1.5.0 // indirect + github.com/DataDog/appsec-internal-go v1.6.0 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect github.com/DataDog/datadog-go/v5 v5.3.0 // indirect diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index f9168824e3..33151b0657 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -1,5 +1,5 @@ -github.com/DataDog/appsec-internal-go v1.5.0 h1:8kS5zSx5T49uZ8dZTdT19QVAvC/B8ByyZdhQKYQWHno= -github.com/DataDog/appsec-internal-go v1.5.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= +github.com/DataDog/appsec-internal-go v1.6.0 h1:QHvPOv/O0s2fSI/BraZJNpRDAtdlrRm5APJFZNBxjAw= +github.com/DataDog/appsec-internal-go v1.6.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= diff --git a/internal/remoteconfig/remoteconfig.go b/internal/remoteconfig/remoteconfig.go index 5c8f026392..86b8f69231 100644 --- a/internal/remoteconfig/remoteconfig.go +++ b/internal/remoteconfig/remoteconfig.go @@ -70,6 +70,8 @@ const ( APMTracingHTTPHeaderTags // APMTracingCustomTags enables APM client to set custom tags on all spans APMTracingCustomTags + // ASMRASPSSRF enables ASM support for runtime protection against SSRF attacks + ASMRASPSSRF = 23 ) // Additional capability bit index values that are non-consecutive from above. diff --git a/internal/stacktrace/event.go b/internal/stacktrace/event.go index feb1e2c581..ac292f13f6 100644 --- a/internal/stacktrace/event.go +++ b/internal/stacktrace/event.go @@ -88,14 +88,16 @@ func AddToSpan(span ddtrace.Span, events ...*Event) { return } - groupByCategory := map[EventCategory][]*Event{ - ExceptionEvent: {}, - VulnerabilityEvent: {}, - ExploitEvent: {}, - } + // TODO(eliott.bouhana): switch to a map[EventCategory][]*Event type when the tinylib/msgp@1.1.10 is out + groupByCategory := make(map[string]any, 3) for _, event := range events { - groupByCategory[event.Category] = append(groupByCategory[event.Category], event) + if _, ok := groupByCategory[string(event.Category)]; !ok { + groupByCategory[string(event.Category)] = []*Event{event} + continue + } + + groupByCategory[string(event.Category)] = append(groupByCategory[string(event.Category)].([]*Event), event) } type rooter interface { diff --git a/internal/stacktrace/event_test.go b/internal/stacktrace/event_test.go index 5da7febd9c..a95db53633 100644 --- a/internal/stacktrace/event_test.go +++ b/internal/stacktrace/event_test.go @@ -13,6 +13,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/internal" "github.com/stretchr/testify/require" + "github.com/tinylib/msgp/msgp" ) func TestNewEvent(t *testing.T) { @@ -38,11 +39,29 @@ func TestEventToSpan(t *testing.T) { require.Len(t, spans, 1) require.Equal(t, "op", spans[0].OperationName()) - eventsMap := spans[0].Tag("_dd.stack").(internal.MetaStructValue).Value.(map[EventCategory][]*Event) - require.Len(t, eventsMap, 3) + eventsMap := spans[0].Tag("_dd.stack").(internal.MetaStructValue).Value.(map[string]any) + require.Len(t, eventsMap, 1) - eventsCat := eventsMap[ExceptionEvent] + eventsCat := eventsMap[string(ExceptionEvent)].([]*Event) require.Len(t, eventsCat, 1) require.Equal(t, *event, *eventsCat[0]) } + +func TestMsgPackSerialization(t *testing.T) { + mt := mocktracer.Start() + defer mt.Stop() + + span := ddtracer.StartSpan("op") + event := NewEvent(ExceptionEvent, WithMessage("message"), WithType("type"), WithID("id")) + AddToSpan(span, event) + span.Finish() + + spans := mt.FinishedSpans() + require.Len(t, spans, 1) + + eventsMap := spans[0].Tag("_dd.stack").(internal.MetaStructValue).Value + + _, err := msgp.AppendIntf(nil, eventsMap) + require.NoError(t, err) +}