Merge branch 'main' into francois.mazeau/sqli

.github/workflows/appsec.yml

@@ -90,11 +90,11 @@ jobs:
 
  macos:
  name: ${{ matrix.runs-on }} go${{ matrix.go-version }}
- runs-on: macos-11 # oldest macos runner available - the full macOS matrix is in go-libddwaf
+ runs-on: ${{ matrix.runs-on }}
  needs: go-mod-caching
  strategy:
  matrix:
- runs-on: [ macos-11, macos-14 ] # oldest and newest macos runners available - macos-14 mainly is here to cover the fact it is an ARM machine
+ runs-on: [ macos-12, macos-14 ] # oldest and newest macos runners available - macos-14 mainly is here to cover the fact it is an ARM machine
  go-version: [ "1.22", "1.21", "1.20" ]
  fail-fast: true # saving some CI time - macos runners too long to get
  steps:

appsec/events/block.go

@@ -11,8 +11,6 @@ import "errors"
 
 var _ error = (*BlockingSecurityEvent)(nil)
 
-var securityError = &BlockingSecurityEvent{}
-
 // BlockingSecurityEvent is the error type returned by function calls blocked by appsec.
 // Even though appsec takes care of responding automatically to the blocked requests, it
 // is your duty to abort the request handlers that are calling functions blocked by appsec.
@@ -29,5 +27,6 @@ func (*BlockingSecurityEvent) Error() string {
 
 // IsSecurityError returns true if the error is a security event.
 func IsSecurityError(err error) bool {
- return errors.Is(err, securityError)
+ var secErr *BlockingSecurityEvent
+ return errors.As(err, &secErr)
 }

contrib/net/http/roundtripper_test.go

@@ -662,7 +662,7 @@ func TestAppsec(t *testing.T) {
  resp, err := client.RoundTrip(req.WithContext(r.Context()))
 
  if enabled {
- require.ErrorIs(t, err, &events.BlockingSecurityEvent{})
+ require.True(t, events.IsSecurityError(err))
  } else {
  require.NoError(t, err)
  }
@@ -690,6 +690,7 @@ func TestAppsec(t *testing.T) {
  require.Contains(t, appsecJSON, httpsec.ServerIoNetURLAddr)
 
  require.Contains(t, serviceSpan.Tags(), "_dd.stack")
+ require.NotContains(t, serviceSpan.Tags(), "error.message")
 
  // This is a nested event so it should contain the child span id in the service entry span
  // TODO(eliott.bouhana): uncomment this once we have the child span id in the service entry span

ddtrace/opentelemetry/span.go

@@ -212,6 +212,8 @@ func toReservedAttributes(k string, v attribute.Value) (string, interface{}) {
  rate = 0
  }
  return ext.EventSampleRate, rate
+ case "http.response.status_code":
+ return "http.status_code", strconv.FormatInt(v.AsInt64(), 10)
  default:
  return k, v.AsInterface()
  }

ddtrace/opentelemetry/span_test.go

@@ -725,6 +725,7 @@ func TestRemapWithMultipleSetAttributes(t *testing.T) {
  sp.SetAttributes(attribute.String("service.name", "new.service.name"))
  sp.SetAttributes(attribute.String("span.type", "new.span.type"))
  sp.SetAttributes(attribute.String("analytics.event", "true"))
+ sp.SetAttributes(attribute.Int("http.response.status_code", 200))
  sp.End()
 
  tracer.Flush()
@@ -739,4 +740,6 @@ func TestRemapWithMultipleSetAttributes(t *testing.T) {
  assert.Equal("new.span.type", p[0]["type"])
  metrics := fmt.Sprintf("%v", p[0]["metrics"])
  assert.Contains(metrics, fmt.Sprintf("%s:%s", "_dd1.sr.eausr", "1"))
+ meta := fmt.Sprintf("%v", p[0]["meta"])
+ assert.Contains(meta, fmt.Sprintf("%s:%s", "http.status_code", "200"))
 }

ddtrace/tracer/log_test.go

@@ -152,6 +152,18 @@ func TestLogSamplingRules(t *testing.T) {
  assert.Regexp(logPrefixRegexp+` WARN: DIAGNOSTICS Error\(s\) parsing sampling rules: found errors:\n\tat index 4: ignoring rule {Rate:9\.10}: rate is out of \[0\.0, 1\.0] range$`, tp.Logs()[0])
 }
 
+func TestLogDefaultSampleRate(t *testing.T) {
+ assert := assert.New(t)
+ tp := new(log.RecordLogger)
+ tp.Ignore("appsec: ", telemetry.LogPrefix)
+ log.UseLogger(tp)
+ t.Setenv("DD_TRACE_SAMPLE_RATE", ``)
+ _, _, _, stop := startTestTracer(t, WithLogger(tp))
+ defer stop()
+
+ assert.Len(tp.Logs(), 0)
+}
+
 func TestLogAgentReachable(t *testing.T) {
  assert := assert.New(t)
  tp := new(log.RecordLogger)

ddtrace/tracer/option.go

@@ -305,15 +305,19 @@ const partialFlushMinSpansDefault = 1000
 func newConfig(opts ...StartOption) *config {
  c := new(config)
  c.sampler = NewAllSampler()
- defaultRate, err := strconv.ParseFloat(getDDorOtelConfig("sampleRate"), 64)
- if err != nil {
- log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err)
- defaultRate = math.NaN()
- } else if defaultRate < 0.0 || defaultRate > 1.0 {
- log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", defaultRate)
- defaultRate = math.NaN()
+ sampleRate := math.NaN()
+ if r := getDDorOtelConfig("sampleRate"); r != "" {
+ var err error
+ sampleRate, err = strconv.ParseFloat(r, 64)
+ if err != nil {
+ log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err)
+ sampleRate = math.NaN()
+ } else if sampleRate < 0.0 || sampleRate > 1.0 {
+ log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", sampleRate)
+ sampleRate = math.NaN()
+ }
  }
- c.globalSampleRate = defaultRate
+ c.globalSampleRate = sampleRate
  c.httpClientTimeout = time.Second * 10 // 10 seconds
 
  if v := os.Getenv("OTEL_LOGS_EXPORTER"); v != "" {

go.mod

@@ -9,7 +9,7 @@ require (
  github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0
  github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1
  github.com/DataDog/datadog-go/v5 v5.3.0
- github.com/DataDog/go-libddwaf/v3 v3.2.0
+ github.com/DataDog/go-libddwaf/v3 v3.2.1
  github.com/DataDog/gostackparse v0.7.0
  github.com/DataDog/sketches-go v1.4.5
  github.com/IBM/sarama v1.40.0

go.sum

@@ -633,8 +633,8 @@ github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8=
 github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q=
-github.com/DataDog/go-libddwaf/v3 v3.2.0 h1:arvhB3A+TCQXT0eg4J9ksdIhuOO5b1OpHdu0EX8WHuc=
-github.com/DataDog/go-libddwaf/v3 v3.2.0/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY=
+github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8=
+github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY=
 github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I=
 github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
 github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=

internal/apps/go.mod

@@ -9,7 +9,7 @@ require (
 
 require (
  github.com/DataDog/appsec-internal-go v1.6.0 // indirect
- github.com/DataDog/go-libddwaf/v3 v3.2.0 // indirect
+ github.com/DataDog/go-libddwaf/v3 v3.2.1 // indirect
  github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
  github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect
  github.com/ebitengine/purego v0.6.0-alpha.5 // indirect

internal/apps/go.sum

@@ -6,8 +6,8 @@ github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3
 github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ=
 github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8=
 github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q=
-github.com/DataDog/go-libddwaf/v3 v3.2.0 h1:arvhB3A+TCQXT0eg4J9ksdIhuOO5b1OpHdu0EX8WHuc=
-github.com/DataDog/go-libddwaf/v3 v3.2.0/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY=
+github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8=
+github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY=
 github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I=
 github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
 github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=

internal/datastreams/pathway.go

@@ -17,25 +17,9 @@ import (
 
 var hashableEdgeTags = map[string]struct{}{"event_type": {}, "exchange": {}, "group": {}, "topic": {}, "type": {}, "direction": {}}
 
-func isValidArn(arn string) bool {
- separators := strings.Count(arn, ":")
- if separators < 5 {
- return false
- }
- if strings.HasPrefix(arn, "arn:") {
- return true
- }
- return false
-}
-
 func isWellFormedEdgeTag(t string) bool {
  if i := strings.IndexByte(t, ':'); i != -1 {
- if j := strings.LastIndexByte(t, ':'); j == i {
- if _, exists := hashableEdgeTags[t[:i]]; exists {
- return true
- }
- }
- if t[:i] == "topic" && isValidArn(t) {
+ if _, exists := hashableEdgeTags[t[:i]]; exists {
  return true
  }
  }

internal/datastreams/pathway_test.go

@@ -135,10 +135,11 @@ func TestPathway(t *testing.T) {
  {"dog:bark", false},
  {"type:", true},
  {"type:dog", true},
- {"type::dog", false},
- {"type:d:o:g", false},
- {"type::", false},
+ {"type::dog", true},
+ {"type:d:o:g", true},
+ {"type::", true},
  {":", false},
+ {"topic:arn:aws:sns:us-east-1:727006795293:dsm-dev-sns-topic", true},
  } {
  assert.Equal(t, isWellFormedEdgeTag(tc.s), tc.b)
  }

internal/exectracetest/go.mod

@@ -13,7 +13,7 @@ require (
  github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect
  github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect
  github.com/DataDog/datadog-go/v5 v5.3.0 // indirect
- github.com/DataDog/go-libddwaf/v3 v3.2.0 // indirect
+ github.com/DataDog/go-libddwaf/v3 v3.2.1 // indirect
  github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
  github.com/DataDog/sketches-go v1.4.5 // indirect
  github.com/Microsoft/go-winio v0.6.1 // indirect

internal/exectracetest/go.sum

@@ -6,8 +6,8 @@ github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3
 github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ=
 github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8=
 github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q=
-github.com/DataDog/go-libddwaf/v3 v3.2.0 h1:arvhB3A+TCQXT0eg4J9ksdIhuOO5b1OpHdu0EX8WHuc=
-github.com/DataDog/go-libddwaf/v3 v3.2.0/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY=
+github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8=
+github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY=
 github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I=
 github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
 github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=

internal/telemetry/client.go

@@ -211,6 +211,18 @@ func (c *client) start(configuration []Configuration, namespace Namespace, flush
  cfg = append(cfg, c.globalAppConfig...)
  cfg = append(cfg, configuration...)
 
+ // State whether the app has its Go dependencies available or not
+ deps, ok := debug.ReadBuildInfo()
+ if !ok {
+ deps = nil // because not guaranteed to be nil by the public doc when !ok
+ }
+ cfg = append(cfg, BoolConfig("dependencies_available", ok))
+ collectDependenciesEnabled := collectDependencies()
+ cfg = append(cfg, BoolConfig("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", collectDependenciesEnabled)) // TODO: report all the possible telemetry config option automatically
+ if !collectDependenciesEnabled {
+ deps = nil // to simplify the condition below to `deps != nil`
+ }
+
  payload := &AppStarted{
  Configuration: cfg,
  Products: productInfo,
@@ -219,18 +231,19 @@ func (c *client) start(configuration []Configuration, namespace Namespace, flush
  appStarted.Body.Payload = payload
  c.scheduleSubmit(appStarted)
 
- if collectDependencies() {
+ if deps != nil {
  var depPayload Dependencies
- if deps, ok := debug.ReadBuildInfo(); ok {
- for _, dep := range deps.Deps {
- depPayload.Dependencies = append(depPayload.Dependencies,
- Dependency{
- Name: dep.Path,
- Version: strings.TrimPrefix(dep.Version, "v"),
- },
- )
- }
+ for _, dep := range deps.Deps {
+ depPayload.Dependencies = append(depPayload.Dependencies,
+ Dependency{
+ Name: dep.Path,
+ Version: strings.TrimPrefix(dep.Version, "v"),
+ },
+ )
  }
+ // Send the telemetry request if and only if the dependencies are actually present in the binary.
+ // For instance, bazel doesn't include them out of the box (cf. https://github.com/bazelbuild/rules_go/issues/3090),
+ // which would result in an empty list of dependencies.
  dep := c.newRequest(RequestTypeDependenciesLoaded)
  dep.Body.Payload = depPayload
  c.scheduleSubmit(dep)

internal/telemetry/telemetry_test.go

@@ -151,7 +151,6 @@ func TestRegisterAppConfig(t *testing.T) {
  require.Equal(t, RequestTypeAppStarted, req.RequestType)
  appStarted := req.Payload.(*AppStarted)
  cfg := appStarted.Configuration
- require.Len(t, cfg, 2)
  require.Contains(t, cfg, Configuration{Name: "key1", Value: "val1", Origin: OriginDefault})
  require.Contains(t, cfg, Configuration{Name: "key2", Value: "val2", Origin: OriginDDConfig})
 

internal/version/version.go

@@ -13,7 +13,7 @@ import (
 // Tag specifies the current release tag. It needs to be manually
 // updated. A test checks that the value of Tag never points to a
 // git tag that is older than HEAD.
-const Tag = "v1.65.0-dev"
+const Tag = "v1.66.0-dev"
 
 // Dissected version number. Filled during init()
 var (