Merge branch 'master' into di-transport

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 ## [Unreleased]
 
+## [2.4.0] - 2024-10-11
+
+### Added
+
+* Core: Allow changing sampling rate for customer defined tags and resources ([#3956][])
+* Profiling: Add GVL profiling for Ruby 3.2+ as a preview feature ([#3929][])
+* Profiling: Otel: Add preview support for correlating profiling with otel ruby gem ([#3984][])
+* Tracing: AppSec: Add http.route tag to Rails, Grape, and Sinatra integrations ([#3849][])
+* Tracing: Add capabilities to remote config: tracing sample rate, tracing logs injection, tracing http header tags ([#3888][])
+* AppSec: Add a force disable of AppSec feature when using Ruby >= 3.3 with old FFI gem version ([#3969][])
+
+### Changed
+
+* AppSec: Improve PII compliance ([#3857][])
+* AppSec: Integrations: Improve accuracy of login tracking for Devise ([#3867][])
+* Crashtracking feature is now disabled by default ([#3970][])
+
+### Fixed
+
+* AppSec: Integrations: Fix GraphQL instrumentation for query fragments ([#3887][])
+* Bug: Profiling: Fix (small) memory leak in profiler when forking ([#3852][])
+* Tracing: Integrations: Fix GraphQL integration reconfiguration ([#3859][])
+
 ## [2.3.0] - 2024-08-22
 
 ### Added
@@ -2962,7 +2985,8 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.3.0...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.4.0...master
+[2.4.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.3.0...v2.4.0
 [2.3.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.2.0...v2.3.0
 [2.2.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.1.0...v2.2.0
 [2.1.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0...v2.1.0
@@ -4383,6 +4407,18 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3837]: https://github.com/DataDog/dd-trace-rb/issues/3837
 [#3839]: https://github.com/DataDog/dd-trace-rb/issues/3839
 [#3841]: https://github.com/DataDog/dd-trace-rb/issues/3841
+[#3849]: https://github.com/DataDog/dd-trace-rb/issues/3849
+[#3852]: https://github.com/DataDog/dd-trace-rb/issues/3852
+[#3857]: https://github.com/DataDog/dd-trace-rb/issues/3857
+[#3859]: https://github.com/DataDog/dd-trace-rb/issues/3859
+[#3867]: https://github.com/DataDog/dd-trace-rb/issues/3867
+[#3887]: https://github.com/DataDog/dd-trace-rb/issues/3887
+[#3888]: https://github.com/DataDog/dd-trace-rb/issues/3888
+[#3929]: https://github.com/DataDog/dd-trace-rb/issues/3929
+[#3956]: https://github.com/DataDog/dd-trace-rb/issues/3956
+[#3969]: https://github.com/DataDog/dd-trace-rb/issues/3969
+[#3970]: https://github.com/DataDog/dd-trace-rb/issues/3970
+[#3984]: https://github.com/DataDog/dd-trace-rb/issues/3984
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot

Matrixfile

@@ -14,8 +14,9 @@
  '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
  },
  'profiling:main' => {
- 'opentelemetry' => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
- '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+ 'opentelemetry' => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
+ 'opentelemetry_otlp' => '❌ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
+ '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
  },
  'profiling:ractors' => {
  '' => '❌ 2.5 / ❌ 2.6 / ❌ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'

Steepfile

@@ -70,7 +70,6 @@ target :datadog do
  ignore 'lib/datadog/core/metrics/metric.rb'
  ignore 'lib/datadog/core/metrics/options.rb'
  ignore 'lib/datadog/core/pin.rb'
- ignore 'lib/datadog/core/rate_limiter.rb'
  # steep fails in this file due to https://github.com/soutaro/steep/issues/1231
  ignore 'lib/datadog/core/remote/tie.rb'
  ignore 'lib/datadog/core/runtime/ext.rb'

appraisal/ruby-2.6.rb

@@ -232,6 +232,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 [3, 4, 5].each do |n|
  appraise "redis-#{n}" do
  gem 'redis', "~> #{n}"

appraisal/ruby-2.7.rb

@@ -235,6 +235,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 [3, 4, 5].each do |n|
  appraise "redis-#{n}" do
  gem 'redis', "~> #{n}"

appraisal/ruby-3.0.rb

@@ -162,6 +162,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
  gem 'dalli', '< 3.0.0'
  gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.1.rb

@@ -162,6 +162,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
  gem 'dalli', '< 3.0.0'
  gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.2.rb

@@ -162,6 +162,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
  gem 'dalli', '< 3.0.0'
  gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.3.rb

@@ -162,6 +162,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
  gem 'dalli', '< 3.0.0'
  gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

appraisal/ruby-3.4.rb

@@ -163,6 +163,11 @@
  gem 'opentelemetry-sdk', '~> 1.1'
 end
 
+appraise 'opentelemetry_otlp' do
+ gem 'opentelemetry-sdk', '~> 1.1'
+ gem 'opentelemetry-exporter-otlp'
+end
+
 appraise 'contrib-old' do
  gem 'dalli', '< 3.0.0'
  gem 'presto-client', '>= 0.5.14' # Renamed to trino-client in >= 1.0

benchmarks/profiler_sample_gvl.rb

@@ -0,0 +1,86 @@
+# Used to quickly run benchmark under RSpec as part of the usual test suite, to validate it didn't bitrot
+VALIDATE_BENCHMARK_MODE = ENV['VALIDATE_BENCHMARK'] == 'true'
+
+return unless __FILE__ == $PROGRAM_NAME || VALIDATE_BENCHMARK_MODE
+
+require_relative 'benchmarks_helper'
+
+if RUBY_VERSION < '3.2'
+ if VALIDATE_BENCHMARK_MODE
+ # To simplify things, we allow this benchmark to be run in VALIDATE_BENCHMARK_MODE even though it's a no-op
+ $stderr.puts "Skipping benchmark because it requires Ruby 3.2 or newer"
+ return
+ else
+ raise 'This benchmark requires Ruby 3.2 or newer'
+ end
+end
+
+# This benchmark measures the performance of the main stack sampling loop of the profiler
+
+class ProfilerSampleGvlBenchmark
+ # This is needed because we're directly invoking the collector through a testing interface; in normal
+ # use a profiler thread is automatically used.
+ PROFILER_OVERHEAD_STACK_THREAD = Thread.new { sleep }
+
+ def initialize
+ create_profiler
+ @target_thread = thread_with_very_deep_stack
+
+ # Sample once to trigger thread context creation for all threads (including @target_thread)
+ Datadog::Profiling::Collectors::ThreadContext::Testing._native_sample(@collector, PROFILER_OVERHEAD_STACK_THREAD)
+ end
+
+ def create_profiler
+ @recorder = Datadog::Profiling::StackRecorder.new(
+ cpu_time_enabled: true,
+ alloc_samples_enabled: false,
+ heap_samples_enabled: false,
+ heap_size_enabled: false,
+ heap_sample_every: 1,
+ timeline_enabled: true,
+ )
+ @collector = Datadog::Profiling::Collectors::ThreadContext.for_testing(
+ recorder: @recorder,
+ waiting_for_gvl_threshold_ns: 0,
+ timeline_enabled: true,
+ )
+ end
+
+ def thread_with_very_deep_stack(depth: 200)
+ deep_stack = proc do |n|
+ if n > 0
+ deep_stack.call(n - 1)
+ else
+ sleep
+ end
+ end
+
+ Thread.new { deep_stack.call(depth) }.tap { |t| t.name = "Deep stack #{depth}" }
+ end
+
+ def run_benchmark
+ Benchmark.ips do |x|
+ benchmark_time = VALIDATE_BENCHMARK_MODE ? { time: 0.01, warmup: 0 } : { time: 20, warmup: 2 }
+ x.config(
+ **benchmark_time,
+ )
+
+ x.report("gvl benchmark samples") do
+ Datadog::Profiling::Collectors::ThreadContext::Testing._native_on_gvl_waiting(@target_thread)
+ Datadog::Profiling::Collectors::ThreadContext::Testing._native_on_gvl_running(@target_thread)
+ Datadog::Profiling::Collectors::ThreadContext::Testing._native_sample_after_gvl_running(@collector, @target_thread)
+ end
+
+ x.save! "#{File.basename(__FILE__)}-results.json" unless VALIDATE_BENCHMARK_MODE
+ x.compare!
+ end
+
+ @recorder.serialize!
+ end
+end
+
+puts "Current pid is #{Process.pid}"
+
+ProfilerSampleGvlBenchmark.new.instance_exec do
+ run_benchmark
+end

benchmarks/run_all.sh

@@ -14,6 +14,7 @@ for file in \
  `dirname "$0"`/profiler_memory_sample_serialize.rb \
  `dirname "$0"`/profiler_sample_loop_v2.rb \
  `dirname "$0"`/profiler_sample_serialize.rb \
+ `dirname "$0"`/profiler_sample_gvl.rb \
  `dirname "$0"`/tracing_trace.rb;
 do
  bundle exec ruby "$file"

docs/ProfilingDevelopment.md

@@ -209,9 +209,9 @@ from the VM callbacks and also messing with cpu/wall-time accounting for threads
 
 The `ThreadContext` collector exposes three APIs for GVL profiling:
 
-* `void thread_context_collector_on_gvl_waiting(VALUE thread)`
-* `bool thread_context_collector_on_gvl_running(VALUE thread)`
-* `VALUE thread_context_collector_sample_after_gvl_running(VALUE self_instance)`
+* `thread_context_collector_on_gvl_waiting`
+* `thread_context_collector_on_gvl_running`
+* `thread_context_collector_sample_after_gvl_running`
 
 The intuition here is that: