Merge branch 'main' into upgrade-latest-redis-version

.github/workflows/unit_tests.yml

@@ -17,6 +17,7 @@ jobs:
  os: [ubuntu-latest, windows-latest, macos-latest]
  # Keep this in sync with hatch.toml
  python-version: ["3.7", "3.10", "3.12"]
+
  steps:
  - uses: actions/checkout@v4
  # Include all history and tags
@@ -32,5 +33,9 @@ jobs:
  with:
  version: "1.12.0"
 
+ - name: Install coreutils for MacOS to get sha256sum
+ if: matrix.os == 'macos-latest'
+ run: brew install coreutils
+
  - name: Run tests
  run: hatch run +py=${{ matrix.python-version }} ddtrace_unit_tests:test

.gitignore

@@ -19,6 +19,7 @@ ddtrace/internal/_encoding.c
 ddtrace/internal/_rand.c
 ddtrace/internal/_tagset.c
 *.so
+*.dylib
 *.a
 
 # Cython annotate HTML files

.gitlab/tests/appsec.yml

@@ -19,7 +19,6 @@ appsec iast:
 
 appsec iast tdd_propagation:
  extends: .test_base_riot_snapshot
- allow_failure: true
  parallel: 2
  variables:
  SUITE_NAME: "appsec_iast_tdd_propagation"
@@ -61,7 +60,7 @@ appsec aggregated leak testing:
  variables:
  SUITE_NAME: "appsec_aggregated_leak_testing"
  retry: 2
- timeout: 25m
+ timeout: 35m
 
 appsec iast native:
  extends: .test_base_hatch

ddtrace/_monkey.py

@@ -79,10 +79,10 @@
  "falcon": True,
  "pyramid": True,
  # Auto-enable logging if the environment variable DD_LOGS_INJECTION is true
- "logbook": config.logs_injection,
- "logging": config.logs_injection,
- "loguru": config.logs_injection,
- "structlog": config.logs_injection,
+ "logbook": config.logs_injection, # type: ignore
+ "logging": config.logs_injection, # type: ignore
+ "loguru": config.logs_injection, # type: ignore
+ "structlog": config.logs_injection, # type: ignore
  "pynamodb": True,
  "pyodbc": True,
  "fastapi": True,

ddtrace/appsec/_iast/__init__.py

@@ -35,6 +35,7 @@ def wrapped_function(wrapped, instance, args, kwargs):
 from ddtrace.internal.module import ModuleWatchdog
 from ddtrace.internal.utils.formats import asbool
 
+from .._constants import IAST
 from ._overhead_control_engine import OverheadControl
 from ._utils import _is_iast_enabled
 
@@ -71,7 +72,8 @@ def ddtrace_iast_flask_patch():
 
 
 def enable_iast_propagation():
- if asbool(os.getenv("DD_IAST_ENABLED", False)):
+ """Add IAST AST patching in the ModuleWatchdog"""
+ if asbool(os.getenv(IAST.ENV, "false")):
  from ddtrace.appsec._iast._utils import _is_python_version_supported
 
  if _is_python_version_supported():
@@ -82,8 +84,20 @@ def enable_iast_propagation():
  ModuleWatchdog.register_pre_exec_module_hook(_should_iast_patch, _exec_iast_patched_module)
 
 
+def disable_iast_propagation():
+ """Remove IAST AST patching from the ModuleWatchdog. Only for testing proposes"""
+ from ddtrace.appsec._iast._ast.ast_patching import _should_iast_patch
+ from ddtrace.appsec._iast._loader import _exec_iast_patched_module
+
+ try:
+ ModuleWatchdog.remove_pre_exec_module_hook(_should_iast_patch, _exec_iast_patched_module)
+ except KeyError:
+ log.warning("IAST is already disabled and it's not in the ModuleWatchdog")
+
+
 __all__ = [
  "oce",
  "ddtrace_iast_flask_patch",
  "enable_iast_propagation",
+ "disable_iast_propagation",
 ]

ddtrace/appsec/_iast/_ast/ast_patching.py

@@ -300,11 +300,6 @@
  "uvicorn.",
  "anyio.",
  "httpcore.",
- "pypika.",
- "pydantic.",
- "pydantic_core.",
- "pydantic_settings.",
- "tomli.",
 )
 
 

ddtrace/bootstrap/preload.py

@@ -6,8 +6,6 @@
 import os # noqa:I001
 
 from ddtrace import config # noqa:F401
-from ddtrace.debugging._config import di_config # noqa:F401
-from ddtrace.debugging._config import er_config # noqa:F401
 from ddtrace.settings.profiling import config as profiling_config # noqa:F401
 from ddtrace.internal.logger import get_logger # noqa:F401
 from ddtrace.internal.module import ModuleWatchdog # noqa:F401
@@ -17,9 +15,7 @@
 from ddtrace.internal.utils.formats import asbool # noqa:F401
 from ddtrace.internal.utils.formats import parse_tags_str # noqa:F401
 from ddtrace.settings.asm import config as asm_config # noqa:F401
-from ddtrace.settings.code_origin import config as co_config # noqa:F401
 from ddtrace.settings.crashtracker import config as crashtracker_config
-from ddtrace.settings.symbol_db import config as symdb_config # noqa:F401
 from ddtrace import tracer
 
 
@@ -72,26 +68,6 @@ def register_post_preload(func: t.Callable) -> None:
  except Exception:
  log.error("failed to enable profiling", exc_info=True)
 
-if symdb_config.enabled:
- from ddtrace.internal import symbol_db
-
- symbol_db.bootstrap()
-
-if di_config.enabled: # Dynamic Instrumentation
- from ddtrace.debugging import DynamicInstrumentation
-
- DynamicInstrumentation.enable()
-
-if co_config.span.enabled:
- from ddtrace.debugging._origin.span import SpanCodeOriginProcessor
-
- SpanCodeOriginProcessor.enable()
-
-if er_config.enabled: # Exception Replay
- from ddtrace.debugging._exception.replay import SpanExceptionHandler
-
- SpanExceptionHandler.enable()
-
 if config._runtime_metrics_enabled:
  RuntimeWorker.enable()
 

ddtrace/bootstrap/sitecustomize.py

@@ -20,6 +20,7 @@
 import sys
 import warnings # noqa:F401
 
+from ddtrace.internal.telemetry import telemetry_writer
 from ddtrace import config # noqa:F401
 from ddtrace._logger import _configure_log_injection
 from ddtrace.internal.logger import get_logger # noqa:F401
@@ -158,7 +159,8 @@ def _(threading):
  else:
  log.debug("additional sitecustomize found in: %s", sys.path)
 
- config._ddtrace_bootstrapped = True
+ telemetry_writer.add_configuration("ddtrace_bootstrapped", True, "unknown")
+ telemetry_writer.add_configuration("ddtrace_auto_used", "ddtrace.auto" in sys.modules, "unknown")
  # Loading status used in tests to detect if the `sitecustomize` has been
  # properly loaded without exceptions. This must be the last action in the module
  # when the execution ends with a success.

ddtrace/debugging/_debugger.py

@@ -54,7 +54,6 @@
 from ddtrace.debugging._signal.tracing import SpanDecoration
 from ddtrace.debugging._uploader import LogsIntakeUploaderV1
 from ddtrace.debugging._uploader import UploaderProduct
-from ddtrace.internal import atexit
 from ddtrace.internal import compat
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.metrics import Metrics
@@ -305,7 +304,6 @@ def enable(cls, run_module: bool = False) -> None:
 
  debugger.start()
 
- atexit.register(cls.disable)
  register_post_run_module_hook(cls._on_run_module)
  telemetry_writer.product_activated(TELEMETRY_APM_PRODUCT.DYNAMIC_INSTRUMENTATION, True)
 
@@ -326,7 +324,6 @@ def disable(cls, join: bool = True) -> None:
 
  remoteconfig_poller.unregister("LIVE_DEBUGGING")
 
- atexit.unregister(cls.disable)
  unregister_post_run_module_hook(cls._on_run_module)
 
  cls._instance.stop(join=join)

ddtrace/debugging/_function/discovery.py

@@ -66,7 +66,9 @@ def __init__(
  origin: Optional[Union[Tuple["ContainerIterator", ContainerKey], Tuple[FullyNamedFunction, str]]] = None,
  ) -> None:
  if isinstance(container, (type, ModuleType)):
- self._iter = iter(container.__dict__.items())
+ # DEV: A module object could be partially initialised, therefore
+ # __dict__ can mutate.
+ self._iter = iter(container.__dict__.copy().items())
  self.__name__ = container.__name__
 
  elif isinstance(container, tuple):

ddtrace/debugging/_products/code_origin/span.py

@@ -0,0 +1,31 @@
+from ddtrace.settings.code_origin import config
+
+
+# TODO[gab]: Uncomment this when the feature is ready
+# requires = ["tracer"]
+
+
+def post_preload():
+ pass
+
+
+def start():
+ if config.span.enabled:
+ from ddtrace.debugging._origin.span import SpanCodeOriginProcessor
+
+ SpanCodeOriginProcessor.enable()
+
+
+def restart(join=False):
+ pass
+
+
+def stop(join=False):
+ if config.span.enabled:
+ from ddtrace.debugging._origin.span import SpanCodeOriginProcessor
+
+ SpanCodeOriginProcessor.disable()
+
+
+def at_exit(join=False):
+ stop(join=join)

ddtrace/debugging/_products/dynamic_instrumentation.py

@@ -0,0 +1,31 @@
+from ddtrace.settings.dynamic_instrumentation import config
+
+
+requires = ["remote-configuration"]
+
+
+def post_preload():
+ pass
+
+
+def start():
+ if config.enabled:
+ from ddtrace.debugging import DynamicInstrumentation
+
+ DynamicInstrumentation.enable()
+
+
+def restart(join=False):
+ # Nothing to do
+ pass
+
+
+def stop(join=False):
+ if config.enabled:
+ from ddtrace.debugging import DynamicInstrumentation
+
+ DynamicInstrumentation.disable(join=join)
+
+
+def at_exit(join=False):
+ stop(join=join)

ddtrace/debugging/_products/exception_replay.py

@@ -0,0 +1,31 @@
+from ddtrace.debugging._config import er_config
+
+
+# TODO[gab]: Uncomment this when the feature is ready
+# requires = ["tracer"]
+
+
+def post_preload():
+ pass
+
+
+def start():
+ if er_config.enabled:
+ from ddtrace.debugging._exception.replay import SpanExceptionHandler
+
+ SpanExceptionHandler.enable()
+
+
+def restart(join=False):
+ pass
+
+
+def stop(join=False):
+ if er_config.enabled:
+ from ddtrace.debugging._exception.replay import SpanExceptionHandler
+
+ SpanExceptionHandler.disable()
+
+
+def at_exit(join=False):
+ stop(join=join)

ddtrace/internal/agent.py

@@ -4,8 +4,8 @@
 from typing import TypeVar
 from typing import Union
 
+from ddtrace.internal.constants import DEFAULT_TIMEOUT
 from ddtrace.internal.logger import get_logger
-from ddtrace.settings import _config as ddconfig
 
 from .http import HTTPConnection
 from .http import HTTPSConnection
@@ -44,15 +44,15 @@ def get_trace_url():
 
  Raises a ``ValueError`` if the URL is not supported by the Agent.
  """
- user_supplied_host = ddconfig._trace_agent_hostname is not None
- user_supplied_port = ddconfig._trace_agent_port is not None
+ user_supplied_host = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_TRACE_AGENT_HOSTNAME"))
+ user_supplied_port = os.environ.get("DD_AGENT_PORT", os.environ.get("DD_TRACE_AGENT_PORT"))
 
- url = ddconfig._trace_agent_url
+ url = os.environ.get("DD_TRACE_AGENT_URL")
 
  if not url:
- if user_supplied_host or user_supplied_port:
- host = ddconfig._trace_agent_hostname or DEFAULT_HOSTNAME
- port = ddconfig._trace_agent_port or DEFAULT_TRACE_PORT
+ if user_supplied_host is not None or user_supplied_port is not None:
+ host = user_supplied_host or DEFAULT_HOSTNAME
+ port = user_supplied_port or DEFAULT_TRACE_PORT
  if is_ipv6_hostname(host):
  host = "[{}]".format(host)
  url = "http://%s:%s" % (host, port)
@@ -66,15 +66,14 @@ def get_trace_url():
 
 def get_stats_url():
  # type: () -> str
- user_supplied_host = ddconfig._stats_agent_hostname is not None
- user_supplied_port = ddconfig._stats_agent_port is not None
-
- url = ddconfig._stats_agent_url
+ user_supplied_host = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_DOGSTATSD_HOST"))
+ user_supplied_port = os.getenv("DD_DOGSTATSD_PORT")
+ url = os.getenv("DD_DOGSTATSD_URL")
 
  if not url:
- if user_supplied_host or user_supplied_port:
- port = ddconfig._stats_agent_port or DEFAULT_STATS_PORT
- host = ddconfig._stats_agent_hostname or DEFAULT_HOSTNAME
+ if user_supplied_host is not None or user_supplied_port is not None:
+ port = user_supplied_port or DEFAULT_STATS_PORT
+ host = user_supplied_host or DEFAULT_HOSTNAME
  if is_ipv6_hostname(host):
  host = "[{}]".format(host)
  url = "udp://{}:{}".format(host, port)
@@ -87,7 +86,8 @@ def get_stats_url():
 
 def info(url=None):
  agent_url = get_trace_url() if url is None else url
- _conn = get_connection(agent_url, timeout=ddconfig._agent_timeout_seconds)
+ timeout = float(os.getenv("DD_TRACE_AGENT_TIMEOUT_SECONDS", DEFAULT_TIMEOUT))
+ _conn = get_connection(agent_url, timeout=timeout)
  try:
  _conn.request("GET", "info", headers={"content-type": "application/json"})
  resp = _conn.getresponse()