From 885df93249cd18b4968cfabbab4aca939cbbd6a5 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Tue, 8 Oct 2024 09:32:59 +0200 Subject: [PATCH 1/9] chore(iast): remove IAST deny list elements (#10961) A memory leak was introduced in #10540 when "py" was removed from the deny list. This caused a leak in FastAPI with the `pypika` package. #10846 patched the issue, and #10947 resolved it. Now, we're re-enabling those packages. This PR is tested in #10902 ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/appsec/_iast/_ast/ast_patching.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ddtrace/appsec/_iast/_ast/ast_patching.py b/ddtrace/appsec/_iast/_ast/ast_patching.py index 9aa7298538d..572fca02ce6 100644 --- a/ddtrace/appsec/_iast/_ast/ast_patching.py +++ b/ddtrace/appsec/_iast/_ast/ast_patching.py @@ -300,11 +300,6 @@ "uvicorn.", "anyio.", "httpcore.", - "pypika.", - "pydantic.", - "pydantic_core.", - "pydantic_settings.", - "tomli.", ) From f757fbfcc49f53e18811152fd51457b2f60c1ab4 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Tue, 8 Oct 2024 09:33:37 +0100 Subject: [PATCH 2/9] chore(di): guard against mutable module container (#10841) When disovering functions on module import, we look at the module as a container of function-like objects. If the module is partially loaded, its `__dict__` might mutate. We ensure to iterate over a copy of the module's `__dict__` when iterating over it. We expect the original module to mutate dynamically at runtime, so by the time we make a copy of its underlying lookup dict, we are able to detected at the very least everything that corresponds to the actual source code. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_function/discovery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddtrace/debugging/_function/discovery.py b/ddtrace/debugging/_function/discovery.py index 5e92ad05a13..9cabb4b3a04 100644 --- a/ddtrace/debugging/_function/discovery.py +++ b/ddtrace/debugging/_function/discovery.py @@ -66,7 +66,9 @@ def __init__( origin: Optional[Union[Tuple["ContainerIterator", ContainerKey], Tuple[FullyNamedFunction, str]]] = None, ) -> None: if isinstance(container, (type, ModuleType)): - self._iter = iter(container.__dict__.items()) + # DEV: A module object could be partially initialised, therefore + # __dict__ can mutate. + self._iter = iter(container.__dict__.copy().items()) self.__name__ = container.__name__ elif isinstance(container, tuple): From a064a195c831ab0be64ff1c31f1104fc6b26d18e Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Tue, 8 Oct 2024 16:49:11 +0200 Subject: [PATCH 3/9] chore(iast): improve iast scripts (#10902) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .gitlab/tests/appsec.yml | 2 +- ddtrace/appsec/_iast/__init__.py | 16 +- ddtrace/internal/module.py | 8 + hatch.toml | 5 + scripts/iast/README | 2 +- ...st_leak_functions.py => leak_functions.py} | 58 +++- scripts/iast/mod_leak_functions.py | 295 +++++++++++++++--- scripts/iast/requirements.txt | 5 +- scripts/iast/run_memory.sh | 4 +- scripts/iast/run_memray.sh | 4 +- scripts/iast/run_references.sh | 3 + scripts/iast/test_references.py | 8 +- .../test_aggregated_memleaks.py | 12 +- 13 files changed, 357 insertions(+), 65 deletions(-) rename scripts/iast/{test_leak_functions.py => leak_functions.py} (54%) diff --git a/.gitlab/tests/appsec.yml b/.gitlab/tests/appsec.yml index ed2acaf7e1a..be67ac46e59 100644 --- a/.gitlab/tests/appsec.yml +++ b/.gitlab/tests/appsec.yml @@ -61,7 +61,7 @@ appsec aggregated leak testing: variables: SUITE_NAME: "appsec_aggregated_leak_testing" retry: 2 - timeout: 25m + timeout: 35m appsec iast native: extends: .test_base_hatch diff --git a/ddtrace/appsec/_iast/__init__.py b/ddtrace/appsec/_iast/__init__.py index e7f73adf313..8b3208baa86 100644 --- a/ddtrace/appsec/_iast/__init__.py +++ b/ddtrace/appsec/_iast/__init__.py @@ -35,6 +35,7 @@ def wrapped_function(wrapped, instance, args, kwargs): from ddtrace.internal.module import ModuleWatchdog from ddtrace.internal.utils.formats import asbool +from .._constants import IAST from ._overhead_control_engine import OverheadControl from ._utils import _is_iast_enabled @@ -71,7 +72,8 @@ def ddtrace_iast_flask_patch(): def enable_iast_propagation(): - if asbool(os.getenv("DD_IAST_ENABLED", False)): + """Add IAST AST patching in the ModuleWatchdog""" + if asbool(os.getenv(IAST.ENV, "false")): from ddtrace.appsec._iast._utils import _is_python_version_supported if _is_python_version_supported(): @@ -82,8 +84,20 @@ def enable_iast_propagation(): ModuleWatchdog.register_pre_exec_module_hook(_should_iast_patch, _exec_iast_patched_module) +def disable_iast_propagation(): + """Remove IAST AST patching from the ModuleWatchdog. Only for testing proposes""" + from ddtrace.appsec._iast._ast.ast_patching import _should_iast_patch + from ddtrace.appsec._iast._loader import _exec_iast_patched_module + + try: + ModuleWatchdog.remove_pre_exec_module_hook(_should_iast_patch, _exec_iast_patched_module) + except KeyError: + log.warning("IAST is already disabled and it's not in the ModuleWatchdog") + + __all__ = [ "oce", "ddtrace_iast_flask_patch", "enable_iast_propagation", + "disable_iast_propagation", ] diff --git a/ddtrace/internal/module.py b/ddtrace/internal/module.py index 2c607990a25..6b5b8192773 100644 --- a/ddtrace/internal/module.py +++ b/ddtrace/internal/module.py @@ -670,6 +670,14 @@ def register_pre_exec_module_hook( instance = t.cast(ModuleWatchdog, cls._instance) instance._pre_exec_module_hooks.add((cond, hook)) + @classmethod + def remove_pre_exec_module_hook( + cls: t.Type["ModuleWatchdog"], cond: PreExecHookCond, hook: PreExecHookType + ) -> None: + """Register a hook to execute before/instead of exec_module. Only for testing proposes""" + instance = t.cast(ModuleWatchdog, cls._instance) + instance._pre_exec_module_hooks.remove((cond, hook)) + @classmethod def register_import_exception_hook( cls: t.Type["ModuleWatchdog"], cond: ImportExceptionHookCond, hook: ImportExceptionHookType diff --git a/hatch.toml b/hatch.toml index c582f8f6a9d..ca74e784da6 100644 --- a/hatch.toml +++ b/hatch.toml @@ -339,10 +339,15 @@ dependencies = [ "pytest-cov", "hypothesis", "requests", + "pytest-asyncio", + "anyio", + "pydantic", + "pydantic-settings", ] [envs.appsec_aggregated_leak_testing.env-vars] CMAKE_BUILD_PARALLEL_LEVEL = "12" +DD_IAST_ENABLED = "true" [envs.appsec_aggregated_leak_testing.scripts] test = [ diff --git a/scripts/iast/README b/scripts/iast/README index 1f01828478f..e442e8916cc 100644 --- a/scripts/iast/README +++ b/scripts/iast/README @@ -53,7 +53,7 @@ sh scripts/iast/run_memory.sh" docker run --rm -it -v ${PWD}:/ddtrace python_311_debug /bin/bash -c "cd /ddtrace && source scripts/iast/.env && \ valgrind --tool=memcheck --leak-check=full --log-file=scripts/iast/valgrind_bench_overload.out --track-origins=yes \ --suppressions=scripts/iast/valgrind-python.supp --show-leak-kinds=all \ -python3.11 scripts/iast/test_leak_functions.py --iterations 100" +python3.11 scripts/iast/leak_functions.py --iterations 100" ##### Understanding results of memcheck diff --git a/scripts/iast/test_leak_functions.py b/scripts/iast/leak_functions.py similarity index 54% rename from scripts/iast/test_leak_functions.py rename to scripts/iast/leak_functions.py index 1f30210bb8a..d85f2f49486 100644 --- a/scripts/iast/test_leak_functions.py +++ b/scripts/iast/leak_functions.py @@ -1,15 +1,19 @@ import argparse +import asyncio +import dis +import io import resource import sys -from tests.appsec.iast.aspects.conftest import _iast_patched_module -from tests.utils import override_env - +import pytest -with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking import create_context - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted - from ddtrace.appsec._iast._taint_tracking import reset_context +from ddtrace.appsec._iast import disable_iast_propagation +from ddtrace.appsec._iast import enable_iast_propagation +from ddtrace.appsec._iast._taint_tracking import active_map_addreses_size +from ddtrace.appsec._iast._taint_tracking import create_context +from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking import reset_context +from tests.utils import override_env def parse_arguments(): @@ -22,26 +26,41 @@ def parse_arguments(): return parser.parse_args() -def test_iast_leaks(iterations: int, fail_percent: float, print_every: int): - if iterations < 60000: +def _pre_checks(module, aspect_to_check="add_aspect"): + """Ensure the module code is replaced by IAST patching. To do that, this function inspects the bytecode""" + dis_output = io.StringIO() + dis.dis(module, file=dis_output) + str_output = dis_output.getvalue() + # Should have replaced the binary op with the aspect in add_test: + assert f"({aspect_to_check})" in str_output + + +@pytest.mark.asyncio +async def iast_leaks(iterations: int, fail_percent: float, print_every: int): + mem_reference_iterations = 50000 + if iterations < mem_reference_iterations: print( "Error: not running with %d iterations. At least 60.000 are needed to stabilize the RSS info" % iterations ) sys.exit(1) try: - mem_reference_iterations = 50000 print("Test %d iterations" % iterations) current_rss = 0 half_rss = 0 + enable_iast_propagation() + from scripts.iast.mod_leak_functions import test_doit + + # TODO(avara1986): pydantic is in the DENY_LIST, remove from it and uncomment this lines + # from pydantic import main + # _pre_checks(main, "index_aspect") - mod = _iast_patched_module("scripts.iast.mod_leak_functions") - test_doit = mod.test_doit + _pre_checks(test_doit) for i in range(iterations): create_context() - result = test_doit() # noqa: F841 - assert result == "DDD_III_extend", f"result is {result}" # noqa: F841 + result = await test_doit() + assert result == "DDD_III_extend", f"result is {result}" assert is_pyobject_tainted(result) reset_context() @@ -52,11 +71,13 @@ def test_iast_leaks(iterations: int, fail_percent: float, print_every: int): current_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 if i % print_every == 0: - print(f"Round {i} Max RSS: {current_rss}") + print( + f"Round {i} Max RSS: {current_rss}, Number of active maps addresses: {active_map_addreses_size()}" + ) final_rss = current_rss - print(f"Round {iterations} Max RSS: {final_rss}") + print(f"Round {iterations} Max RSS: {final_rss}, Number of active maps addresses: {active_map_addreses_size()}") percent_increase = ((final_rss - half_rss) / half_rss) * 100 if percent_increase > fail_percent: @@ -74,9 +95,12 @@ def test_iast_leaks(iterations: int, fail_percent: float, print_every: int): except KeyboardInterrupt: print("Test interrupted.") + finally: + disable_iast_propagation() if __name__ == "__main__": + loop = asyncio.get_event_loop() args = parse_arguments() with override_env({"DD_IAST_ENABLED": "True"}): - sys.exit(test_iast_leaks(args.iterations, args.fail_percent, args.print_every)) + sys.exit(loop.run_until_complete(iast_leaks(args.iterations, args.fail_percent, args.print_every))) diff --git a/scripts/iast/mod_leak_functions.py b/scripts/iast/mod_leak_functions.py index e55480a1d36..afaf79b13fd 100644 --- a/scripts/iast/mod_leak_functions.py +++ b/scripts/iast/mod_leak_functions.py @@ -1,57 +1,231 @@ +from datetime import datetime import os import random import re import subprocess +from typing import Optional +from typing import Tuple +import anyio +from pydantic import BaseModel +from pydantic import Field +from pydantic_core import SchemaValidator import requests -from tests.utils import override_env +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking import taint_pyobject -with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject +v = SchemaValidator( + { + "type": "typed-dict", + "fields": { + "name": { + "type": "typed-dict-field", + "schema": { + "type": "str", + }, + }, + "age": { + "type": "typed-dict-field", + "schema": { + "type": "int", + "ge": 18, + }, + }, + "is_developer": { + "type": "typed-dict-field", + "schema": { + "type": "default", + "schema": {"type": "bool"}, + "default": True, + }, + }, + }, + } +) -def test_doit(): - origin_string1 = "hiroot" - tainted_string_2 = taint_pyobject( - pyobject="1234", source_name="abcdefghijk", source_value="1234", source_origin=OriginType.PARAMETER +class AspectModel(BaseModel): + foo: str = "bar" + apple: int = 1 + + +class Aspectvalidation(BaseModel): + timestamp: datetime + tuple_strings: Tuple[str, str] + dictionary_strs: dict[str, str] + tag: Optional[str] = None + author: Optional[str] = None + favorited: Optional[str] = None + limit: int = Field(20, ge=1) + offset: int = Field(0, ge=0) + + +class ImSubClassOfAString(str): + def __add__(self, other): + return "ImNotAString.__add__!!" + other + + def __iadd__(self, other): + return "ImNotAString.__iadd__!!" + other + + +class ImNotAString: + def __add__(self, other): + return "ImNotAString.__add__!!" + other + + def __iadd__(self, other): + return "ImNotAString.__iadd__!!" + other + + +def add_variants(string_tainted, string_no_tainted) -> str: + new_string_tainted = string_tainted + string_no_tainted + im_not_a_string = ImNotAString() + # TODO(avara1986): it raises seg fault instead of TypeError: can only concatenate str (not "ImNotAString") to str + # try: + # new_string_no_tainted = string_no_tainted + im_not_a_string + # assert False + # except TypeError: + # pass + new_string_no_tainted = im_not_a_string + string_no_tainted + new_string_no_tainted = ImSubClassOfAString() + new_string_no_tainted + string_no_tainted + new_string_no_tainted += string_no_tainted + # TODO(avara1986): it raises seg fault instead of TypeError: can only concatenate str (not "ImNotAString") to str + # try: + # new_string_no_tainted += ImNotAString() + # assert False + # except TypeError: + # pass + + im_not_a_string += new_string_no_tainted + new_string_no_tainted += ImSubClassOfAString() + new_string_tainted += new_string_no_tainted + new_string_tainted += new_string_no_tainted + new_string_tainted += new_string_tainted + + new_bytes_no_tainted = bytes(string_no_tainted, encoding="utf-8") + bytes(string_no_tainted, encoding="utf-8") + new_bytes_no_tainted += bytes(string_no_tainted, encoding="utf-8") + new_bytes_no_tainted += bytes(string_no_tainted, encoding="utf-8") + new_bytes_tainted = bytes(string_tainted, encoding="utf-8") + bytes(string_tainted, encoding="utf-8") + new_bytes_tainted += bytes(string_no_tainted, encoding="utf-8") + new_bytes_tainted += bytes(string_tainted, encoding="utf-8") + new_bytes_tainted += bytes(string_tainted, encoding="utf-8") + new_bytearray_tainted = bytearray(bytes(string_tainted, encoding="utf-8")) + bytearray( + bytes(string_tainted, encoding="utf-8") ) + new_bytearray_tainted += bytearray(bytes(string_tainted, encoding="utf-8")) + new_bytearray_tainted += bytearray(bytes(string_no_tainted, encoding="utf-8")) - string1 = str(origin_string1) # String with 1 propagation range - string2 = str(tainted_string_2) # String with 1 propagation range + new_string_tainted = ( + new_string_tainted + + new_string_no_tainted + + str(new_bytes_no_tainted, encoding="utf-8") + + str(new_bytes_tainted, encoding="utf-8") + + str(new_bytearray_tainted, encoding="utf-8") + ) + # print(new_string_tainted) + return new_string_tainted - string3 = string1 + string2 # 2 propagation ranges: hiroot1234 - string4 = "-".join([string3, string3, string3]) # 6 propagation ranges: hiroot1234-hiroot1234-hiroot1234 - string4_2 = string1 - string4_2 += " " + " ".join(string_ for string_ in [string4, string4, string4]) - string5 = string4_2[0:20] # 1 propagation range: hiroot1234-hiroot123 - string6 = string5.title() # 1 propagation range: Hiroot1234-Hiroot123 - string7 = string6.upper() # 1 propagation range: HIROOT1234-HIROOT123 - string8 = "%s_notainted" % string7 # 1 propagation range: HIROOT1234-HIROOT123_notainted - string9 = "notainted_{}".format(string8) # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string10 = "nottainted\n" + string9 # 2 propagation ranges: notainted\nnotainted_HIROOT1234-HIROOT123_notainted - string11 = string10.splitlines()[1] # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string12 = string11 + "_notainted" # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted_notainted - string13 = string12.rsplit("_", 1)[0] # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted + +def format_variants(string_tainted, string_no_tainted) -> str: + string_tainted_2 = "My name is {} and I am {} years old.".format(string_tainted, 30) + string_tainted_3 = "My name is {name} and I am {age} years old.".format(name=string_tainted_2, age=25) + string_tainted_4 = "{0} is {1} years old. {0} lives in {2}.".format(string_tainted_3, 35, string_no_tainted) + string_tainted_5 = "|{:<10}|{:^10}|{:>10}|".format(string_tainted_4, string_no_tainted, string_tainted_4) + string_tainted_6 = "|{:-<10}|{:*^10}|{:.>10}|".format(string_no_tainted, string_tainted_5, string_no_tainted) + string_tainted_7 = "{} is approximately {:.3f}".format(string_tainted_6, 3.1415926535) + string_tainted_8 = "The {} is {:,}".format(string_tainted_7, 1000000) + string_tainted_9 = "{1} Hex: {0:x}, Bin: {0:b}, Oct: {0:o}".format(255, string_tainted_8) + string_tainted_10 = "{} Success rate: {:.2%}".format(string_tainted_9, 0.8765) + string_tainted_11 = "{} {:+d}, {:+d}".format(string_tainted_10, 42, -42) + return string_tainted_11 + + +def modulo_exceptions(string8_4): + # Validate we're not leaking in modulo exceptions + try: + string8_5 = "notainted_%s_" % (string8_4, string8_4) # noqa: F841, F507 + except TypeError: + pass + + try: + string8_5 = "notainted_%d" % string8_4 # noqa: F841 + except TypeError: + pass + + try: + string8_5 = "notainted_%s %s" % "abc", string8_4 # noqa: F841 + except TypeError: + pass + + try: + string8_5 = "notainted_%s %s" % "abc", string8_4 # noqa: F841 + except TypeError: + pass + + try: + string8_5 = "notainted_%s" % (string8_4) # noqa: F841 + except TypeError: + pass + try: + string8_5 = "notainted_%s %(name)s" % string8_4, {"name": string8_4} # noqa: F841, F506 + except TypeError: + pass + + try: + string8_5 = "notainted_%(age)d" % {"age": string8_4} # noqa: F841 + except TypeError: + pass + + +def pydantic_object(tag, string_tainted): + m = Aspectvalidation( + timestamp="2020-01-02T03:04:05Z", + tuple_strings=[string_tainted, string_tainted], + dictionary_strs={ + "wine": string_tainted, + b"cheese": string_tainted, + "cabbage": string_tainted, + }, + tag=string_tainted, + author=string_tainted, + favorited=string_tainted, + limit=20, + offset=0, + ) + + r1 = v.validate_python({"name": m.tuple_strings[0], "age": 35}) + + assert is_pyobject_tainted(m.tuple_strings[0]) + assert is_pyobject_tainted(m.tuple_strings[1]) + assert is_pyobject_tainted(m.dictionary_strs["cabbage"]) + + r2 = v.validate_json('{"name": "' + m.tuple_strings[0] + '", "age": 35}') + + assert r1 == {"name": m.tuple_strings[0], "age": 35, "is_developer": True} + assert r1 == r2 + return m + + +def sink_points(string_tainted): try: # Path traversal vulnerability - m = open("/" + string13 + ".txt") + m = open("/" + string_tainted + ".txt") _ = m.read() except Exception: pass try: # Command Injection vulnerability - _ = subprocess.Popen("ls " + string9) + _ = subprocess.Popen("ls " + string_tainted) except Exception: pass try: # SSRF vulnerability - requests.get("http://" + "foobar") + requests.get("http://" + string_tainted, timeout=1) # urllib3.request("GET", "http://" + "foobar") except Exception: pass @@ -59,16 +233,63 @@ def test_doit(): # Weak Randomness vulnerability _ = random.randint(1, 10) + +async def test_doit(): + sample_str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + + origin_string1 = "".join(random.choices(sample_str, k=5)) + + string_2 = "".join(random.choices(sample_str, k=20)) + tainted_string_2 = taint_pyobject( + pyobject=string_2, source_name="abcdefghijk", source_value=string_2, source_origin=OriginType.PARAMETER + ) + + string1 = str(origin_string1) # String with 1 propagation range + string2 = str(tainted_string_2) # String with 1 propagation range + + string3 = add_variants(string2, string1) + + string4 = "-".join([string3, string3, string3]) + string4_2 = string1 + string4_2 += " " + " ".join(string_ for string_ in [string4, string4, string4]) + string4_2 += " " + " ".join(string_ for string_ in [string1, string1, string1]) + + string4_2 += " " + " ".join([string_ for string_ in [string4_2, string4_2, string4_2]]) + + string5 = string4_2[0:100] + string6 = string5.title() + string7 = string6.upper() + string8 = "%s_notainted" % string7 + string8_2 = "%s_%s_notainted" % (string8, string8) + string8_3 = "notainted_%s_" + string8_2 + string8_4 = string8_3 % "notainted" + + string8_5 = format_variants(string8_4, string1) + await anyio.to_thread.run_sync(modulo_exceptions, string8_5) + + string9 = "notainted#{}".format(string8_5) + string9_2 = f"{string9}_notainted" + string9_3 = f"{string9_2:=^30}_notainted" + string10 = "nottainted\n" + string9_3 + string11 = string10.splitlines()[1] + string12 = string11 + "_notainted" + string13 = string12.rsplit("_", 1)[0] + string13_2 = string13 + " " + string13 + try: + string13_3, string13_5, string13_5 = string13_2.split(" ") + except ValueError: + pass + + sink_points(string13_2) + # os path propagation - string14 = os.path.join(string13, "a") # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted/a - string15 = os.path.split(string14)[0] # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string16 = os.path.dirname( - string15 + "/" + "foobar" - ) # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string17 = os.path.basename("/foobar/" + string16) # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string18 = os.path.splitext(string17 + ".jpg")[0] # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string19 = os.path.normcase(string18) # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted - string20 = os.path.splitdrive(string19)[1] # 1 propagation range: notainted_HIROOT1234-HIROOT123_notainted + string14 = os.path.join(string13_2, "a") + string15 = os.path.split(string14)[0] + string16 = os.path.dirname(string15 + "/" + "foobar") + string17 = os.path.basename("/foobar/" + string16) + string18 = os.path.splitext(string17 + ".jpg")[0] + string19 = os.path.normcase(string18) + string20 = os.path.splitdrive(string19)[1] re_slash = re.compile(r"[_.][a-zA-Z]*") string21 = re_slash.findall(string20)[0] # 1 propagation: '_HIROOT @@ -97,4 +318,8 @@ def test_doit(): tmp_str2 = "_extend" string27 += tmp_str2 + # TODO(avara1986): pydantic is in the DENY_LIST, remove from it and uncomment this lines + # result = await anyio.to_thread.run_sync(functools.partial(pydantic_object, string_tainted=string27), string27) + # result = pydantic_object(tag="test2", string_tainted=string27) + # return result.tuple_strings[0] return string27 diff --git a/scripts/iast/requirements.txt b/scripts/iast/requirements.txt index 0fd2996a90d..e6ae243a21d 100644 --- a/scripts/iast/requirements.txt +++ b/scripts/iast/requirements.txt @@ -1 +1,4 @@ -memray==1.12.0 \ No newline at end of file +memray==1.12.0 +anyio +pydantic +pydantic-settings \ No newline at end of file diff --git a/scripts/iast/run_memory.sh b/scripts/iast/run_memory.sh index 0731643b865..a0dd73730eb 100644 --- a/scripts/iast/run_memory.sh +++ b/scripts/iast/run_memory.sh @@ -2,4 +2,6 @@ set -o xtrace PYTHON="${PYTHON_VERSION:-python3.11}" $PYTHON -m pip install -r scripts/iast/requirements.txt -$PYTHON scripts/iast/test_leak_functions.py --iterations 1000000 --print_every 250 \ No newline at end of file +export DD_IAST_ENABLED=true +export _DD_IAST_DEBUG=true +$PYTHON scripts/iast/leak_functions.py --iterations 1000000 --print_every 250 \ No newline at end of file diff --git a/scripts/iast/run_memray.sh b/scripts/iast/run_memray.sh index c785fb520e2..96982a2e513 100644 --- a/scripts/iast/run_memray.sh +++ b/scripts/iast/run_memray.sh @@ -2,5 +2,7 @@ set -o xtrace PYTHON="${PYTHON_VERSION:-python3.11d}" $PYTHON -m pip install -r scripts/iast/requirements.txt -$PYTHON -m memray run --trace-python-allocators --aggregate --native -o lel.bin -f scripts/iast/test_leak_functions.py --iterations 100 +export DD_IAST_ENABLED=true +export _DD_IAST_DEBUG=true +$PYTHON -m memray run --trace-python-allocators --aggregate --native -o lel.bin -f scripts/iast/leak_functions.py --iterations 100 # $PYTHON -m memray flamegraph lel.bin --leaks -f \ No newline at end of file diff --git a/scripts/iast/run_references.sh b/scripts/iast/run_references.sh index 071696b9d94..769771fc632 100644 --- a/scripts/iast/run_references.sh +++ b/scripts/iast/run_references.sh @@ -1,4 +1,7 @@ set -o xtrace PYTHON="${PYTHON_VERSION:-python3.11d}" +$PYTHON -m pip install -r scripts/iast/requirements.txt +export DD_IAST_ENABLED=true +export _DD_IAST_DEBUG=true ${PYTHON} -m ddtrace.commands.ddtrace_run ${PYTHON} scripts/iast/test_references.py \ No newline at end of file diff --git a/scripts/iast/test_references.py b/scripts/iast/test_references.py index e6b91bd3de0..d4cdd2fcc16 100644 --- a/scripts/iast/test_references.py +++ b/scripts/iast/test_references.py @@ -1,3 +1,4 @@ +import asyncio import gc import sys @@ -8,13 +9,13 @@ from ddtrace.appsec._iast._taint_tracking import reset_context -def test_main(): +async def test_main(): for i in range(1000): gc.collect() a = sys.gettotalrefcount() try: create_context() - result = test_doit() # noqa: F841 + result = await test_doit() # noqa: F841 assert is_pyobject_tainted(result) reset_context() except KeyboardInterrupt: @@ -25,4 +26,5 @@ def test_main(): if __name__ == "__main__": - test_main() + loop = asyncio.get_event_loop() + sys.exit(loop.run_until_complete(test_main())) diff --git a/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py b/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py index ce213cbe113..c64ca7504d2 100644 --- a/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py +++ b/tests/appsec/iast_aggregated_memcheck/test_aggregated_memleaks.py @@ -1,8 +1,12 @@ +import pytest + from tests.utils import override_env -def test_aggregated_leaks(): - with override_env({"DD_IAST_ENABLED": "True"}): - from scripts.iast.test_leak_functions import test_iast_leaks +@pytest.mark.asyncio +async def test_aggregated_leaks(): + with override_env({"DD_IAST_ENABLED": "true"}): + from scripts.iast.leak_functions import iast_leaks - assert test_iast_leaks(75000, 2.0, 100) == 0 + result = await iast_leaks(75000, 1.0, 100) == 0 + assert result From cd1753d92387f177a547ed043cf73fbd4ff703d1 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Tue, 8 Oct 2024 18:31:00 +0200 Subject: [PATCH 4/9] chore(profiling): update relnote for endpoint stack v2 (#10962) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .../notes/profiling-fix-stack-v2-endpoint-82a1e26366166b8d.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/profiling-fix-stack-v2-endpoint-82a1e26366166b8d.yaml b/releasenotes/notes/profiling-fix-stack-v2-endpoint-82a1e26366166b8d.yaml index 0505c26e550..73813f7cd70 100644 --- a/releasenotes/notes/profiling-fix-stack-v2-endpoint-82a1e26366166b8d.yaml +++ b/releasenotes/notes/profiling-fix-stack-v2-endpoint-82a1e26366166b8d.yaml @@ -1,6 +1,6 @@ --- fixes: - | - profiling: enables endpoint profiling for stack v2, ``DD_PROFILING_STACK_V2_ENABLED`` + profiling: fixes endpoint profiling for stack v2, when ``DD_PROFILING_STACK_V2_ENABLED`` is set. From 48c2a2c4892c343938495a872d7f7dc185f1ac00 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Tue, 8 Oct 2024 17:06:50 -0400 Subject: [PATCH 5/9] chore(telemetry): decouple telemetry writer from global config (#10863) ## Description Currently, the telemetry writer relies on `ddtrace.config` to enable and send telemetry payloads. `ddtrace.config` is also used to generate and queue `app-started` and `app-client-configuration-changed` telemetry events. This creates a circular dependency that can be painful to manage. Currently we leverage inlined imports and hope that `ddtrace.config` is initialized before `ddtrace.internal.telemetry` is imported. This change resolves this circular dependency by refactoring components in`ddtrace.internal.telemetry` to access environment variables directly (instead of using ddtrace.config). ## Motiviation This change will allow ddtrace contributors to queue telemetry configurations, metrics, and logs before the global config object is initialized. ## Changes - Removes all references to `ddtrace.config` and all envier configuration objects from `ddtrace.internal.telemetry` package - Ensures configuration telemetry is queued when configurations are set (currently this information is collected 10 seconds after ddtrace is first imported). - Ensures the origin of a configuration value is recorded (possible values: remote_config, env_var, code, and unknown). Previously the origin of all configurations was set to unknown. - Updates the name of telemetry configurations to match the name of environment variables (where possible). ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Emmett Butler <723615+emmettbutler@users.noreply.github.com> --- ddtrace/_monkey.py | 8 +- ddtrace/bootstrap/sitecustomize.py | 4 +- ddtrace/internal/agent.py | 30 +- ddtrace/internal/runtime/runtime_metrics.py | 3 +- ddtrace/internal/telemetry/constants.py | 75 ---- ddtrace/internal/telemetry/data.py | 1 - ddtrace/internal/telemetry/writer.py | 278 +++----------- ddtrace/settings/_core.py | 50 +++ ddtrace/settings/asm.py | 2 + ddtrace/settings/config.py | 239 ++++++------ ddtrace/settings/crashtracker.py | 2 + ddtrace/settings/dynamic_instrumentation.py | 2 + ddtrace/settings/exception_replay.py | 3 + ddtrace/settings/peer_service.py | 2 + ddtrace/settings/profiling.py | 2 + ddtrace/settings/symbol_db.py | 3 + tests/conftest.py | 11 + tests/integration/test_settings.py | 2 +- tests/telemetry/test_writer.py | 405 +++++++++++++------- tests/tracer/test_global_config.py | 3 +- 20 files changed, 534 insertions(+), 591 deletions(-) create mode 100644 ddtrace/settings/_core.py diff --git a/ddtrace/_monkey.py b/ddtrace/_monkey.py index 3864df247b9..40f4d20bb28 100644 --- a/ddtrace/_monkey.py +++ b/ddtrace/_monkey.py @@ -79,10 +79,10 @@ "falcon": True, "pyramid": True, # Auto-enable logging if the environment variable DD_LOGS_INJECTION is true - "logbook": config.logs_injection, - "logging": config.logs_injection, - "loguru": config.logs_injection, - "structlog": config.logs_injection, + "logbook": config.logs_injection, # type: ignore + "logging": config.logs_injection, # type: ignore + "loguru": config.logs_injection, # type: ignore + "structlog": config.logs_injection, # type: ignore "pynamodb": True, "pyodbc": True, "fastapi": True, diff --git a/ddtrace/bootstrap/sitecustomize.py b/ddtrace/bootstrap/sitecustomize.py index 7b9f5e69c12..9a27dbd8bb1 100644 --- a/ddtrace/bootstrap/sitecustomize.py +++ b/ddtrace/bootstrap/sitecustomize.py @@ -20,6 +20,7 @@ import sys import warnings # noqa:F401 +from ddtrace.internal.telemetry import telemetry_writer from ddtrace import config # noqa:F401 from ddtrace._logger import _configure_log_injection from ddtrace.internal.logger import get_logger # noqa:F401 @@ -158,7 +159,8 @@ def _(threading): else: log.debug("additional sitecustomize found in: %s", sys.path) - config._ddtrace_bootstrapped = True + telemetry_writer.add_configuration("ddtrace_bootstrapped", True, "unknown") + telemetry_writer.add_configuration("ddtrace_auto_used", "ddtrace.auto" in sys.modules, "unknown") # Loading status used in tests to detect if the `sitecustomize` has been # properly loaded without exceptions. This must be the last action in the module # when the execution ends with a success. diff --git a/ddtrace/internal/agent.py b/ddtrace/internal/agent.py index 513b4af0027..d1ee2fa6716 100644 --- a/ddtrace/internal/agent.py +++ b/ddtrace/internal/agent.py @@ -4,8 +4,8 @@ from typing import TypeVar from typing import Union +from ddtrace.internal.constants import DEFAULT_TIMEOUT from ddtrace.internal.logger import get_logger -from ddtrace.settings import _config as ddconfig from .http import HTTPConnection from .http import HTTPSConnection @@ -44,15 +44,15 @@ def get_trace_url(): Raises a ``ValueError`` if the URL is not supported by the Agent. """ - user_supplied_host = ddconfig._trace_agent_hostname is not None - user_supplied_port = ddconfig._trace_agent_port is not None + user_supplied_host = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_TRACE_AGENT_HOSTNAME")) + user_supplied_port = os.environ.get("DD_AGENT_PORT", os.environ.get("DD_TRACE_AGENT_PORT")) - url = ddconfig._trace_agent_url + url = os.environ.get("DD_TRACE_AGENT_URL") if not url: - if user_supplied_host or user_supplied_port: - host = ddconfig._trace_agent_hostname or DEFAULT_HOSTNAME - port = ddconfig._trace_agent_port or DEFAULT_TRACE_PORT + if user_supplied_host is not None or user_supplied_port is not None: + host = user_supplied_host or DEFAULT_HOSTNAME + port = user_supplied_port or DEFAULT_TRACE_PORT if is_ipv6_hostname(host): host = "[{}]".format(host) url = "http://%s:%s" % (host, port) @@ -66,15 +66,14 @@ def get_trace_url(): def get_stats_url(): # type: () -> str - user_supplied_host = ddconfig._stats_agent_hostname is not None - user_supplied_port = ddconfig._stats_agent_port is not None - - url = ddconfig._stats_agent_url + user_supplied_host = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_DOGSTATSD_HOST")) + user_supplied_port = os.getenv("DD_DOGSTATSD_PORT") + url = os.getenv("DD_DOGSTATSD_URL") if not url: - if user_supplied_host or user_supplied_port: - port = ddconfig._stats_agent_port or DEFAULT_STATS_PORT - host = ddconfig._stats_agent_hostname or DEFAULT_HOSTNAME + if user_supplied_host is not None or user_supplied_port is not None: + port = user_supplied_port or DEFAULT_STATS_PORT + host = user_supplied_host or DEFAULT_HOSTNAME if is_ipv6_hostname(host): host = "[{}]".format(host) url = "udp://{}:{}".format(host, port) @@ -87,7 +86,8 @@ def get_stats_url(): def info(url=None): agent_url = get_trace_url() if url is None else url - _conn = get_connection(agent_url, timeout=ddconfig._agent_timeout_seconds) + timeout = float(os.getenv("DD_TRACE_AGENT_TIMEOUT_SECONDS", DEFAULT_TIMEOUT)) + _conn = get_connection(agent_url, timeout=timeout) try: _conn.request("GET", "info", headers={"content-type": "application/json"}) resp = _conn.getresponse() diff --git a/ddtrace/internal/runtime/runtime_metrics.py b/ddtrace/internal/runtime/runtime_metrics.py index e6ebd0073aa..4f214b54645 100644 --- a/ddtrace/internal/runtime/runtime_metrics.py +++ b/ddtrace/internal/runtime/runtime_metrics.py @@ -8,7 +8,6 @@ from ddtrace.internal import atexit from ddtrace.internal import forksafe from ddtrace.internal import telemetry -from ddtrace.internal.telemetry.constants import TELEMETRY_RUNTIMEMETRICS_ENABLED from ddtrace.vendor.dogstatsd import DogStatsd from .. import periodic @@ -21,6 +20,8 @@ from .tag_collectors import TracerTagCollector +TELEMETRY_RUNTIMEMETRICS_ENABLED = "DD_RUNTIME_METRICS_ENABLED" + log = get_logger(__name__) diff --git a/ddtrace/internal/telemetry/constants.py b/ddtrace/internal/telemetry/constants.py index d8dfa7d4f1f..e6ac6ca6359 100644 --- a/ddtrace/internal/telemetry/constants.py +++ b/ddtrace/internal/telemetry/constants.py @@ -21,78 +21,3 @@ class TELEMETRY_APM_PRODUCT(Enum): DYNAMIC_INSTRUMENTATION = "dynamic_instrumentation" PROFILER = "profiler" APPSEC = "appsec" - - -# Configuration names must map to values supported by backend services: -# https://github.com/DataDog/dd-go/blob/f88e85d64b173e7733ac03e23576d1c9be37f32e/trace/apps/tracer-telemetry-intake/telemetry-payload/static/config_norm_rules.json -TELEMETRY_DYNAMIC_INSTRUMENTATION_ENABLED = "DD_DYNAMIC_INSTRUMENTATION_ENABLED" -TELEMETRY_EXCEPTION_REPLAY_ENABLED = "DD_EXCEPTION_REPLAY_ENABLED" - - -# Tracing Features - -TELEMETRY_TRACE_DEBUG = "DD_TRACE_DEBUG" -TELEMETRY_ANALYTICS_ENABLED = "DD_TRACE_ANALYTICS_ENABLED" -TELEMETRY_STARTUP_LOGS_ENABLED = "DD_TRACE_STARTUP_LOGS" -TELEMETRY_CLIENT_IP_ENABLED = "DD_TRACE_CLIENT_IP_ENABLED" -TELEMETRY_128_BIT_TRACEID_GENERATION_ENABLED = "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED" -TELEMETRY_TRACE_COMPUTE_STATS = "DD_TRACE_COMPUTE_STATS" -TELEMETRY_OBFUSCATION_QUERY_STRING_PATTERN = "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP" -TELEMETRY_OTEL_ENABLED = "DD_TRACE_OTEL_ENABLED" -TELEMETRY_TRACE_HEALTH_METRICS_ENABLED = "DD_TRACE_HEALTH_METRICS_ENABLED" -TELEMETRY_ENABLED = "DD_INSTRUMENTATION_TELEMETRY_ENABLED" -TELEMETRY_RUNTIMEMETRICS_ENABLED = "DD_RUNTIME_METRICS_ENABLED" -TELEMETRY_REMOTE_CONFIGURATION_ENABLED = "DD_REMOTE_CONFIGURATION_ENABLED" -TELEMETRY_REMOTE_CONFIGURATION_INTERVAL = "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS" -TELEMETRY_SERVICE_MAPPING = "DD_SERVICE_MAPPING" -TELEMETRY_SPAN_SAMPLING_RULES = "DD_SPAN_SAMPLING_RULES" -TELEMETRY_SPAN_SAMPLING_RULES_FILE = "DD_SPAN_SAMPLING_RULES_FILE" -TELEMETRY_PROPAGATION_STYLE_INJECT = "DD_TRACE_PROPAGATION_STYLE_INJECT" -TELEMETRY_PROPAGATION_STYLE_EXTRACT = "DD_TRACE_PROPAGATION_STYLE_EXTRACT" -TELEMETRY_TRACE_SAMPLING_RULES = "DD_TRACE_SAMPLING_RULES" -TELEMETRY_TRACE_SAMPLING_LIMIT = "DD_TRACE_RATE_LIMIT" -TELEMETRY_PARTIAL_FLUSH_ENABLED = "DD_TRACE_PARTIAL_FLUSH_ENABLED" -TELEMETRY_PARTIAL_FLUSH_MIN_SPANS = "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS" -TELEMETRY_TRACE_SPAN_ATTRIBUTE_SCHEMA = "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA" -TELEMETRY_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED = "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED" -TELEMETRY_TRACE_PEER_SERVICE_DEFAULTS_ENABLED = "DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED" -TELEMETRY_TRACE_PEER_SERVICE_MAPPING = "DD_TRACE_PEER_SERVICE_MAPPING" - -TELEMETRY_TRACE_API_VERSION = "DD_TRACE_API_VERSION" -TELEMETRY_TRACE_WRITER_BUFFER_SIZE_BYTES = "DD_TRACE_WRITER_BUFFER_SIZE_BYTES" -TELEMETRY_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES = "DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES" -TELEMETRY_TRACE_WRITER_INTERVAL_SECONDS = "DD_TRACE_WRITER_INTERVAL_SECONDS" -TELEMETRY_TRACE_WRITER_REUSE_CONNECTIONS = "DD_TRACE_WRITER_REUSE_CONNECTIONS" - -TELEMETRY_DOGSTATSD_PORT = "DD_DOGSTATSD_PORT" -TELEMETRY_DOGSTATSD_URL = "DD_DOGSTATSD_URL" - -TELEMETRY_AGENT_HOST = "DD_AGENT_HOST" -TELEMETRY_AGENT_PORT = "DD_AGENT_PORT" -TELEMETRY_AGENT_URL = "DD_TRACE_AGENT_URL" -TELEMETRY_TRACE_AGENT_TIMEOUT_SECONDS = "DD_TRACE_AGENT_TIMEOUT_SECONDS" - -# Profiling features -TELEMETRY_PROFILING_STACK_ENABLED = "DD_PROFILING_STACK_ENABLED" -TELEMETRY_PROFILING_MEMORY_ENABLED = "DD_PROFILING_MEMORY_ENABLED" -TELEMETRY_PROFILING_HEAP_ENABLED = "DD_PROFILING_HEAP_ENABLED" -TELEMETRY_PROFILING_LOCK_ENABLED = "DD_PROFILING_LOCK_ENABLED" -TELEMETRY_PROFILING_EXPORT_LIBDD_ENABLED = "DD_PROFILING_EXPORT_LIBDD_ENABLED" -TELEMETRY_PROFILING_CAPTURE_PCT = "DD_PROFILING_CAPTURE_PCT" -TELEMETRY_PROFILING_UPLOAD_INTERVAL = "DD_PROFILING_UPLOAD_INTERVAL" -TELEMETRY_PROFILING_MAX_FRAMES = "DD_PROFILING_MAX_FRAMES" - -TELEMETRY_INJECT_WAS_ATTEMPTED = "DD_LIB_INJECTION_ATTEMPTED" -TELEMETRY_LIB_WAS_INJECTED = "DD_LIB_INJECTED" -TELEMETRY_LIB_INJECTION_FORCED = "DD_INJECT_FORCE" - - -# Crashtracker -TELEMETRY_CRASHTRACKING_ENABLED = "crashtracking_enabled" # Env var enabled -TELEMETRY_CRASHTRACKING_AVAILABLE = "crashtracking_available" # Feature is available -TELEMETRY_CRASHTRACKING_STARTED = "crashtracking_started" # Crashtracking is running -TELEMETRY_CRASHTRACKING_STDOUT_FILENAME = "crashtracking_stdout_filename" -TELEMETRY_CRASHTRACKING_STDERR_FILENAME = "crashtracking_stderr_filename" -TELEMETRY_CRASHTRACKING_ALT_STACK = "crashtracking_alt_stack" -TELEMETRY_CRASHTRACKING_STACKTRACE_RESOLVER = "crashtracking_stacktrace_resolver" -TELEMETRY_CRASHTRACKING_DEBUG_URL = "crashtracking_debug_url" diff --git a/ddtrace/internal/telemetry/data.py b/ddtrace/internal/telemetry/data.py index 88be33a6117..3b73ac8b97d 100644 --- a/ddtrace/internal/telemetry/data.py +++ b/ddtrace/internal/telemetry/data.py @@ -12,7 +12,6 @@ from ddtrace.internal.utils.cache import cached from ddtrace.version import get_version -from ...settings import _config as config # noqa:F401 from ..hostname import get_hostname diff --git a/ddtrace/internal/telemetry/writer.py b/ddtrace/internal/telemetry/writer.py index 3612f704c83..4e8cecfb534 100644 --- a/ddtrace/internal/telemetry/writer.py +++ b/ddtrace/internal/telemetry/writer.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import http.client as httplib # noqa: E402 import itertools +from logging import getLogger import os import sys import time @@ -11,26 +13,14 @@ from typing import Set # noqa:F401 from typing import Tuple # noqa:F401 from typing import Union # noqa:F401 +import urllib.parse as parse from ...internal import atexit from ...internal import forksafe -from ...internal.compat import parse -from ...internal.core import crashtracking -from ...internal.schema import SCHEMA_VERSION -from ...internal.schema import _remove_client_service_names -from ...settings import _config as config -from ...settings.config import _ConfigSource -from ...settings.crashtracker import config as crashtracker_config -from ...settings.dynamic_instrumentation import config as di_config -from ...settings.exception_replay import config as er_config -from ...settings.peer_service import _ps_config -from ...settings.profiling import config as prof_config from ..agent import get_connection from ..agent import get_trace_url from ..compat import get_connection_response -from ..compat import httplib from ..encoding import JSONEncoderV2 -from ..logger import get_logger from ..periodic import PeriodicService from ..runtime import container from ..runtime import get_runtime_id @@ -39,65 +29,8 @@ from ..utils.time import StopWatch from ..utils.version import _pep440_to_semver from . import modules -from .constants import TELEMETRY_128_BIT_TRACEID_GENERATION_ENABLED -from .constants import TELEMETRY_AGENT_HOST -from .constants import TELEMETRY_AGENT_PORT -from .constants import TELEMETRY_AGENT_URL -from .constants import TELEMETRY_ANALYTICS_ENABLED from .constants import TELEMETRY_APM_PRODUCT -from .constants import TELEMETRY_CLIENT_IP_ENABLED -from .constants import TELEMETRY_CRASHTRACKING_ALT_STACK -from .constants import TELEMETRY_CRASHTRACKING_AVAILABLE -from .constants import TELEMETRY_CRASHTRACKING_DEBUG_URL -from .constants import TELEMETRY_CRASHTRACKING_ENABLED -from .constants import TELEMETRY_CRASHTRACKING_STACKTRACE_RESOLVER -from .constants import TELEMETRY_CRASHTRACKING_STARTED -from .constants import TELEMETRY_CRASHTRACKING_STDERR_FILENAME -from .constants import TELEMETRY_CRASHTRACKING_STDOUT_FILENAME -from .constants import TELEMETRY_DOGSTATSD_PORT -from .constants import TELEMETRY_DOGSTATSD_URL -from .constants import TELEMETRY_DYNAMIC_INSTRUMENTATION_ENABLED -from .constants import TELEMETRY_ENABLED -from .constants import TELEMETRY_EXCEPTION_REPLAY_ENABLED -from .constants import TELEMETRY_INJECT_WAS_ATTEMPTED -from .constants import TELEMETRY_LIB_INJECTION_FORCED -from .constants import TELEMETRY_LIB_WAS_INJECTED from .constants import TELEMETRY_LOG_LEVEL # noqa:F401 -from .constants import TELEMETRY_OBFUSCATION_QUERY_STRING_PATTERN -from .constants import TELEMETRY_OTEL_ENABLED -from .constants import TELEMETRY_PARTIAL_FLUSH_ENABLED -from .constants import TELEMETRY_PARTIAL_FLUSH_MIN_SPANS -from .constants import TELEMETRY_PROFILING_CAPTURE_PCT -from .constants import TELEMETRY_PROFILING_EXPORT_LIBDD_ENABLED -from .constants import TELEMETRY_PROFILING_HEAP_ENABLED -from .constants import TELEMETRY_PROFILING_LOCK_ENABLED -from .constants import TELEMETRY_PROFILING_MAX_FRAMES -from .constants import TELEMETRY_PROFILING_MEMORY_ENABLED -from .constants import TELEMETRY_PROFILING_STACK_ENABLED -from .constants import TELEMETRY_PROFILING_UPLOAD_INTERVAL -from .constants import TELEMETRY_PROPAGATION_STYLE_EXTRACT -from .constants import TELEMETRY_PROPAGATION_STYLE_INJECT -from .constants import TELEMETRY_REMOTE_CONFIGURATION_ENABLED -from .constants import TELEMETRY_REMOTE_CONFIGURATION_INTERVAL -from .constants import TELEMETRY_RUNTIMEMETRICS_ENABLED -from .constants import TELEMETRY_SERVICE_MAPPING -from .constants import TELEMETRY_SPAN_SAMPLING_RULES -from .constants import TELEMETRY_SPAN_SAMPLING_RULES_FILE -from .constants import TELEMETRY_STARTUP_LOGS_ENABLED -from .constants import TELEMETRY_TRACE_AGENT_TIMEOUT_SECONDS -from .constants import TELEMETRY_TRACE_API_VERSION -from .constants import TELEMETRY_TRACE_COMPUTE_STATS -from .constants import TELEMETRY_TRACE_DEBUG -from .constants import TELEMETRY_TRACE_HEALTH_METRICS_ENABLED -from .constants import TELEMETRY_TRACE_PEER_SERVICE_DEFAULTS_ENABLED -from .constants import TELEMETRY_TRACE_PEER_SERVICE_MAPPING -from .constants import TELEMETRY_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED -from .constants import TELEMETRY_TRACE_SAMPLING_LIMIT -from .constants import TELEMETRY_TRACE_SPAN_ATTRIBUTE_SCHEMA -from .constants import TELEMETRY_TRACE_WRITER_BUFFER_SIZE_BYTES -from .constants import TELEMETRY_TRACE_WRITER_INTERVAL_SECONDS -from .constants import TELEMETRY_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES -from .constants import TELEMETRY_TRACE_WRITER_REUSE_CONNECTIONS from .constants import TELEMETRY_TYPE_DISTRIBUTION from .constants import TELEMETRY_TYPE_GENERATE_METRICS from .constants import TELEMETRY_TYPE_LOGS @@ -114,7 +47,23 @@ from .metrics_namespaces import NamespaceMetricType # noqa:F401 -log = get_logger(__name__) +log = getLogger(__name__) + + +class _TelemetryConfig: + API_KEY = os.environ.get("DD_API_KEY", None) + SITE = os.environ.get("DD_SITE", "datadoghq.com") + ENV = os.environ.get("DD_ENV", "") + SERVICE = os.environ.get("DD_SERVICE", "unnamed-python-service") + VERSION = os.environ.get("DD_VERSION", "") + AGENTLESS_MODE = asbool(os.environ.get("DD_CIVISIBILITY_AGENTLESS_ENABLED", False)) + HEARTBEAT_INTERVAL = float(os.environ.get("DD_TELEMETRY_HEARTBEAT_INTERVAL", "60")) + TELEMETRY_ENABLED = asbool(os.environ.get("DD_INSTRUMENTATION_TELEMETRY_ENABLED", "true").lower()) + DEPENDENCY_COLLECTION = asbool(os.environ.get("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", "true")) + INSTALL_ID = os.environ.get("DD_INSTRUMENTATION_INSTALL_ID", None) + INSTALL_TYPE = os.environ.get("DD_INSTRUMENTATION_INSTALL_TYPE", None) + INSTALL_TIME = os.environ.get("DD_INSTRUMENTATION_INSTALL_TIME", None) + FORCE_START = asbool(os.environ.get("_DD_INSTRUMENTATION_TELEMETRY_TESTS_FORCE_APP_STARTED", "false")) class LogData(dict): @@ -136,7 +85,7 @@ class _TelemetryClient: def __init__(self, agentless): # type: (bool) -> None - self._telemetry_url = self.get_host(config._dd_site, agentless) + self._telemetry_url = self.get_host(_TelemetryConfig.SITE, agentless) self._endpoint = self.get_endpoint(agentless) self._encoder = JSONEncoderV2() self._agentless = agentless @@ -147,8 +96,8 @@ def __init__(self, agentless): "DD-Client-Library-Version": _pep440_to_semver(), } - if agentless and config._dd_api_key: - self._headers["dd-api-key"] = config._dd_api_key + if agentless and _TelemetryConfig.API_KEY: + self._headers["dd-api-key"] = _TelemetryConfig.API_KEY @property def url(self): @@ -213,11 +162,11 @@ class TelemetryWriter(PeriodicService): def __init__(self, is_periodic=True, agentless=None): # type: (bool, Optional[bool]) -> None - super(TelemetryWriter, self).__init__(interval=min(config._telemetry_heartbeat_interval, 10)) + super(TelemetryWriter, self).__init__(interval=min(_TelemetryConfig.HEARTBEAT_INTERVAL, 10)) # Decouples the aggregation and sending of the telemetry events # TelemetryWriter events will only be sent when _periodic_count == _periodic_threshold. # By default this will occur at 10 second intervals. - self._periodic_threshold = int(config._telemetry_heartbeat_interval // self.interval) - 1 + self._periodic_threshold = int(_TelemetryConfig.HEARTBEAT_INTERVAL // self.interval) - 1 self._periodic_count = 0 self._is_periodic = is_periodic self._integrations_queue = dict() # type: Dict[str, Dict] @@ -238,14 +187,14 @@ def __init__(self, is_periodic=True, agentless=None): self.started = False # Debug flag that enables payload debug mode. - self._debug = asbool(os.environ.get("DD_TELEMETRY_DEBUG", "false")) + self._debug = os.environ.get("DD_TELEMETRY_DEBUG", "false").lower() in ("true", "1") - self._enabled = config._telemetry_enabled + self._enabled = _TelemetryConfig.TELEMETRY_ENABLED if agentless is None: - agentless = config._ci_visibility_agentless_enabled or config._dd_api_key is not None + agentless = _TelemetryConfig.AGENTLESS_MODE or _TelemetryConfig.API_KEY not in (None, "") - if agentless and not config._dd_api_key: + if agentless and not _TelemetryConfig.API_KEY: log.debug("Disabling telemetry: no Datadog API key found in agentless mode") self._enabled = False self._client = _TelemetryClient(agentless) @@ -263,7 +212,7 @@ def __init__(self, is_periodic=True, agentless=None): # This will occur when the agent writer starts. self.enable() # Force app started for unit tests - if asbool(os.environ.get("_DD_INSTRUMENTATION_TELEMETRY_TESTS_FORCE_APP_STARTED", "false")): + if _TelemetryConfig.FORCE_START: self._app_started() def enable(self): @@ -283,7 +232,7 @@ def enable(self): return True self.status = ServiceStatus.RUNNING - if config._telemetry_dependency_collection: + if _TelemetryConfig.DEPENDENCY_COLLECTION: modules.install_import_hook() return True @@ -330,7 +279,9 @@ def add_event(self, payload, payload_type): "api_version": "v2", "seq_id": next(self._sequence), "debug": self._debug, - "application": get_application(config.service, config.version, config.env), + "application": get_application( + _TelemetryConfig.SERVICE, _TelemetryConfig.VERSION, _TelemetryConfig.ENV + ), "host": get_host_info(), "payload": payload, "request_type": payload_type, @@ -369,49 +320,6 @@ def add_error(self, code, msg, filename, line_number): msg = "%s:%s: %s" % (filename, line_number, msg) self._error = (code, msg) - def add_configs_changed(self, cfg_names): - cs = [{"name": n, "value": v, "origin": o} for n, v, o in [self._telemetry_entry(n) for n in cfg_names]] - self._app_client_configuration_changed_event(cs) - - def _telemetry_entry(self, cfg_name: str) -> Tuple[str, str, _ConfigSource]: - item = config._config[cfg_name] - if cfg_name == "_profiling_enabled": - name = "profiling_enabled" - value = "true" if item.value() else "false" - elif cfg_name == "_asm_enabled": - name = "appsec_enabled" - value = "true" if item.value() else "false" - elif cfg_name == "_dsm_enabled": - name = "data_streams_enabled" - value = "true" if item.value() else "false" - elif cfg_name == "_trace_sample_rate": - name = "trace_sample_rate" - value = str(item.value()) - elif cfg_name == "_trace_sampling_rules": - name = "trace_sampling_rules" - value = str(item.value()) - elif cfg_name == "logs_injection": - name = "logs_injection_enabled" - value = "true" if item.value() else "false" - elif cfg_name == "trace_http_header_tags": - name = "trace_header_tags" - value = ",".join(":".join(x) for x in item.value().items()) - elif cfg_name == "tags": - name = "trace_tags" - value = ",".join(":".join(x) for x in item.value().items()) - elif cfg_name == "_tracing_enabled": - name = "trace_enabled" - value = "true" if item.value() else "false" - elif cfg_name == "_sca_enabled": - name = "DD_APPSEC_SCA_ENABLED" - if item.value() is None: - value = "" - else: - value = "true" if item.value() else "false" - else: - raise ValueError("Unknown configuration item: %s" % cfg_name) - return name, value, item.source() - def _app_started(self, register_app_shutdown=True): # type: (bool) -> None """Sent when TelemetryWriter is enabled or forks""" @@ -422,108 +330,14 @@ def _app_started(self, register_app_shutdown=True): self.started = True - inst_config_id_entry = ("instrumentation_config_id", "", "default") - if "DD_INSTRUMENTATION_CONFIG_ID" in os.environ: - inst_config_id_entry = ( - "instrumentation_config_id", - os.environ["DD_INSTRUMENTATION_CONFIG_ID"], - "env_var", - ) - - self.add_configurations( - [ - self._telemetry_entry("_profiling_enabled"), - self._telemetry_entry("_asm_enabled"), - self._telemetry_entry("_sca_enabled"), - self._telemetry_entry("_dsm_enabled"), - self._telemetry_entry("_trace_sample_rate"), - self._telemetry_entry("_trace_sampling_rules"), - self._telemetry_entry("logs_injection"), - self._telemetry_entry("trace_http_header_tags"), - self._telemetry_entry("tags"), - self._telemetry_entry("_tracing_enabled"), - inst_config_id_entry, - (TELEMETRY_STARTUP_LOGS_ENABLED, config._startup_logs_enabled, "unknown"), - (TELEMETRY_DYNAMIC_INSTRUMENTATION_ENABLED, di_config.enabled, "unknown"), - (TELEMETRY_EXCEPTION_REPLAY_ENABLED, er_config.enabled, "unknown"), - (TELEMETRY_PROPAGATION_STYLE_INJECT, ",".join(config._propagation_style_inject), "unknown"), - (TELEMETRY_PROPAGATION_STYLE_EXTRACT, ",".join(config._propagation_style_extract), "unknown"), - ("ddtrace_bootstrapped", config._ddtrace_bootstrapped, "unknown"), - ("ddtrace_auto_used", "ddtrace.auto" in sys.modules, "unknown"), - (TELEMETRY_RUNTIMEMETRICS_ENABLED, config._runtime_metrics_enabled, "unknown"), - (TELEMETRY_TRACE_DEBUG, config._debug_mode, "unknown"), - (TELEMETRY_ENABLED, config._telemetry_enabled, "unknown"), - (TELEMETRY_ANALYTICS_ENABLED, config.analytics_enabled, "unknown"), - (TELEMETRY_CLIENT_IP_ENABLED, config.client_ip_header, "unknown"), - (TELEMETRY_128_BIT_TRACEID_GENERATION_ENABLED, config._128_bit_trace_id_enabled, "unknown"), - (TELEMETRY_TRACE_COMPUTE_STATS, config._trace_compute_stats, "unknown"), - ( - TELEMETRY_OBFUSCATION_QUERY_STRING_PATTERN, - ( - config._obfuscation_query_string_pattern.pattern.decode("ascii") - if config._obfuscation_query_string_pattern - else "" - ), - "unknown", - ), - (TELEMETRY_OTEL_ENABLED, config._otel_enabled, "unknown"), - (TELEMETRY_TRACE_HEALTH_METRICS_ENABLED, config.health_metrics_enabled, "unknown"), - (TELEMETRY_RUNTIMEMETRICS_ENABLED, config._runtime_metrics_enabled, "unknown"), - (TELEMETRY_REMOTE_CONFIGURATION_ENABLED, config._remote_config_enabled, "unknown"), - (TELEMETRY_REMOTE_CONFIGURATION_INTERVAL, config._remote_config_poll_interval, "unknown"), - (TELEMETRY_TRACE_SAMPLING_LIMIT, config._trace_rate_limit, "unknown"), - (TELEMETRY_SPAN_SAMPLING_RULES, config._sampling_rules, "unknown"), - (TELEMETRY_SPAN_SAMPLING_RULES_FILE, config._sampling_rules_file, "unknown"), - (TELEMETRY_PARTIAL_FLUSH_ENABLED, config._partial_flush_enabled, "unknown"), - (TELEMETRY_PARTIAL_FLUSH_MIN_SPANS, config._partial_flush_min_spans, "unknown"), - (TELEMETRY_TRACE_SPAN_ATTRIBUTE_SCHEMA, SCHEMA_VERSION, "unknown"), - (TELEMETRY_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED, _remove_client_service_names, "unknown"), - (TELEMETRY_TRACE_PEER_SERVICE_DEFAULTS_ENABLED, _ps_config.set_defaults_enabled, "unknown"), - (TELEMETRY_TRACE_PEER_SERVICE_MAPPING, _ps_config._unparsed_peer_service_mapping, "unknown"), - (TELEMETRY_SERVICE_MAPPING, config._unparsed_service_mapping, "unknown"), - (TELEMETRY_TRACE_API_VERSION, config._trace_api, "unknown"), - (TELEMETRY_TRACE_WRITER_BUFFER_SIZE_BYTES, config._trace_writer_buffer_size, "unknown"), - (TELEMETRY_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES, config._trace_writer_payload_size, "unknown"), - (TELEMETRY_TRACE_WRITER_INTERVAL_SECONDS, config._trace_writer_interval_seconds, "unknown"), - (TELEMETRY_TRACE_WRITER_REUSE_CONNECTIONS, config._trace_writer_connection_reuse, "unknown"), - (TELEMETRY_DOGSTATSD_PORT, config._stats_agent_port, "unknown"), - (TELEMETRY_DOGSTATSD_URL, config._stats_agent_url, "unknown"), - (TELEMETRY_AGENT_HOST, config._trace_agent_hostname, "unknown"), - (TELEMETRY_AGENT_PORT, config._trace_agent_port, "unknown"), - (TELEMETRY_AGENT_URL, config._trace_agent_url, "unknown"), - (TELEMETRY_TRACE_AGENT_TIMEOUT_SECONDS, config._agent_timeout_seconds, "unknown"), - (TELEMETRY_PROFILING_STACK_ENABLED, prof_config.stack.enabled, "unknown"), - (TELEMETRY_PROFILING_MEMORY_ENABLED, prof_config.memory.enabled, "unknown"), - (TELEMETRY_PROFILING_HEAP_ENABLED, prof_config.heap.sample_size > 0, "unknown"), - (TELEMETRY_PROFILING_LOCK_ENABLED, prof_config.lock.enabled, "unknown"), - (TELEMETRY_PROFILING_EXPORT_LIBDD_ENABLED, prof_config.export.libdd_enabled, "unknown"), - (TELEMETRY_PROFILING_CAPTURE_PCT, prof_config.capture_pct, "unknown"), - (TELEMETRY_PROFILING_MAX_FRAMES, prof_config.max_frames, "unknown"), - (TELEMETRY_PROFILING_UPLOAD_INTERVAL, prof_config.upload_interval, "unknown"), - (TELEMETRY_INJECT_WAS_ATTEMPTED, config._inject_was_attempted, "unknown"), - (TELEMETRY_LIB_WAS_INJECTED, config._lib_was_injected, "unknown"), - (TELEMETRY_LIB_INJECTION_FORCED, config._inject_force, "unknown"), - # Crashtracker - (TELEMETRY_CRASHTRACKING_ENABLED, crashtracker_config.enabled, "unknown"), - (TELEMETRY_CRASHTRACKING_STARTED, crashtracking.is_started(), "unknown"), - (TELEMETRY_CRASHTRACKING_AVAILABLE, crashtracking.is_available, "unknown"), - (TELEMETRY_CRASHTRACKING_STACKTRACE_RESOLVER, str(crashtracker_config.stacktrace_resolver), "unknown"), - (TELEMETRY_CRASHTRACKING_STDOUT_FILENAME, str(crashtracker_config.stdout_filename), "unknown"), - (TELEMETRY_CRASHTRACKING_STDERR_FILENAME, str(crashtracker_config.stderr_filename), "unknown"), - (TELEMETRY_CRASHTRACKING_DEBUG_URL, str(crashtracker_config.debug_url), "unknown"), - (TELEMETRY_CRASHTRACKING_ALT_STACK, crashtracker_config.alt_stack, "unknown"), - ] - + get_python_config_vars() - ) - - if config._config["_sca_enabled"].value() is None: - self.remove_configuration("DD_APPSEC_SCA_ENABLED") - products = { product: {"version": _pep440_to_semver(), "enabled": status} for product, status in self._product_enablement.items() } + # SOABI should help us identify which wheels people are getting from PyPI + self.add_configurations(get_python_config_vars()) # type: ignore + payload = { "configuration": self._flush_configuration_queue(), "error": { @@ -533,11 +347,11 @@ def _app_started(self, register_app_shutdown=True): "products": products, } # type: Dict[str, Union[Dict[str, Any], List[Any]]] # Add time to value telemetry metrics for single step instrumentation - if config._telemetry_install_id or config._telemetry_install_type or config._telemetry_install_time: + if _TelemetryConfig.INSTALL_ID or _TelemetryConfig.INSTALL_TYPE or _TelemetryConfig.INSTALL_TIME: payload["install_signature"] = { - "install_id": config._telemetry_install_id, - "install_type": config._telemetry_install_type, - "install_time": config._telemetry_install_time, + "install_id": _TelemetryConfig.INSTALL_ID, + "install_type": _TelemetryConfig.INSTALL_TYPE, + "install_time": _TelemetryConfig.INSTALL_TIME, } # Reset the error after it has been reported. @@ -605,7 +419,7 @@ def _app_client_configuration_changed_event(self, configurations): def _app_dependencies_loaded_event(self, newly_imported_deps: List[str]): """Adds events to report imports done since the last periodic run""" - if not config._telemetry_dependency_collection or not self._enabled: + if not _TelemetryConfig.DEPENDENCY_COLLECTION or not self._enabled: return with self._lock: @@ -649,8 +463,12 @@ def remove_configuration(self, configuration_name): del self._configuration_queue[configuration_name] def add_configuration(self, configuration_name, configuration_value, origin="unknown"): - # type: (str, Union[bool, float, str], str) -> None + # type: (str, Any, str) -> None """Creates and queues the name, origin, value of a configuration""" + if not isinstance(configuration_value, (bool, str, int, float, type(None))): + # convert unsupported types to strings + configuration_value = str(configuration_value) + with self._lock: self._configuration_queue[configuration_name] = { "name": configuration_name, @@ -806,7 +624,7 @@ def periodic(self, force_flush=False, shutting_down=False): if configurations: self._app_client_configuration_changed_event(configurations) - if config._telemetry_dependency_collection: + if _TelemetryConfig.DEPENDENCY_COLLECTION: newly_imported_deps = self._flush_new_imported_dependencies() if newly_imported_deps: self._app_dependencies_loaded_event(newly_imported_deps) diff --git a/ddtrace/settings/_core.py b/ddtrace/settings/_core.py new file mode 100644 index 00000000000..15964726749 --- /dev/null +++ b/ddtrace/settings/_core.py @@ -0,0 +1,50 @@ +import os +from typing import Any # noqa:F401 +from typing import Callable # noqa:F401 +from typing import List # noqa:F401 +from typing import Optional # noqa:F401 +from typing import Union # noqa:F401 + +from envier.env import EnvVariable +from envier.env import _normalized + +from ddtrace.internal.telemetry import telemetry_writer + + +def report_telemetry(env: Any) -> None: + for name, e in list(env.__class__.__dict__.items()): + if isinstance(e, EnvVariable) and not e.private: + env_name = env._full_prefix + _normalized(e.name) + env_val = e(env, env._full_prefix) + raw_val = env.source.get(env_name) + if env_name in env.source and env_val == e._cast(e.type, raw_val, env): + source = "env_var" + elif env_val == e.default: + source = "default" + else: + source = "unknown" + telemetry_writer.add_configuration(env_name, env_val, source) + + +def get_config( + envs: Union[str, List[str]], + default: Any = None, + modifier: Optional[Callable[[Any], Any]] = None, + report_telemetry=True, +): + if isinstance(envs, str): + envs = [envs] + val = default + source = "default" + effective_env = envs[0] + for env in envs: + if env in os.environ: + val = os.environ[env] + if modifier: + val = modifier(val) + source = "env_var" + effective_env = env + break + if report_telemetry: + telemetry_writer.add_configuration(effective_env, val, source) + return val diff --git a/ddtrace/settings/asm.py b/ddtrace/settings/asm.py index 706305744f6..d3fb62b6c1e 100644 --- a/ddtrace/settings/asm.py +++ b/ddtrace/settings/asm.py @@ -17,6 +17,7 @@ from ddtrace.constants import APPSEC_ENV from ddtrace.constants import IAST_ENV from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.settings._core import report_telemetry as _report_telemetry from ddtrace.vendor.debtcollector import deprecate @@ -231,6 +232,7 @@ def _user_event_mode(self) -> str: config = ASMConfig() +_report_telemetry(config) if not config._asm_libddwaf_available: config._asm_enabled = False diff --git a/ddtrace/settings/config.py b/ddtrace/settings/config.py index 6de9774f25b..94e1163ce1d 100644 --- a/ddtrace/settings/config.py +++ b/ddtrace/settings/config.py @@ -14,6 +14,7 @@ from ddtrace.internal._file_queue import File_Queue from ddtrace.internal.serverless import in_azure_function from ddtrace.internal.serverless import in_gcp_function +from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.utils.cache import cachedmethod from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate @@ -34,6 +35,7 @@ from ..internal.utils.formats import asbool from ..internal.utils.formats import parse_tags_str from ..pin import Pin +from ._core import get_config as _get_config from ._otel_remapper import otel_remapping as _otel_remapping from .http import HttpConfig from .integration import IntegrationConfig @@ -94,8 +96,8 @@ ) -def _parse_propagation_styles(name, default): - # type: (str, Optional[str]) -> Optional[List[str]] +def _parse_propagation_styles(styles_str): + # type: (str) -> Optional[List[str]] """Helper to parse http propagation extract/inject styles via env variables. The expected format is:: @@ -131,10 +133,7 @@ def _parse_propagation_styles(name, default): DD_TRACE_PROPAGATION_STYLE_INJECT="b3" """ styles = [] - envvar = os.getenv(name, default=default) - if envvar is None: - return None - for style in envvar.split(","): + for style in styles_str.split(","): style = style.strip().lower() if style == "b3 single header": deprecate( @@ -147,7 +146,7 @@ def _parse_propagation_styles(name, default): if not style: continue if style not in PROPAGATION_STYLE_ALL: - log.warning("Unknown style {!r} provided for %r, allowed values are %r", style, name, PROPAGATION_STYLE_ALL) + log.warning("Unknown DD_TRACE_PROPAGATION_STYLE: {!r}, allowed values are %r", style, PROPAGATION_STYLE_ALL) continue styles.append(style) return styles @@ -201,6 +200,7 @@ class _ConfigItem: def __init__(self, name, default, envs): # type: (str, Union[_JSONType, Callable[[], _JSONType]], List[Tuple[str, Callable[[str], Any]]]) -> None + # _ConfigItem._name is only used in __repr__ and instrumentation telemetry self._name = name self._env_value: _JSONType = None self._code_value: _JSONType = None @@ -214,6 +214,7 @@ def __init__(self, name, default, envs): if env_var in os.environ: self._env_value = parser(os.environ[env_var]) break + telemetry_writer.add_configuration(name, self._telemetry_value(), self.source()) def set_value_source(self, value: Any, source: _ConfigSource) -> None: if source == "code": @@ -247,6 +248,16 @@ def source(self) -> _ConfigSource: return "env_var" return "default" + def _telemetry_value(self) -> str: + val = self.value() + if val is None: + return "" + elif isinstance(val, (bool, int, float)): + return str(val).lower() + elif isinstance(val, dict): + return ",".join(":".join((k, str(v))) for k, v in val.items()) + return str(val) + def __repr__(self): return "<{} name={} default={} env_value={} user_value={} remote_config_value={}>".format( self.__class__.__name__, @@ -276,22 +287,22 @@ def _default_config() -> Dict[str, _ConfigItem]: envs=[("DD_TRACE_SAMPLING_RULES", str)], ), "logs_injection": _ConfigItem( - name="logs_injection", + name="logs_injection_enabled", default=False, envs=[("DD_LOGS_INJECTION", asbool)], ), "trace_http_header_tags": _ConfigItem( - name="trace_http_header_tags", + name="trace_header_tags", default=lambda: {}, envs=[("DD_TRACE_HEADER_TAGS", parse_tags_str)], ), "tags": _ConfigItem( - name="tags", + name="trace_tags", default=lambda: {}, envs=[("DD_TAGS", _parse_global_tags)], ), "_tracing_enabled": _ConfigItem( - name="tracing_enabled", + name="trace_enabled", default=True, envs=[("DD_TRACE_ENABLED", asbool)], ), @@ -301,17 +312,17 @@ def _default_config() -> Dict[str, _ConfigItem]: envs=[("DD_PROFILING_ENABLED", asbool)], ), "_asm_enabled": _ConfigItem( - name="asm_enabled", + name="appsec_enabled", default=False, envs=[("DD_APPSEC_ENABLED", asbool)], ), "_sca_enabled": _ConfigItem( - name="sca_enabled", + name="DD_APPSEC_SCA_ENABLED", default=None, envs=[("DD_APPSEC_SCA_ENABLED", asbool)], ), "_dsm_enabled": _ConfigItem( - name="dsm_enabled", + name="data_streams_enabled", default=False, envs=[("DD_DATA_STREAMS_ENABLED", asbool)], ), @@ -326,7 +337,7 @@ class Config(object): """ class _HTTPServerConfig(object): - _error_statuses = os.getenv("DD_TRACE_HTTP_SERVER_ERROR_STATUSES", "500-599") # type: str + _error_statuses = _get_config("DD_TRACE_HTTP_SERVER_ERROR_STATUSES", "500-599") # type: str _error_ranges = get_error_ranges(_error_statuses) # type: List[Tuple[int, int]] @property @@ -370,6 +381,12 @@ def __init__(self): # Must come before _integration_configs due to __setattr__ self._config = _default_config() + # Remove the SCA configuration from instrumentation telemetry if it is not set + # this behavior is validated by system tests + # FIXME(munir): Is this really needed? Should report the default value (None) instead? + if self._config["_sca_enabled"].value() is None: + telemetry_writer.remove_configuration("DD_APPSEC_SCA_ENABLED") + sample_rate = os.getenv("DD_TRACE_SAMPLE_RATE") if sample_rate is not None: deprecate( @@ -381,8 +398,8 @@ def __init__(self): # Use a dict as underlying storing mechanism for integration configs self._integration_configs = {} - self._debug_mode = asbool(os.getenv("DD_TRACE_DEBUG", default=False)) - self._startup_logs_enabled = asbool(os.getenv("DD_TRACE_STARTUP_LOGS", False)) + self._debug_mode = _get_config("DD_TRACE_DEBUG", False, asbool) + self._startup_logs_enabled = _get_config("DD_TRACE_STARTUP_LOGS", False, asbool) rate_limit = os.getenv("DD_TRACE_RATE_LIMIT") if rate_limit is not None and self._trace_sampling_rules in ("", "[]"): @@ -395,18 +412,16 @@ def __init__(self): "All other spans will be rate limited by the Datadog Agent via DD_APM_MAX_TPS.", rate_limit, ) - self._trace_rate_limit = int(rate_limit or DEFAULT_SAMPLING_RATE_LIMIT) - self._partial_flush_enabled = asbool(os.getenv("DD_TRACE_PARTIAL_FLUSH_ENABLED", default=True)) - self._partial_flush_min_spans = int(os.getenv("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", default=300)) + self._trace_rate_limit = _get_config("DD_TRACE_RATE_LIMIT", DEFAULT_SAMPLING_RATE_LIMIT, int) + self._partial_flush_enabled = _get_config("DD_TRACE_PARTIAL_FLUSH_ENABLED", True, asbool) + self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 300, int) self.http = HttpConfig(header_tags=self.trace_http_header_tags) - self._remote_config_enabled = asbool(os.getenv("DD_REMOTE_CONFIGURATION_ENABLED", default=True)) - self._remote_config_poll_interval = float( - os.getenv( - "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS", default=os.getenv("DD_REMOTECONFIG_POLL_SECONDS", default=5.0) - ) + self._remote_config_enabled = _get_config("DD_REMOTE_CONFIGURATION_ENABLED", True, asbool) + self._remote_config_poll_interval = _get_config( + ["DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS", "DD_REMOTECONFIG_POLL_SECONDS"], 5.0, float ) - self._trace_api = os.getenv("DD_TRACE_API_VERSION") + self._trace_api = _get_config("DD_TRACE_API_VERSION") if self._trace_api == "v0.3": deprecate( "DD_TRACE_API_VERSION=v0.3 is deprecated", @@ -415,37 +430,33 @@ def __init__(self): category=DDTraceDeprecationWarning, ) self._trace_api = "v0.4" - self._trace_writer_buffer_size = int( - os.getenv("DD_TRACE_WRITER_BUFFER_SIZE_BYTES", default=DEFAULT_BUFFER_SIZE) - ) - self._trace_writer_payload_size = int( - os.getenv("DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES", default=DEFAULT_MAX_PAYLOAD_SIZE) + self._trace_writer_buffer_size = _get_config("DD_TRACE_WRITER_BUFFER_SIZE_BYTES", DEFAULT_BUFFER_SIZE, int) + self._trace_writer_payload_size = _get_config( + "DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES", DEFAULT_MAX_PAYLOAD_SIZE, int ) - self._trace_writer_interval_seconds = float( - os.getenv("DD_TRACE_WRITER_INTERVAL_SECONDS", default=DEFAULT_PROCESSING_INTERVAL) + self._trace_writer_interval_seconds = _get_config( + "DD_TRACE_WRITER_INTERVAL_SECONDS", DEFAULT_PROCESSING_INTERVAL, float ) - self._trace_writer_connection_reuse = asbool( - os.getenv("DD_TRACE_WRITER_REUSE_CONNECTIONS", DEFAULT_REUSE_CONNECTIONS) + self._trace_writer_connection_reuse = _get_config( + "DD_TRACE_WRITER_REUSE_CONNECTIONS", DEFAULT_REUSE_CONNECTIONS, asbool ) - self._trace_writer_log_err_payload = asbool(os.environ.get("_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS", False)) + self._trace_writer_log_err_payload = _get_config("_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS", False, asbool) - self._trace_agent_hostname = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_TRACE_AGENT_HOSTNAME")) - self._trace_agent_port = os.environ.get("DD_AGENT_PORT", os.environ.get("DD_TRACE_AGENT_PORT")) - self._trace_agent_url = os.environ.get("DD_TRACE_AGENT_URL") + self._trace_agent_hostname = _get_config(["DD_AGENT_HOST", "DD_TRACE_AGENT_HOSTNAME"]) + self._trace_agent_port = _get_config(["DD_AGENT_PORT", "DD_TRACE_AGENT_PORT"]) + self._trace_agent_url = _get_config("DD_TRACE_AGENT_URL") - self._stats_agent_hostname = os.environ.get("DD_AGENT_HOST", os.environ.get("DD_DOGSTATSD_HOST")) - self._stats_agent_port = os.getenv("DD_DOGSTATSD_PORT") - self._stats_agent_url = os.getenv("DD_DOGSTATSD_URL") - self._agent_timeout_seconds = float(os.getenv("DD_TRACE_AGENT_TIMEOUT_SECONDS", DEFAULT_TIMEOUT)) + self._stats_agent_hostname = _get_config(["DD_AGENT_HOST", "DD_DOGSTATSD_HOST"]) + self._stats_agent_port = _get_config("DD_DOGSTATSD_PORT") + self._stats_agent_url = _get_config("DD_DOGSTATSD_URL") + self._agent_timeout_seconds = _get_config("DD_TRACE_AGENT_TIMEOUT_SECONDS", DEFAULT_TIMEOUT, float) - self._span_traceback_max_size = int(os.getenv("DD_TRACE_SPAN_TRACEBACK_MAX_SIZE", default=30)) + self._span_traceback_max_size = _get_config("DD_TRACE_SPAN_TRACEBACK_MAX_SIZE", 30, int) # Master switch for turning on and off trace search by default # this weird invocation of getenv is meant to read the DD_ANALYTICS_ENABLED # legacy environment variable. It should be removed in the future - self.analytics_enabled = asbool( - os.getenv("DD_TRACE_ANALYTICS_ENABLED", default=os.getenv("DD_ANALYTICS_ENABLED", default=False)) - ) + self.analytics_enabled = _get_config(["DD_TRACE_ANALYTICS_ENABLED", "DD_ANALYTICS_ENABLED"], False, asbool) if self.analytics_enabled: deprecate( "Datadog App Analytics is deprecated and will be removed in a future version. " @@ -455,27 +466,25 @@ def __init__(self): category=DDTraceDeprecationWarning, ) - self.client_ip_header = os.getenv("DD_TRACE_CLIENT_IP_HEADER") - self.retrieve_client_ip = asbool(os.getenv("DD_TRACE_CLIENT_IP_ENABLED", default=False)) + self.client_ip_header = _get_config("DD_TRACE_CLIENT_IP_HEADER") + self.retrieve_client_ip = _get_config("DD_TRACE_CLIENT_IP_ENABLED", False, asbool) - self.propagation_http_baggage_enabled = asbool( - os.getenv("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", default=False) - ) + self.propagation_http_baggage_enabled = _get_config("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", False, asbool) - self.env = os.getenv("DD_ENV") or self.tags.get("env") - self.service = os.getenv("DD_SERVICE", default=self.tags.get("service", DEFAULT_SPAN_SERVICE_NAME)) + self.env = _get_config("DD_ENV", self.tags.get("env")) + self.service = _get_config("DD_SERVICE", self.tags.get("service", DEFAULT_SPAN_SERVICE_NAME)) if self.service is None and in_gcp_function(): - self.service = os.environ.get("K_SERVICE", os.environ.get("FUNCTION_NAME")) + self.service = _get_config(["K_SERVICE", "FUNCTION_NAME"], DEFAULT_SPAN_SERVICE_NAME) if self.service is None and in_azure_function(): - self.service = os.environ.get("WEBSITE_SITE_NAME") + self.service = _get_config("WEBSITE_SITE_NAME", DEFAULT_SPAN_SERVICE_NAME) self._extra_services = set() self._extra_services_queue = None if in_aws_lambda() or not self._remote_config_enabled else File_Queue() - self.version = os.getenv("DD_VERSION", default=self.tags.get("version")) + self.version = _get_config("DD_VERSION", self.tags.get("version")) self.http_server = self._HTTPServerConfig() - self._unparsed_service_mapping = os.getenv("DD_SERVICE_MAPPING", default="") + self._unparsed_service_mapping = _get_config("DD_SERVICE_MAPPING", "") self.service_mapping = parse_tags_str(self._unparsed_service_mapping) # The service tag corresponds to span.service and should not be @@ -487,19 +496,19 @@ def __init__(self): if self.version and "version" in self.tags: del self.tags["version"] - self.report_hostname = asbool(os.getenv("DD_TRACE_REPORT_HOSTNAME", default=False)) + self.report_hostname = _get_config("DD_TRACE_REPORT_HOSTNAME", False, asbool) - self.health_metrics_enabled = asbool(os.getenv("DD_TRACE_HEALTH_METRICS_ENABLED", default=False)) + self.health_metrics_enabled = _get_config("DD_TRACE_HEALTH_METRICS_ENABLED", False, asbool) - self._telemetry_enabled = asbool(os.getenv("DD_INSTRUMENTATION_TELEMETRY_ENABLED", True)) - self._telemetry_heartbeat_interval = float(os.getenv("DD_TELEMETRY_HEARTBEAT_INTERVAL", "60")) - self._telemetry_dependency_collection = asbool(os.getenv("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", True)) + self._telemetry_enabled = _get_config("DD_INSTRUMENTATION_TELEMETRY_ENABLED", True, asbool) + self._telemetry_heartbeat_interval = _get_config("DD_TELEMETRY_HEARTBEAT_INTERVAL", 60, float) + self._telemetry_dependency_collection = _get_config("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", True, asbool) - self._runtime_metrics_enabled = asbool(os.getenv("DD_RUNTIME_METRICS_ENABLED", False)) + self._runtime_metrics_enabled = _get_config("DD_RUNTIME_METRICS_ENABLED", False, asbool) - self._128_bit_trace_id_enabled = asbool(os.getenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", True)) + self._128_bit_trace_id_enabled = _get_config("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", True, asbool) - self._128_bit_trace_id_logging_enabled = asbool(os.getenv("DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED", False)) + self._128_bit_trace_id_logging_enabled = _get_config("DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED", False, asbool) if self._128_bit_trace_id_logging_enabled: deprecate( "Using DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED is deprecated.", @@ -508,27 +517,25 @@ def __init__(self): category=DDTraceDeprecationWarning, ) - self._sampling_rules = os.getenv("DD_SPAN_SAMPLING_RULES") - self._sampling_rules_file = os.getenv("DD_SPAN_SAMPLING_RULES_FILE") + self._sampling_rules = _get_config("DD_SPAN_SAMPLING_RULES") + self._sampling_rules_file = _get_config("DD_SPAN_SAMPLING_RULES_FILE") # Propagation styles - self._propagation_style_extract = self._propagation_style_inject = _parse_propagation_styles( - "DD_TRACE_PROPAGATION_STYLE", default=_PROPAGATION_STYLE_DEFAULT - ) # DD_TRACE_PROPAGATION_STYLE_EXTRACT and DD_TRACE_PROPAGATION_STYLE_INJECT # take precedence over DD_TRACE_PROPAGATION_STYLE - propagation_style_extract = _parse_propagation_styles("DD_TRACE_PROPAGATION_STYLE_EXTRACT", default=None) - if propagation_style_extract is not None: - self._propagation_style_extract = propagation_style_extract - - propagation_style_inject = _parse_propagation_styles("DD_TRACE_PROPAGATION_STYLE_INJECT", default=None) - if propagation_style_inject is not None: - self._propagation_style_inject = propagation_style_inject + self._propagation_style_extract = _parse_propagation_styles( + _get_config( + ["DD_TRACE_PROPAGATION_STYLE_EXTRACT", "DD_TRACE_PROPAGATION_STYLE"], _PROPAGATION_STYLE_DEFAULT + ) + ) + self._propagation_style_inject = _parse_propagation_styles( + _get_config(["DD_TRACE_PROPAGATION_STYLE_INJECT", "DD_TRACE_PROPAGATION_STYLE"], _PROPAGATION_STYLE_DEFAULT) + ) - self._propagation_extract_first = asbool(os.getenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", False)) + self._propagation_extract_first = _get_config("DD_TRACE_PROPAGATION_EXTRACT_FIRST", False, asbool) # Datadog tracer tags propagation - x_datadog_tags_max_length = int(os.getenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", default=512)) + x_datadog_tags_max_length = _get_config("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", 512, int) if x_datadog_tags_max_length < 0: log.warning( ( @@ -542,19 +549,17 @@ def __init__(self): self._x_datadog_tags_enabled = x_datadog_tags_max_length > 0 # Raise certain errors only if in testing raise mode to prevent crashing in production with non-critical errors - self._raise = asbool(os.getenv("DD_TESTING_RAISE", False)) + self._raise = _get_config("DD_TESTING_RAISE", False, asbool) trace_compute_stats_default = in_gcp_function() or in_azure_function() - self._trace_compute_stats = asbool( - os.getenv( - "DD_TRACE_COMPUTE_STATS", os.getenv("DD_TRACE_STATS_COMPUTATION_ENABLED", trace_compute_stats_default) - ) + self._trace_compute_stats = _get_config( + ["DD_TRACE_COMPUTE_STATS", "DD_TRACE_STATS_COMPUTATION_ENABLED"], trace_compute_stats_default, asbool ) - self._data_streams_enabled = asbool(os.getenv("DD_DATA_STREAMS_ENABLED", False)) + self._data_streams_enabled = _get_config("DD_DATA_STREAMS_ENABLED", False, asbool) - legacy_client_tag_enabled = os.getenv("DD_HTTP_CLIENT_TAG_QUERY_STRING", None) + legacy_client_tag_enabled = _get_config("DD_HTTP_CLIENT_TAG_QUERY_STRING") if legacy_client_tag_enabled is None: - self._http_client_tag_query_string = os.getenv("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING", default="true") + self._http_client_tag_query_string = _get_config("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING", "true") else: deprecate( "DD_HTTP_CLIENT_TAG_QUERY_STRING is deprecated", @@ -564,7 +569,7 @@ def __init__(self): ) self._http_client_tag_query_string = legacy_client_tag_enabled.lower() - dd_trace_obfuscation_query_string_regexp = os.getenv( + dd_trace_obfuscation_query_string_regexp = _get_config( "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT ) self.global_query_string_obfuscation_disabled = dd_trace_obfuscation_query_string_regexp == "" @@ -578,26 +583,21 @@ def __init__(self): log.warning("Invalid obfuscation pattern, disabling query string tracing", exc_info=True) self.http_tag_query_string = False # Disable query string tagging if malformed obfuscation pattern - # Test Visibility config items - self._ci_visibility_agentless_enabled = asbool(os.getenv("DD_CIVISIBILITY_AGENTLESS_ENABLED", default=False)) - self._ci_visibility_agentless_url = os.getenv("DD_CIVISIBILITY_AGENTLESS_URL", default="") - self._ci_visibility_intelligent_testrunner_enabled = asbool( - os.getenv("DD_CIVISIBILITY_ITR_ENABLED", default=True) + self._ci_visibility_agentless_enabled = _get_config("DD_CIVISIBILITY_AGENTLESS_ENABLED", False, asbool) + self._ci_visibility_agentless_url = _get_config("DD_CIVISIBILITY_AGENTLESS_URL", "") + self._ci_visibility_intelligent_testrunner_enabled = _get_config("DD_CIVISIBILITY_ITR_ENABLED", True, asbool) + self.ci_visibility_log_level = _get_config("DD_CIVISIBILITY_LOG_LEVEL", "info") + self.test_session_name = _get_config("DD_TEST_SESSION_NAME") + self._test_visibility_early_flake_detection_enabled = _get_config( + "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED", True, asbool ) - self.ci_visibility_log_level = os.getenv("DD_CIVISIBILITY_LOG_LEVEL", default="info") - self.test_session_name = os.getenv("DD_TEST_SESSION_NAME") - self._test_visibility_early_flake_detection_enabled = asbool( - os.getenv("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED", default=True) - ) - - self._otel_enabled = asbool(os.getenv("DD_TRACE_OTEL_ENABLED", False)) + self._otel_enabled = _get_config("DD_TRACE_OTEL_ENABLED", False, asbool) if self._otel_enabled: # Replaces the default otel api runtime context with DDRuntimeContext # https://github.com/open-telemetry/opentelemetry-python/blob/v1.16.0/opentelemetry-api/src/opentelemetry/context/__init__.py#L53 os.environ["OTEL_PYTHON_CONTEXT"] = "ddcontextvars_context" - self._ddtrace_bootstrapped = False self._subscriptions = [] # type: List[Tuple[List[str], Callable[[Config, List[str]], None]]] - self._span_aggregator_rlock = asbool(os.getenv("DD_TRACE_SPAN_AGGREGATOR_RLOCK", True)) + self._span_aggregator_rlock = _get_config("DD_TRACE_SPAN_AGGREGATOR_RLOCK", True, asbool) if self._span_aggregator_rlock is False: deprecate( "DD_TRACE_SPAN_AGGREGATOR_RLOCK is deprecated", @@ -607,23 +607,23 @@ def __init__(self): removal_version="3.0.0", ) - self.trace_methods = os.getenv("DD_TRACE_METHODS") + self.trace_methods = _get_config("DD_TRACE_METHODS") - self._telemetry_install_id = os.getenv("DD_INSTRUMENTATION_INSTALL_ID", None) - self._telemetry_install_type = os.getenv("DD_INSTRUMENTATION_INSTALL_TYPE", None) - self._telemetry_install_time = os.getenv("DD_INSTRUMENTATION_INSTALL_TIME", None) + self._telemetry_install_id = _get_config("DD_INSTRUMENTATION_INSTALL_ID") + self._telemetry_install_type = _get_config("DD_INSTRUMENTATION_INSTALL_TYPE") + self._telemetry_install_time = _get_config("DD_INSTRUMENTATION_INSTALL_TYPE") - self._dd_api_key = os.getenv("DD_API_KEY") - self._dd_site = os.getenv("DD_SITE", "datadoghq.com") + self._dd_api_key = _get_config("DD_API_KEY") + self._dd_site = _get_config("DD_SITE", "datadoghq.com") - self._llmobs_enabled = asbool(os.getenv("DD_LLMOBS_ENABLED", False)) - self._llmobs_sample_rate = float(os.getenv("DD_LLMOBS_SAMPLE_RATE", 1.0)) - self._llmobs_ml_app = os.getenv("DD_LLMOBS_ML_APP") - self._llmobs_agentless_enabled = asbool(os.getenv("DD_LLMOBS_AGENTLESS_ENABLED", False)) + self._llmobs_enabled = _get_config("DD_LLMOBS_ENABLED", False, asbool) + self._llmobs_sample_rate = _get_config("DD_LLMOBS_SAMPLE_RATE", 1.0, float) + self._llmobs_ml_app = _get_config("DD_LLMOBS_ML_APP") + self._llmobs_agentless_enabled = _get_config("DD_LLMOBS_AGENTLESS_ENABLED", False, asbool) - self._inject_force = asbool(os.getenv("DD_INJECT_FORCE", False)) + self._inject_force = _get_config("DD_INJECT_FORCE", False, asbool) self._lib_was_injected = False - self._inject_was_attempted = asbool(os.getenv("_DD_INJECT_WAS_ATTEMPTED", False)) + self._inject_was_attempted = _get_config("_DD_INJECT_WAS_ATTEMPTED", False, asbool) def __getattr__(self, name) -> Any: if name in self._config: @@ -761,11 +761,10 @@ def _set_config_items(self, items): item_names = [] for key, value, origin in items: item_names.append(key) - self._config[key].set_value_source(value, origin) - if self._telemetry_enabled: - from ..internal.telemetry import telemetry_writer - - telemetry_writer.add_configs_changed(item_names) + item = self._config[key] + item.set_value_source(value, origin) + if self._telemetry_enabled: + telemetry_writer.add_configuration(item._name, item._telemetry_value(), item.source()) self._notify_subscribers(item_names) def _reset(self): diff --git a/ddtrace/settings/crashtracker.py b/ddtrace/settings/crashtracker.py index f22b363f56a..07d838693d8 100644 --- a/ddtrace/settings/crashtracker.py +++ b/ddtrace/settings/crashtracker.py @@ -3,6 +3,7 @@ from envier import En from ddtrace.internal.utils.formats import parse_tags_str +from ddtrace.settings._core import report_telemetry as _report_telemetry resolver_default = "full" @@ -111,3 +112,4 @@ class CrashtrackingConfig(En): config = CrashtrackingConfig() +_report_telemetry(config) diff --git a/ddtrace/settings/dynamic_instrumentation.py b/ddtrace/settings/dynamic_instrumentation.py index fcd57d4e593..27008bec692 100644 --- a/ddtrace/settings/dynamic_instrumentation.py +++ b/ddtrace/settings/dynamic_instrumentation.py @@ -8,6 +8,7 @@ from ddtrace.internal.constants import DEFAULT_SERVICE_NAME from ddtrace.internal.utils.config import get_application_name from ddtrace.settings import _config as ddconfig +from ddtrace.settings._core import report_telemetry as _report_telemetry from ddtrace.version import get_version @@ -131,3 +132,4 @@ class DynamicInstrumentationConfig(En): config = DynamicInstrumentationConfig() +_report_telemetry(config) diff --git a/ddtrace/settings/exception_replay.py b/ddtrace/settings/exception_replay.py index d85e38b1fad..c78ae29b73d 100644 --- a/ddtrace/settings/exception_replay.py +++ b/ddtrace/settings/exception_replay.py @@ -1,5 +1,7 @@ from envier import En +from ddtrace.settings._core import report_telemetry as _report_telemetry + class ExceptionReplayConfig(En): __prefix__ = "dd.exception" @@ -15,3 +17,4 @@ class ExceptionReplayConfig(En): config = ExceptionReplayConfig() +_report_telemetry(config) diff --git a/ddtrace/settings/peer_service.py b/ddtrace/settings/peer_service.py index 46ce6bf2aec..7bc56849a95 100644 --- a/ddtrace/settings/peer_service.py +++ b/ddtrace/settings/peer_service.py @@ -4,6 +4,7 @@ from ddtrace.internal.schema import SCHEMA_VERSION from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import parse_tags_str +from ddtrace.settings._core import report_telemetry as _report_telemetry class PeerServiceConfig(object): @@ -37,3 +38,4 @@ def peer_service_mapping(self): _ps_config = PeerServiceConfig() +_report_telemetry(_ps_config) diff --git a/ddtrace/settings/profiling.py b/ddtrace/settings/profiling.py index 03b59b39cae..b77cde1bb3c 100644 --- a/ddtrace/settings/profiling.py +++ b/ddtrace/settings/profiling.py @@ -13,6 +13,7 @@ from ddtrace.internal import gitmetadata from ddtrace.internal.logger import get_logger from ddtrace.internal.utils.formats import parse_tags_str +from ddtrace.settings._core import report_telemetry as _report_telemetry logger = get_logger(__name__) @@ -418,6 +419,7 @@ class ProfilingConfigExport(En): ProfilingConfig.include(ProfilingConfigExport, namespace="export") config = ProfilingConfig() +_report_telemetry(config) # If during processing we discover that the configuration was injected, we need to do a few things # - Mark it as such diff --git a/ddtrace/settings/symbol_db.py b/ddtrace/settings/symbol_db.py index 59e46d917cf..2203b6f3f75 100644 --- a/ddtrace/settings/symbol_db.py +++ b/ddtrace/settings/symbol_db.py @@ -2,6 +2,8 @@ from envier import En +from ddtrace.settings._core import report_telemetry as _report_telemetry + class SymbolDatabaseConfig(En): __prefix__ = "dd.symbol_database" @@ -36,3 +38,4 @@ class SymbolDatabaseConfig(En): config = SymbolDatabaseConfig() +_report_telemetry(config) diff --git a/tests/conftest.py b/tests/conftest.py index 82638f4f43f..2c99e7a9e4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -576,6 +576,17 @@ def get_dependencies(self, name=None): deps.sort(key=lambda x: x["name"], reverse=False) return deps + def get_configurations(self, name=None, ignores=None): + ignores = ignores or [] + configurations = [] + events_with_configs = self.get_events("app-started") + self.get_events("app-client-configuration-change") + for event in events_with_configs: + for c in event["payload"]["configuration"]: + if c["name"] == name or (name is None and c["name"] not in ignores): + configurations.append(c) + configurations.sort(key=lambda x: x["name"], reverse=False) + return configurations + @pytest.fixture def test_agent_session(telemetry_writer, request): diff --git a/tests/integration/test_settings.py b/tests/integration/test_settings.py index af6add6e49c..08e9aadb530 100644 --- a/tests/integration/test_settings.py +++ b/tests/integration/test_settings.py @@ -175,7 +175,7 @@ def test_remoteconfig_sampling_rate_default(test_agent_session, run_python_code_ events = test_agent_session.get_events() events_trace_sample_rate = _get_telemetry_config_items(events, "trace_sample_rate") - assert {"name": "trace_sample_rate", "value": "0.5", "origin": "remote_config"} in events_trace_sample_rate + assert {"name": "trace_sample_rate", "value": "1.0", "origin": "default"} in events_trace_sample_rate @pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py index 058d01f1c46..ef66017a078 100644 --- a/tests/telemetry/test_writer.py +++ b/tests/telemetry/test_writer.py @@ -52,6 +52,33 @@ def test_add_event_disabled_writer(telemetry_writer, test_agent_session): assert len(test_agent_session.get_requests(payload_type)) == 1 +@pytest.mark.parametrize( + "env_var,value,expected_value", + [ + ("DD_APPSEC_SCA_ENABLED", "true", "true"), + ("DD_APPSEC_SCA_ENABLED", "True", "true"), + ("DD_APPSEC_SCA_ENABLED", "1", "true"), + ("DD_APPSEC_SCA_ENABLED", "false", "false"), + ("DD_APPSEC_SCA_ENABLED", "False", "false"), + ("DD_APPSEC_SCA_ENABLED", "0", "false"), + ], +) +def test_app_started_event_configuration_override_asm( + test_agent_session, run_python_code_in_subprocess, env_var, value, expected_value +): + """asserts that asm configuration value is changed and queues a valid telemetry request""" + env = os.environ.copy() + env["_DD_INSTRUMENTATION_TELEMETRY_TESTS_FORCE_APP_STARTED"] = "true" + env["DD_APPSEC_ENABLED"] = "true" + env[env_var] = value + _, stderr, status, _ = run_python_code_in_subprocess("import ddtrace.auto", env=env) + assert status == 0, stderr + + configuration = test_agent_session.get_configurations(name=env_var) + assert len(configuration) == 1, configuration + assert configuration[0] == {"name": env_var, "origin": "env_var", "value": expected_value} + + def test_app_started_event(telemetry_writer, test_agent_session, mock_time): """asserts that app_started() queues a valid telemetry request which is then sent by periodic()""" with override_global_config(dict(_telemetry_dependency_collection=False)): @@ -127,12 +154,12 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time): {"name": "appsec_enabled", "origin": "default", "value": "false"}, {"name": "crashtracking_alt_stack", "origin": "unknown", "value": False}, {"name": "crashtracking_available", "origin": "unknown", "value": sys.platform == "linux"}, - {"name": "crashtracking_debug_url", "origin": "unknown", "value": "None"}, + {"name": "crashtracking_debug_url", "origin": "unknown", "value": None}, {"name": "crashtracking_enabled", "origin": "unknown", "value": sys.platform == "linux"}, {"name": "crashtracking_stacktrace_resolver", "origin": "unknown", "value": "full"}, {"name": "crashtracking_started", "origin": "unknown", "value": False}, - {"name": "crashtracking_stderr_filename", "origin": "unknown", "value": "None"}, - {"name": "crashtracking_stdout_filename", "origin": "unknown", "value": "None"}, + {"name": "crashtracking_stderr_filename", "origin": "unknown", "value": None}, + {"name": "crashtracking_stdout_filename", "origin": "unknown", "value": None}, { "name": "python_build_gnu_type", "origin": "unknown", @@ -170,20 +197,7 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time): assert payload == result["payload"] -@pytest.mark.parametrize( - "env_var,value,expected_value", - [ - ("DD_APPSEC_SCA_ENABLED", "true", "true"), - ("DD_APPSEC_SCA_ENABLED", "True", "true"), - ("DD_APPSEC_SCA_ENABLED", "1", "true"), - ("DD_APPSEC_SCA_ENABLED", "false", "false"), - ("DD_APPSEC_SCA_ENABLED", "False", "false"), - ("DD_APPSEC_SCA_ENABLED", "0", "false"), - ], -) -def test_app_started_event_configuration_override( - test_agent_session, run_python_code_in_subprocess, tmpdir, env_var, value, expected_value -): +def test_app_started_event_configuration_override(test_agent_session, run_python_code_in_subprocess, tmpdir): """ asserts that default configuration value is changed and queues a valid telemetry request @@ -249,7 +263,6 @@ def test_app_started_event_configuration_override( env["DD_TRACE_WRITER_REUSE_CONNECTIONS"] = "True" env["DD_TAGS"] = "team:apm,component:web" env["DD_INSTRUMENTATION_CONFIG_ID"] = "abcedf123" - env[env_var] = value file = tmpdir.join("moon_ears.json") file.write('[{"service":"xy?","name":"a*c"}]') @@ -257,98 +270,203 @@ def test_app_started_event_configuration_override( env["DD_SPAN_SAMPLING_RULES_FILE"] = str(file) env["DD_TRACE_PARTIAL_FLUSH_ENABLED"] = "false" env["DD_TRACE_PARTIAL_FLUSH_MIN_SPANS"] = "3" + env["DD_SITE"] = "datadoghq.com" _, stderr, status, _ = run_python_code_in_subprocess(code, env=env) assert status == 0, stderr - app_started_events = test_agent_session.get_events("app-started") - assert len(app_started_events) == 1 - - app_started_events[0]["payload"]["configuration"].sort(key=lambda c: c["name"]) - result = sorted(app_started_events[0]["payload"]["configuration"], key=lambda x: x["name"]) - result = [k for k in result if k["name"] != "DD_TRACE_AGENT_URL"] - expected = sorted( - [ - {"name": "DD_AGENT_HOST", "origin": "unknown", "value": None}, - {"name": "DD_AGENT_PORT", "origin": "unknown", "value": None}, - {"name": env_var, "origin": "env_var", "value": expected_value}, - {"name": "DD_DOGSTATSD_PORT", "origin": "unknown", "value": None}, - {"name": "DD_DOGSTATSD_URL", "origin": "unknown", "value": None}, - {"name": "DD_DYNAMIC_INSTRUMENTATION_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_EXCEPTION_REPLAY_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_INSTRUMENTATION_TELEMETRY_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_PROFILING_STACK_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_PROFILING_MEMORY_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_PROFILING_HEAP_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_PROFILING_LOCK_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_PROFILING_EXPORT_LIBDD_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_PROFILING_CAPTURE_PCT", "origin": "unknown", "value": 5.0}, - {"name": "DD_PROFILING_UPLOAD_INTERVAL", "origin": "unknown", "value": 10.0}, - {"name": "DD_PROFILING_MAX_FRAMES", "origin": "unknown", "value": 512}, - {"name": "DD_REMOTE_CONFIGURATION_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS", "origin": "unknown", "value": 1.0}, - {"name": "DD_RUNTIME_METRICS_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_SERVICE_MAPPING", "origin": "unknown", "value": "default_dd_service:remapped_dd_service"}, - {"name": "DD_SPAN_SAMPLING_RULES", "origin": "unknown", "value": '[{"service":"xyz", "sample_rate":0.23}]'}, - {"name": "DD_SPAN_SAMPLING_RULES_FILE", "origin": "unknown", "value": str(file)}, - {"name": "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_AGENT_TIMEOUT_SECONDS", "origin": "unknown", "value": 2.0}, - {"name": "DD_TRACE_ANALYTICS_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_API_VERSION", "origin": "unknown", "value": "v0.5"}, - {"name": "DD_TRACE_CLIENT_IP_ENABLED", "origin": "unknown", "value": None}, - {"name": "DD_TRACE_COMPUTE_STATS", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_DEBUG", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_HEALTH_METRICS_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", "origin": "unknown", "value": ".*"}, - {"name": "DD_TRACE_OTEL_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_PARTIAL_FLUSH_ENABLED", "origin": "unknown", "value": False}, - {"name": "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", "origin": "unknown", "value": 3}, - {"name": "DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_PEER_SERVICE_MAPPING", "origin": "unknown", "value": "default_service:remapped_service"}, - {"name": "DD_TRACE_PROPAGATION_STYLE_EXTRACT", "origin": "unknown", "value": "tracecontext"}, - {"name": "DD_TRACE_PROPAGATION_STYLE_INJECT", "origin": "unknown", "value": "tracecontext"}, - {"name": "DD_TRACE_RATE_LIMIT", "origin": "unknown", "value": 50}, - {"name": "DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", "origin": "unknown", "value": "v1"}, - {"name": "DD_TRACE_STARTUP_LOGS", "origin": "unknown", "value": True}, - {"name": "DD_TRACE_WRITER_BUFFER_SIZE_BYTES", "origin": "unknown", "value": 1000}, - {"name": "DD_TRACE_WRITER_INTERVAL_SECONDS", "origin": "unknown", "value": 30.0}, - {"name": "DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES", "origin": "unknown", "value": 9999}, - {"name": "DD_TRACE_WRITER_REUSE_CONNECTIONS", "origin": "unknown", "value": True}, - {"name": "ddtrace_auto_used", "origin": "unknown", "value": True}, - {"name": "ddtrace_bootstrapped", "origin": "unknown", "value": True}, - {"name": "trace_enabled", "origin": "env_var", "value": "false"}, - {"name": "profiling_enabled", "origin": "env_var", "value": "true"}, - {"name": "data_streams_enabled", "origin": "env_var", "value": "true"}, - {"name": "appsec_enabled", "origin": "env_var", "value": "true"}, - {"name": "crashtracking_alt_stack", "origin": "unknown", "value": False}, - {"name": "crashtracking_available", "origin": "unknown", "value": sys.platform == "linux"}, - {"name": "crashtracking_debug_url", "origin": "unknown", "value": "None"}, - {"name": "crashtracking_enabled", "origin": "unknown", "value": sys.platform == "linux"}, - {"name": "crashtracking_stacktrace_resolver", "origin": "unknown", "value": "full"}, - {"name": "crashtracking_started", "origin": "unknown", "value": sys.platform == "linux"}, - {"name": "crashtracking_stderr_filename", "origin": "unknown", "value": "None"}, - {"name": "crashtracking_stdout_filename", "origin": "unknown", "value": "None"}, - {"name": "python_build_gnu_type", "origin": "unknown", "value": sysconfig.get_config_var("BUILD_GNU_TYPE")}, - {"name": "python_host_gnu_type", "origin": "unknown", "value": sysconfig.get_config_var("HOST_GNU_TYPE")}, - {"name": "python_soabi", "origin": "unknown", "value": sysconfig.get_config_var("SOABI")}, - {"name": "trace_sample_rate", "origin": "env_var", "value": "0.5"}, - { - "name": "trace_sampling_rules", - "origin": "env_var", - "value": '[{"sample_rate":1.0,"service":"xyz","name":"abc"}]', - }, - {"name": "logs_injection_enabled", "origin": "env_var", "value": "true"}, - {"name": "trace_header_tags", "origin": "default", "value": ""}, - {"name": "trace_tags", "origin": "env_var", "value": "team:apm,component:web"}, - {"name": "instrumentation_config_id", "origin": "env_var", "value": "abcedf123"}, - {"name": "DD_INJECT_FORCE", "origin": "unknown", "value": True}, - {"name": "DD_LIB_INJECTED", "origin": "unknown", "value": False}, - {"name": "DD_LIB_INJECTION_ATTEMPTED", "origin": "unknown", "value": False}, - ], - key=lambda x: x["name"], - ) - assert result == expected + # DD_TRACE_AGENT_URL in gitlab is different from CI, to keep things simple we will + # skip validating this config + configurations = test_agent_session.get_configurations(ignores=["DD_TRACE_AGENT_URL"]) + assert configurations + + expected = [ + {"name": "DD_AGENT_HOST", "origin": "default", "value": None}, + {"name": "DD_AGENT_PORT", "origin": "default", "value": None}, + {"name": "DD_API_KEY", "origin": "default", "value": None}, + {"name": "DD_API_SECURITY_ENABLED", "origin": "default", "value": True}, + {"name": "DD_API_SECURITY_PARSE_RESPONSE_BODY", "origin": "default", "value": True}, + {"name": "DD_API_SECURITY_SAMPLE_DELAY", "origin": "default", "value": 30.0}, + {"name": "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING", "origin": "default", "value": ""}, + {"name": "DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING_ENABLED", "origin": "default", "value": True}, + {"name": "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE", "origin": "default", "value": ""}, + {"name": "DD_APPSEC_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_APPSEC_MAX_STACK_TRACES", "origin": "default", "value": 2}, + {"name": "DD_APPSEC_MAX_STACK_TRACE_DEPTH", "origin": "default", "value": 32}, + { + "name": "DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP", + "origin": "default", + "value": "(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer" + "[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\\" + ".net[_-]sessionid|sid|jwt", + }, + { + "name": "DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP", + "origin": "default", + "value": "(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private" + "|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?" + "(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\\" + '.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\\s+[a-z0-9\\._\\-]+|token:' + "[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?|[\\-]" + "{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}", + }, + {"name": "DD_APPSEC_RASP_ENABLED", "origin": "default", "value": True}, + {"name": "DD_APPSEC_RULES", "origin": "default", "value": None}, + {"name": "DD_APPSEC_STACK_TRACE_ENABLED", "origin": "default", "value": True}, + {"name": "DD_APPSEC_WAF_TIMEOUT", "origin": "default", "value": 5.0}, + {"name": "DD_CIVISIBILITY_AGENTLESS_ENABLED", "origin": "env_var", "value": False}, + {"name": "DD_CIVISIBILITY_AGENTLESS_URL", "origin": "default", "value": ""}, + {"name": "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED", "origin": "default", "value": True}, + {"name": "DD_CIVISIBILITY_ITR_ENABLED", "origin": "default", "value": True}, + {"name": "DD_CIVISIBILITY_LOG_LEVEL", "origin": "default", "value": "info"}, + {"name": "DD_CRASHTRACKING_ALT_STACK", "origin": "default", "value": False}, + {"name": "DD_CRASHTRACKING_DEBUG_URL", "origin": "default", "value": None}, + {"name": "DD_CRASHTRACKING_ENABLED", "origin": "default", "value": True}, + {"name": "DD_CRASHTRACKING_STACKTRACE_RESOLVER", "origin": "default", "value": "full"}, + {"name": "DD_CRASHTRACKING_STDERR_FILENAME", "origin": "default", "value": None}, + {"name": "DD_CRASHTRACKING_STDOUT_FILENAME", "origin": "default", "value": None}, + {"name": "DD_CRASHTRACKING_TAGS", "origin": "default", "value": "{}"}, + {"name": "DD_CRASHTRACKING_WAIT_FOR_RECEIVER", "origin": "default", "value": True}, + {"name": "DD_DATA_STREAMS_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_DOGSTATSD_PORT", "origin": "default", "value": None}, + {"name": "DD_DOGSTATSD_URL", "origin": "default", "value": None}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_DIAGNOSTICS_INTERVAL", "origin": "default", "value": 3600}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_ENABLED", "origin": "default", "value": False}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_MAX_PAYLOAD_SIZE", "origin": "default", "value": 1048576}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_METRICS_ENABLED", "origin": "default", "value": True}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS", "origin": "default", "value": "set()"}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES", "origin": "default", "value": "set()"}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL", "origin": "default", "value": 1.0}, + {"name": "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_TIMEOUT", "origin": "default", "value": 30}, + {"name": "DD_ENV", "origin": "default", "value": None}, + {"name": "DD_EXCEPTION_REPLAY_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED", "origin": "default", "value": False}, + {"name": "DD_HTTP_CLIENT_TAG_QUERY_STRING", "origin": "default", "value": None}, + {"name": "DD_IAST_ENABLED", "origin": "default", "value": False}, + {"name": "DD_IAST_REDACTION_ENABLED", "origin": "default", "value": True}, + { + "name": "DD_IAST_REDACTION_NAME_PATTERN", + "origin": "default", + "value": "(?i)^.*(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?" + "|secret_?)key(?:_?id)?|password|token|username|user_id|last.name|consumer_?(?:id|key|secret)|sign(" + "?:ed|ature)?|auth(?:entication|orization)?)", + }, + { + "name": "DD_IAST_REDACTION_VALUE_NUMERAL", + "origin": "default", + "value": "^[+-]?((0b[01]+)|(0x[0-9A-Fa-f]+)|(\\d+\\.?\\d*(?:[Ee][+-]?\\d+)?|\\.\\d+(?:[Ee][+-]?" + "\\d+)?)|(X\\'[0-9A-Fa-f]+\\')|(B\\'[01]+\\'))$", + }, + { + "name": "DD_IAST_REDACTION_VALUE_PATTERN", + "origin": "default", + "value": "(?i)bearer\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|password|gh[opsu]_[0-9a-zA-Z]{36}|ey" + "[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(\\.[\\w.+\\/=-]+)?|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}" + "[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}", + }, + {"name": "DD_INJECT_FORCE", "origin": "env_var", "value": True}, + {"name": "DD_INSTRUMENTATION_INSTALL_ID", "origin": "default", "value": None}, + {"name": "DD_INSTRUMENTATION_INSTALL_TYPE", "origin": "default", "value": None}, + {"name": "DD_INSTRUMENTATION_TELEMETRY_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_LLMOBS_AGENTLESS_ENABLED", "origin": "default", "value": False}, + {"name": "DD_LLMOBS_ENABLED", "origin": "default", "value": False}, + {"name": "DD_LLMOBS_ML_APP", "origin": "default", "value": None}, + {"name": "DD_LLMOBS_SAMPLE_RATE", "origin": "default", "value": 1.0}, + {"name": "DD_PROFILING_AGENTLESS", "origin": "default", "value": False}, + {"name": "DD_PROFILING_API_TIMEOUT", "origin": "default", "value": 10.0}, + {"name": "DD_PROFILING_CAPTURE_PCT", "origin": "env_var", "value": 5.0}, + {"name": "DD_PROFILING_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_PROFILING_ENABLE_ASSERTS", "origin": "default", "value": False}, + {"name": "DD_PROFILING_ENABLE_CODE_PROVENANCE", "origin": "default", "value": True}, + {"name": "DD_PROFILING_ENDPOINT_COLLECTION_ENABLED", "origin": "default", "value": True}, + {"name": "DD_PROFILING_IGNORE_PROFILER", "origin": "default", "value": False}, + {"name": "DD_PROFILING_MAX_EVENTS", "origin": "default", "value": 16384}, + {"name": "DD_PROFILING_MAX_FRAMES", "origin": "env_var", "value": 512}, + {"name": "DD_PROFILING_MAX_TIME_USAGE_PCT", "origin": "default", "value": 1.0}, + {"name": "DD_PROFILING_OUTPUT_PPROF", "origin": "default", "value": None}, + {"name": "DD_PROFILING_SAMPLE_POOL_CAPACITY", "origin": "default", "value": 4}, + {"name": "DD_PROFILING_TAGS", "origin": "default", "value": "{}"}, + {"name": "DD_PROFILING_TIMELINE_ENABLED", "origin": "default", "value": False}, + {"name": "DD_PROFILING_UPLOAD_INTERVAL", "origin": "env_var", "value": 10.0}, + {"name": "DD_PROFILING__FORCE_LEGACY_EXPORTER", "origin": "env_var", "value": True}, + {"name": "DD_REMOTE_CONFIGURATION_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS", "origin": "env_var", "value": 1.0}, + {"name": "DD_RUNTIME_METRICS_ENABLED", "origin": "unknown", "value": True}, + {"name": "DD_RUNTIME_METRICS_ENABLED", "origin": "unknown", "value": False}, + {"name": "DD_SERVICE", "origin": "default", "value": "unnamed-python-service"}, + {"name": "DD_SERVICE_MAPPING", "origin": "env_var", "value": "default_dd_service:remapped_dd_service"}, + {"name": "DD_SITE", "origin": "env_var", "value": "datadoghq.com"}, + {"name": "DD_SPAN_SAMPLING_RULES", "origin": "env_var", "value": '[{"service":"xyz", "sample_rate":0.23}]'}, + { + "name": "DD_SPAN_SAMPLING_RULES_FILE", + "origin": "env_var", + "value": str(file), + }, + {"name": "DD_SYMBOL_DATABASE_INCLUDES", "origin": "default", "value": "set()"}, + {"name": "DD_SYMBOL_DATABASE_UPLOAD_ENABLED", "origin": "default", "value": False}, + {"name": "DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", "origin": "default", "value": True}, + {"name": "DD_TELEMETRY_HEARTBEAT_INTERVAL", "origin": "default", "value": 60}, + {"name": "DD_TESTING_RAISE", "origin": "env_var", "value": True}, + {"name": "DD_TEST_SESSION_NAME", "origin": "default", "value": None}, + {"name": "DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED", "origin": "default", "value": False}, + {"name": "DD_TRACE_AGENT_TIMEOUT_SECONDS", "origin": "default", "value": 2.0}, + {"name": "DD_TRACE_ANALYTICS_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_API_VERSION", "origin": "env_var", "value": "v0.5"}, + {"name": "DD_TRACE_CLIENT_IP_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_CLIENT_IP_HEADER", "origin": "default", "value": None}, + {"name": "DD_TRACE_COMPUTE_STATS", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_DEBUG", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_HEALTH_METRICS_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING", "origin": "default", "value": "true"}, + {"name": "DD_TRACE_HTTP_SERVER_ERROR_STATUSES", "origin": "default", "value": "500-599"}, + {"name": "DD_TRACE_METHODS", "origin": "default", "value": None}, + {"name": "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", "origin": "env_var", "value": ".*"}, + {"name": "DD_TRACE_OTEL_ENABLED", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_PARTIAL_FLUSH_ENABLED", "origin": "env_var", "value": False}, + {"name": "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", "origin": "env_var", "value": 3}, + {"name": "DD_TRACE_PROPAGATION_EXTRACT_FIRST", "origin": "default", "value": False}, + {"name": "DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "origin": "default", "value": False}, + {"name": "DD_TRACE_PROPAGATION_STYLE_EXTRACT", "origin": "env_var", "value": "tracecontext"}, + {"name": "DD_TRACE_PROPAGATION_STYLE_INJECT", "origin": "env_var", "value": "tracecontext"}, + {"name": "DD_TRACE_RATE_LIMIT", "origin": "env_var", "value": 50}, + {"name": "DD_TRACE_REPORT_HOSTNAME", "origin": "default", "value": False}, + {"name": "DD_TRACE_SPAN_AGGREGATOR_RLOCK", "origin": "default", "value": True}, + {"name": "DD_TRACE_SPAN_TRACEBACK_MAX_SIZE", "origin": "default", "value": 30}, + {"name": "DD_TRACE_STARTUP_LOGS", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_WRITER_BUFFER_SIZE_BYTES", "origin": "env_var", "value": 1000}, + {"name": "DD_TRACE_WRITER_INTERVAL_SECONDS", "origin": "env_var", "value": 30.0}, + {"name": "DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES", "origin": "env_var", "value": 9999}, + {"name": "DD_TRACE_WRITER_REUSE_CONNECTIONS", "origin": "env_var", "value": True}, + {"name": "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "origin": "default", "value": 512}, + {"name": "DD_USER_MODEL_EMAIL_FIELD", "origin": "default", "value": ""}, + {"name": "DD_USER_MODEL_LOGIN_FIELD", "origin": "default", "value": ""}, + {"name": "DD_USER_MODEL_NAME_FIELD", "origin": "default", "value": ""}, + {"name": "DD_VERSION", "origin": "default", "value": None}, + {"name": "_DD_APPSEC_DEDUPLICATION_ENABLED", "origin": "default", "value": True}, + {"name": "_DD_IAST_LAZY_TAINT", "origin": "default", "value": False}, + {"name": "_DD_INJECT_WAS_ATTEMPTED", "origin": "default", "value": False}, + {"name": "_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS", "origin": "default", "value": False}, + {"name": "appsec_enabled", "origin": "env_var", "value": "true"}, + {"name": "data_streams_enabled", "origin": "env_var", "value": "true"}, + {"name": "ddtrace_auto_used", "origin": "unknown", "value": True}, + {"name": "ddtrace_bootstrapped", "origin": "unknown", "value": True}, + {"name": "logs_injection_enabled", "origin": "env_var", "value": "true"}, + {"name": "profiling_enabled", "origin": "env_var", "value": "true"}, + {"name": "python_build_gnu_type", "origin": "unknown", "value": sysconfig.get_config_var("BUILD_GNU_TYPE")}, + {"name": "python_host_gnu_type", "origin": "unknown", "value": sysconfig.get_config_var("HOST_GNU_TYPE")}, + {"name": "python_soabi", "origin": "unknown", "value": sysconfig.get_config_var("SOABI")}, + {"name": "trace_enabled", "origin": "env_var", "value": "false"}, + {"name": "trace_header_tags", "origin": "default", "value": ""}, + {"name": "trace_sample_rate", "origin": "env_var", "value": "0.5"}, + { + "name": "trace_sampling_rules", + "origin": "env_var", + "value": '[{"sample_rate":1.0,"service":"xyz","name":"abc"}]', + }, + {"name": "trace_tags", "origin": "env_var", "value": "team:apm,component:web"}, + ] + assert configurations == expected, configurations def test_update_dependencies_event(test_agent_session, ddtrace_run_python_code_in_subprocess): @@ -633,52 +751,55 @@ def test_telemetry_writer_agent_setup_agentless_arg_overrides_env(env_agentless, assert new_telemetry_writer._client._endpoint == expected_endpoint +@pytest.mark.subprocess( + env={"DD_SITE": "datad0g.com", "DD_API_KEY": "foobarkey", "DD_CIVISIBILITY_AGENTLESS_ENABLED": "true"} +) def test_telemetry_writer_agentless_setup(): - with override_global_config( - {"_dd_site": "datad0g.com", "_dd_api_key": "foobarkey", "_ci_visibility_agentless_enabled": True} - ): - new_telemetry_writer = ddtrace.internal.telemetry.TelemetryWriter() - assert new_telemetry_writer._enabled - assert new_telemetry_writer._client._endpoint == "api/v2/apmtelemetry" - assert new_telemetry_writer._client._telemetry_url == "https://all-http-intake.logs.datad0g.com" - assert new_telemetry_writer._client._headers["dd-api-key"] == "foobarkey" + from ddtrace.internal.telemetry import telemetry_writer + + assert telemetry_writer._enabled + assert telemetry_writer._client._endpoint == "api/v2/apmtelemetry" + assert telemetry_writer._client._telemetry_url == "https://all-http-intake.logs.datad0g.com" + assert telemetry_writer._client._headers["dd-api-key"] == "foobarkey" +@pytest.mark.subprocess( + env={"DD_SITE": "datadoghq.eu", "DD_API_KEY": "foobarkey", "DD_CIVISIBILITY_AGENTLESS_ENABLED": "true"} +) def test_telemetry_writer_agentless_setup_eu(): - with override_global_config( - {"_dd_site": "datadoghq.eu", "_dd_api_key": "foobarkey", "_ci_visibility_agentless_enabled": True} - ): - new_telemetry_writer = ddtrace.internal.telemetry.TelemetryWriter() - assert new_telemetry_writer._enabled - assert new_telemetry_writer._client._endpoint == "api/v2/apmtelemetry" - assert new_telemetry_writer._client._telemetry_url == "https://instrumentation-telemetry-intake.datadoghq.eu" - assert new_telemetry_writer._client._headers["dd-api-key"] == "foobarkey" + from ddtrace.internal.telemetry import telemetry_writer + + assert telemetry_writer._enabled + assert telemetry_writer._client._endpoint == "api/v2/apmtelemetry" + assert telemetry_writer._client._telemetry_url == "https://instrumentation-telemetry-intake.datadoghq.eu" + assert telemetry_writer._client._headers["dd-api-key"] == "foobarkey" +@pytest.mark.subprocess(env={"DD_SITE": "datad0g.com", "DD_API_KEY": "", "DD_CIVISIBILITY_AGENTLESS_ENABLED": "true"}) def test_telemetry_writer_agentless_disabled_without_api_key(): - with override_global_config( - {"_dd_site": "datad0g.com", "_dd_api_key": None, "_ci_visibility_agentless_enabled": True} - ): - new_telemetry_writer = ddtrace.internal.telemetry.TelemetryWriter() - assert not new_telemetry_writer._enabled - assert new_telemetry_writer._client._endpoint == "api/v2/apmtelemetry" - assert new_telemetry_writer._client._telemetry_url == "https://all-http-intake.logs.datad0g.com" - assert "dd-api-key" not in new_telemetry_writer._client._headers + from ddtrace.internal.telemetry import telemetry_writer + assert not telemetry_writer._enabled + assert telemetry_writer._client._endpoint == "api/v2/apmtelemetry" + assert telemetry_writer._client._telemetry_url == "https://all-http-intake.logs.datad0g.com" + assert "dd-api-key" not in telemetry_writer._client._headers + +@pytest.mark.subprocess(env={"DD_SITE": "datad0g.com", "DD_API_KEY": "foobarkey"}) def test_telemetry_writer_is_using_agentless_by_default_if_api_key_is_available(): - with override_global_config({"_dd_site": "datad0g.com", "_dd_api_key": "foobarkey"}): - new_telemetry_writer = ddtrace.internal.telemetry.TelemetryWriter() - assert new_telemetry_writer._enabled - assert new_telemetry_writer._client._endpoint == "api/v2/apmtelemetry" - assert new_telemetry_writer._client._telemetry_url == "https://all-http-intake.logs.datad0g.com" - assert new_telemetry_writer._client._headers["dd-api-key"] == "foobarkey" + from ddtrace.internal.telemetry import telemetry_writer + assert telemetry_writer._enabled + assert telemetry_writer._client._endpoint == "api/v2/apmtelemetry" + assert telemetry_writer._client._telemetry_url == "https://all-http-intake.logs.datad0g.com" + assert telemetry_writer._client._headers["dd-api-key"] == "foobarkey" + +@pytest.mark.subprocess(env={"DD_API_KEY": "", "DD_CIVISIBILITY_AGENTLESS_ENABLED": "false"}) def test_telemetry_writer_is_using_agent_by_default_if_api_key_is_not_available(): - with override_global_config({"_dd_api_key": None, "_ci_visibility_agentless_enabled": False}): - new_telemetry_writer = ddtrace.internal.telemetry.TelemetryWriter() - assert new_telemetry_writer._enabled - assert new_telemetry_writer._client._endpoint == "telemetry/proxy/api/v2/apmtelemetry" - assert new_telemetry_writer._client._telemetry_url in ("http://localhost:9126", "http://testagent:9126") - assert "dd-api-key" not in new_telemetry_writer._client._headers + from ddtrace.internal.telemetry import telemetry_writer + + assert telemetry_writer._enabled + assert telemetry_writer._client._endpoint == "telemetry/proxy/api/v2/apmtelemetry" + assert telemetry_writer._client._telemetry_url in ("http://localhost:9126", "http://testagent:9126") + assert "dd-api-key" not in telemetry_writer._client._headers diff --git a/tests/tracer/test_global_config.py b/tests/tracer/test_global_config.py index 49bc0030e78..a83b8352074 100644 --- a/tests/tracer/test_global_config.py +++ b/tests/tracer/test_global_config.py @@ -1,3 +1,4 @@ +import os from unittest import TestCase import mock @@ -286,5 +287,5 @@ def test_parse_propagation_styles_b3_deprecation(capsys): with pytest.warns( DeprecationWarning, match='Using DD_TRACE_PROPAGATION_STYLE="b3 single header" is deprecated' ), override_env(dict(DD_TRACE_PROPAGATION_STYLE="b3 single header")): - style = _parse_propagation_styles("DD_TRACE_PROPAGATION_STYLE", default="datadog") + style = _parse_propagation_styles(os.environ["DD_TRACE_PROPAGATION_STYLE"]) assert style == ["b3"] From b9708de40822a614c1df09123311dc7e427d1754 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Wed, 9 Oct 2024 02:13:15 +0200 Subject: [PATCH 6/9] chore(profiling): apple silicon mac builds for libdatadog (#10940) Native tests builds but some of them fails, `./build_standalone.sh -- Release all_test` - will follow up on another PR. Development purposes only. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: David Sanchez <838104+sanchda@users.noreply.github.com> --- .github/workflows/unit_tests.yml | 5 ++ .gitignore | 1 + .../datadog/profiling/build_standalone.sh | 53 ++++++++++++--- .../profiling/cmake/AnalysisFunc.cmake | 68 ++++++++++++------- .../profiling/cmake/tools/fetch_libdatadog.sh | 26 +++++-- .../profiling/crashtracker/CMakeLists.txt | 40 ++++++++--- .../dd_wrapper/src/ddup_interface.cpp | 2 +- .../datadog/profiling/ddup/CMakeLists.txt | 25 +++++-- .../datadog/profiling/stack_v2/CMakeLists.txt | 13 +++- .../stack_v2/include/stack_renderer.hpp | 4 +- .../profiling/stack_v2/src/sampler.cpp | 4 +- .../profiling/stack_v2/src/stack_renderer.cpp | 2 +- setup.py | 4 +- 13 files changed, 186 insertions(+), 61 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 4abce66d343..a81755cf20f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -17,6 +17,7 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] # Keep this in sync with hatch.toml python-version: ["3.7", "3.10", "3.12"] + steps: - uses: actions/checkout@v4 # Include all history and tags @@ -32,5 +33,9 @@ jobs: with: version: "1.12.0" + - name: Install coreutils for MacOS to get sha256sum + if: matrix.os == 'macos-latest' + run: brew install coreutils + - name: Run tests run: hatch run +py=${{ matrix.python-version }} ddtrace_unit_tests:test diff --git a/.gitignore b/.gitignore index f6b1906f30c..b2f71518924 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ ddtrace/internal/_encoding.c ddtrace/internal/_rand.c ddtrace/internal/_tagset.c *.so +*.dylib *.a # Cython annotate HTML files diff --git a/ddtrace/internal/datadog/profiling/build_standalone.sh b/ddtrace/internal/datadog/profiling/build_standalone.sh index 62ca35ba19f..d6af47b4d56 100755 --- a/ddtrace/internal/datadog/profiling/build_standalone.sh +++ b/ddtrace/internal/datadog/profiling/build_standalone.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail ### Useful globals @@ -8,6 +8,8 @@ BUILD_DIR="build" ### Compiler discovery # Initialize variables to store the highest versions +highest_cc="" +highest_cxx="" highest_gcc="" highest_gxx="" highest_clang="" @@ -18,24 +20,44 @@ highest_clangxx="" find_highest_compiler_version() { local base_name=$1 local highest_var_name=$2 + local highest_version=0 + local current_version + local found_version="" - # Try to find the latest versions of both GCC and Clang - # The range 5-20 is arbitrary (GCC 5 was released in 2015, and 20 is just a - # a high number since Clang is on version 17) + # Function to extract the numeric version from the compiler output + get_version() { + $1 --version 2>&1 | grep -oE '[0-9]+(\.[0-9]+)?' | head -n 1 + } + + # Try to find the latest versions of both GCC and Clang (numbered versions) + # The range 5-20 is arbitrary (GCC 5 was released in 2015, and 20 is a high number since Clang is on version 17) for version in {20..5}; do if command -v "${base_name}-${version}" &> /dev/null; then - eval "$highest_var_name=${base_name}-${version}" - return + current_version=$(get_version "${base_name}-${version}") + if (( $(echo "$current_version > $highest_version" | bc -l) )); then + highest_version=$current_version + found_version="${base_name}-${version}" + fi fi done - # Check for the base version if no numbered version was found + # Check the base version if it exists if command -v "$base_name" &> /dev/null; then - eval "$highest_var_name=$base_name" + current_version=$(get_version "$base_name") + if (( $(echo "$current_version > $highest_version" | bc -l) )); then + found_version="$base_name" + fi + fi + + # Assign the result to the variable name passed + if [[ -n $found_version ]]; then + eval "$highest_var_name=$found_version" fi } # Find highest versions for each compiler +find_highest_compiler_version cc highest_cc +find_highest_compiler_version c++ highest_cxx find_highest_compiler_version gcc highest_gcc find_highest_compiler_version g++ highest_gxx find_highest_compiler_version clang highest_clang @@ -78,6 +100,19 @@ cmake_args=( # Initial build targets; no matter what, dd_wrapper is the base dependency, so it's always built targets=("dd_wrapper") +set_cc() { + if [ -z "${CC:-}" ]; then + export CC=$highest_cc + fi + if [ -z "${CXX:-}" ]; then + export CXX=$highest_cxx + fi + cmake_args+=( + -DCMAKE_C_COMPILER=$CC + -DCMAKE_CXX_COMPILER=$CXX + ) +} + # Helper functions for finding the compiler(s) set_clang() { if [ -z "${CC:-}" ]; then @@ -252,7 +287,7 @@ add_compiler_args() { set_gcc ;; --) - set_gcc # Default to GCC, since this is what will happen in the official build + set_cc # Use system default compiler ;; *) echo "Unknown option: $1" diff --git a/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake b/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake index 671c4a13fc5..d462d25e573 100644 --- a/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake +++ b/ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake @@ -1,32 +1,54 @@ include(CheckIPOSupported) function(add_ddup_config target) - target_compile_options( - ${target} - PRIVATE "$<$:-Og;-ggdb3>" - "$<$:-Os;-ggdb3>" - "$<$:-Os>" - -ffunction-sections - -fno-semantic-interposition - -Wall - -Werror - -Wextra - -Wshadow - -Wnon-virtual-dtor - -Wold-style-cast) - target_link_options( - ${target} - PRIVATE - "$<$:-s>" - "$<$:>" - -Wl,--as-needed - -Wl,-Bsymbolic-functions - -Wl,--gc-sections - -Wl,-z,nodelete - -Wl,--exclude-libs,ALL) + # Common compile options + target_compile_options(${target} PRIVATE "$<$:-Os>" -ffunction-sections + -Wall + -Werror + -Wextra + -Wshadow + -Wnon-virtual-dtor + -Wold-style-cast) + + if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # macOS-specific options + target_compile_options( + ${target} + PRIVATE "$<$:-Og;-g>" + "$<$:-Os;-g>" + ) + else() + # Non-macOS (e.g., Linux) options + target_compile_options( + ${target} + PRIVATE "$<$:-Og;-ggdb3>" + "$<$:-Os;-ggdb3>" + -fno-semantic-interposition + ) + endif() + + # Common link options + target_link_options(${target} PRIVATE "$<$:>") + + if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # macOS-specific linker options + target_link_options(${target} PRIVATE "$<$:-Wl,-dead_strip>") + else() + # Linux/ELF-based linker options + target_link_options( + ${target} + PRIVATE + "$<$:-s>" + -Wl,--as-needed + -Wl,-Bsymbolic-functions + -Wl,--gc-sections + -Wl,-z,nodelete + -Wl,--exclude-libs,ALL) + endif() # If we can IPO, then do so check_ipo_supported(RESULT result) + if(result) set_property(TARGET ${target} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) endif() diff --git a/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh b/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh index afd56a1a2a3..a1e55066089 100755 --- a/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh +++ b/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh @@ -19,6 +19,7 @@ fi SCRIPTPATH=$(readlink -f "$0") SCRIPTDIR=$(dirname "$SCRIPTPATH") +OS_NAME=$(uname -s) MARCH=$(uname -m) TAG_LIBDATADOG=$1 @@ -26,12 +27,27 @@ TARGET_EXTRACT=$2 CHECKSUM_FILE=${SCRIPTDIR}/libdatadog_checksums.txt -# Test for musl -MUSL_LIBC=$(ldd /bin/ls | grep 'musl' | head -1 | cut -d ' ' -f1 || true) -if [[ -n ${MUSL_LIBC-""} ]]; then - DISTRIBUTION="alpine-linux-musl" +# if os is darwin, set distribution to apple-darwin and march to aarch64 +if [[ "$OS_NAME" == "Darwin" ]]; then + DISTRIBUTION="apple-darwin" + # if march is arm64 set it to aarch64 + if [[ "$MARCH" == "arm64" ]]; then + MARCH="aarch64" + else + echo "Unsupported architecture $MARCH for $OS_NAME" + exit 1 + fi +elif [[ "$OS_NAME" == "Linux" ]]; then + # Test for musl + MUSL_LIBC=$(ldd /bin/ls | grep 'musl' | head -1 | cut -d ' ' -f1 || true) + if [[ -n ${MUSL_LIBC-""} ]]; then + DISTRIBUTION="alpine-linux-musl" + else + DISTRIBUTION="unknown-linux-gnu" + fi else - DISTRIBUTION="unknown-linux-gnu" + echo "Unsupported OS $OS_NAME" + exit 1 fi # https://github.com/DataDog/libdatadog/releases/download/v0.7.0-rc.1/libdatadog-aarch64-alpine-linux-musl.tar.gz diff --git a/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt b/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt index 678ef83291c..d17ad669667 100644 --- a/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt @@ -18,6 +18,7 @@ include(FindLibdatadog) add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) find_package(Python3 COMPONENTS Interpreter Development) + # Make sure we have necessary Python variables if(NOT Python3_INCLUDE_DIRS) message(FATAL_ERROR "Python3_INCLUDE_DIRS not found") @@ -45,10 +46,19 @@ add_library(${EXTENSION_NAME} SHARED ${CRASHTRACKER_CPP_SRC}) # We can't add common Profiling configuration because cython generates messy code, so we just setup some basic flags and # features -target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Og;-ggdb3>" "$<$:-Os>" - -ffunction-sections -fno-semantic-interposition) -target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-s>" -Wl,--as-needed -Wl,-Bsymbolic-functions - -Wl,--gc-sections) +target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Os>" -ffunction-sections) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # macOS-specific options + target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Og;-g>") + target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-Wl,-dead_strip>") +else() + # Linux/ELF-based options + target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Og;-ggdb3>" -fno-semantic-interposition) + target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-s>" -Wl,--as-needed -Wl,-Bsymbolic-functions + -Wl,--gc-sections) +endif() + set_property(TARGET ${EXTENSION_NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) target_compile_features(${EXTENSION_NAME} PUBLIC cxx_std_17) @@ -62,9 +72,14 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # typical. set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..") target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ${Datadog_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS}) + ${Python3_INCLUDE_DIRS}) -target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) +if(Python3_LIBRARIES) + target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper ${Python3_LIBRARIES}) +else() + # for manylinux builds + target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) +endif() # Extensions are built as dynamic libraries, so PIC is required. set_target_properties(${EXTENSION_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -88,8 +103,17 @@ if(NOT CRASHTRACKER_EXE_TARGET_NAME) endif() set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH "$ORIGIN/.." OUTPUT_NAME - ${CRASHTRACKER_EXE_TARGET_NAME}) -target_link_libraries(crashtracker_exe PRIVATE dd_wrapper) + ${CRASHTRACKER_EXE_TARGET_NAME}) + +# To let crashtracker find Python library at runtime +set_target_properties(crashtracker_exe PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) + +if(Python3_LIBRARIES) + target_link_libraries(crashtracker_exe PRIVATE dd_wrapper ${Python3_LIBRARIES}) +else() + # for manylinux builds + target_link_libraries(crashtracker_exe PRIVATE dd_wrapper) +endif() # See the dd_wrapper CMakeLists.txt for a more detailed explanation of why we do what we do. if(INPLACE_LIB_INSTALL_DIR) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp index 255e179f21c..baee51a7eda 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp @@ -122,7 +122,7 @@ ddup_config_output_filename(std::string_view output_filename) // cppcheck-suppre } void -ddup_config_sample_pool_capacity(size_t capacity) // cppcheck-suppress unusedFunction +ddup_config_sample_pool_capacity(uint64_t capacity) // cppcheck-suppress unusedFunction { Datadog::SampleManager::set_sample_pool_capacity(capacity); } diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index e0e71bad113..920db8bf571 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -51,10 +51,19 @@ add_library(${EXTENSION_NAME} SHARED ${DDUP_CPP_SRC}) # We can't add common Profiling configuration because cython generates messy code, so we just setup some basic flags and # features -target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Og;-ggdb3>" "$<$:-Os>" - -ffunction-sections -fno-semantic-interposition) -target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-s>" -Wl,--as-needed -Wl,-Bsymbolic-functions - -Wl,--gc-sections) +target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-Os>" -ffunction-sections) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # macOS-specific options + target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Og;-g>") + target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-Wl,-dead_strip>") +else() + # Linux/ELF-based options + target_compile_options(${EXTENSION_NAME} PRIVATE "$<$:-Og;-ggdb3>" -fno-semantic-interposition) + target_link_options(${EXTENSION_NAME} PRIVATE "$<$:-s>" -Wl,--as-needed -Wl,-Bsymbolic-functions + -Wl,--gc-sections) +endif() + set_property(TARGET ${EXTENSION_NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) target_compile_features(${EXTENSION_NAME} PUBLIC cxx_std_17) @@ -68,9 +77,13 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # typical. set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..") target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ${Datadog_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS}) + ${Python3_INCLUDE_DIRS}) -target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) +if(Python3_LIBRARIES) + target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper ${Python3_LIBRARIES}) +else() + target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) +endif() # Extensions are built as dynamic libraries, so PIC is required. set_target_properties(${EXTENSION_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index bc0caae58d8..610d08ca473 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -29,13 +29,14 @@ endif() # Add echion set(ECHION_COMMIT - "b2f2d49f2f5d5c4dd78d1d9b83280db8ac2949c0" + "9d5bcc5867d7aefff73c837adcba4ef46eecebc6" CACHE STRING "Commit hash of echion to use") FetchContent_Declare( echion - GIT_REPOSITORY "https://github.com/sanchda/echion.git" + GIT_REPOSITORY "https://github.com/taegyunkim/echion.git" GIT_TAG ${ECHION_COMMIT}) FetchContent_GetProperties(echion) + if(NOT echion_POPULATED) FetchContent_Populate(echion) endif() @@ -90,7 +91,13 @@ set_target_properties(${EXTENSION_NAME} PROPERTIES SUFFIX "") # RPATH is needed for sofile discovery at runtime, since Python packages are not installed in the system path. This is # typical. set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..") -target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) + +if(Python3_LIBRARIES) + target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper ${Python3_LIBRARIES}) +else() + # for manylinux builds + target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper) +endif() # Extensions are built as dynamic libraries, so PIC is required. set_target_properties(${EXTENSION_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/ddtrace/internal/datadog/profiling/stack_v2/include/stack_renderer.hpp b/ddtrace/internal/datadog/profiling/stack_v2/include/stack_renderer.hpp index 378caf769ef..463fdb596a6 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/include/stack_renderer.hpp +++ b/ddtrace/internal/datadog/profiling/stack_v2/include/stack_renderer.hpp @@ -41,11 +41,11 @@ class StackRenderer : public RendererInterface microsecond_t wall_time_us, uintptr_t thread_id, unsigned long native_id) override; - virtual void render_task_begin(std::string_view name); + virtual void render_task_begin(std::string_view name) override; virtual void render_stack_begin() override; virtual void render_python_frame(std::string_view name, std::string_view file, uint64_t line) override; virtual void render_native_frame(std::string_view name, std::string_view file, uint64_t line) override; - virtual void render_cpu_time(microsecond_t cpu_time_us) override; + virtual void render_cpu_time(uint64_t cpu_time_us) override; virtual void render_stack_end() override; virtual bool is_valid() override; }; diff --git a/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp index 29f08f66a42..3f02e20cd43 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp @@ -91,7 +91,7 @@ Sampler::one_time_setup() } void -Sampler::register_thread(uintptr_t id, uint64_t native_id, const char* name) +Sampler::register_thread(uint64_t id, uint64_t native_id, const char* name) { // Registering threads requires coordinating with one of echion's global locks, which we take here. const std::lock_guard thread_info_guard{ thread_info_map_lock }; @@ -122,7 +122,7 @@ Sampler::register_thread(uintptr_t id, uint64_t native_id, const char* name) } void -Sampler::unregister_thread(uintptr_t id) +Sampler::unregister_thread(uint64_t id) { // unregistering threads requires coordinating with one of echion's global locks, which we take here. const std::lock_guard thread_info_guard{ thread_info_map_lock }; diff --git a/ddtrace/internal/datadog/profiling/stack_v2/src/stack_renderer.cpp b/ddtrace/internal/datadog/profiling/stack_v2/src/stack_renderer.cpp index dc1ac1239b9..8abc37ee55a 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/src/stack_renderer.cpp +++ b/ddtrace/internal/datadog/profiling/stack_v2/src/stack_renderer.cpp @@ -132,7 +132,7 @@ StackRenderer::render_native_frame(std::string_view name, std::string_view file, } void -StackRenderer::render_cpu_time(microsecond_t cpu_time_us) +StackRenderer::render_cpu_time(uint64_t cpu_time_us) { if (sample == nullptr) { std::cerr << "Received a CPU time without sample storage. Some profiling data has been lost." << std::endl; diff --git a/setup.py b/setup.py index f45e9b086c2..7d028d57127 100644 --- a/setup.py +++ b/setup.py @@ -525,7 +525,9 @@ def get_exts_for(name): ext_modules.append(CMakeExtension("ddtrace.appsec._iast._taint_tracking._native", source_dir=IAST_DIR)) - if platform.system() == "Linux" and is_64_bit_python(): + if ( + platform.system() == "Linux" or (platform.system() == "Darwin" and platform.machine() == "arm64") + ) and is_64_bit_python(): ext_modules.append( CMakeExtension( "ddtrace.internal.datadog.profiling.ddup._ddup", From 24389a7acb275d2c50313ebdc3a99c641d25685a Mon Sep 17 00:00:00 2001 From: lievan <42917263+lievan@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:34:51 -0400 Subject: [PATCH 7/9] chore(llmobs): add sampling for ragas skeleton code (#10719) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V0 sampling implementation for evaluator runner. The evaluator runner is a period service that stores a list of (evaluations to generate / "evaluators") on finished LLM obs span events. The runner will have a list of sampling rules that it applies to spans before triggering any evaluator on that span. Evaluator sampler rules are configured by setting `_DD_LLMOBS_EVALUATOR_SAMPLING_RULES` to a json list of rules ``` [ {"sample_rate": ..., "evaluator_label":.., "span_name": ...}, {... ] ``` Each rule consists of the following: - `sample_rate` (required) - `evaluator_label` (optional, the evaluator name) - `span_name` (optional, the span name). _Not that for APM trace sampling rules, `span_name` is just `name`. But since we're dealing with both evaluator names/labels and span names, and perhaps more names such as ml app in the future, I think it's better to be more verbose for clarity._ Supporting sampling rules based on **evaluator label** and **span name** is key since most evaluators do not apply to all types of spans. For example, a faithfulness evaluation only applies to an LLM generation that uses a ground truth reference context. Example Usage: ``` _DD_LLMOBS_EVALUATOR_SAMPLING_RULES=‘[{"sample_rate":0.5, “evaluator_label”: “ragas_faithfulness”, “span_name”: ”augmented_generation"}]' python3 app.py ``` Code Changes: 1. The evaluation runner buffer now includes both the span event dict and also a span object. This is because the sampler the `span._trace_id_64bits` field is used for sampling 2. We implement a brand new `EvaluatorSampler` helper class that the `EvaluationRunner` uses for sampling. The EvaluationRunner stores **one instance** of `EvaluatorSampler`, which internally stores a list of `EvaluatorSamplingRule`(s). The `EvaluatorSamplingRule` class inherits from `SamplingRule` so we can re-use some helpful utilities e.g. the `sample` method. 3. Rule matching is basic-string-equality only for now. **Follow ups** - implement more matching capabilities for rules - support sampling on more span fields, e.g ml app ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: lievan --- ddtrace/llmobs/_evaluators/runner.py | 27 ++- ddtrace/llmobs/_evaluators/sampler.py | 94 ++++++++ ddtrace/llmobs/_trace_processor.py | 2 +- tests/llmobs/_utils.py | 15 ++ tests/llmobs/conftest.py | 6 + ...bs_evaluator_runner.send_score_metric.yaml | 37 +++ tests/llmobs/test_llmobs_evaluator_runner.py | 222 ++++++++++++++++-- tests/llmobs/test_llmobs_service.py | 4 +- 8 files changed, 369 insertions(+), 38 deletions(-) create mode 100644 ddtrace/llmobs/_evaluators/sampler.py create mode 100644 tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml diff --git a/ddtrace/llmobs/_evaluators/runner.py b/ddtrace/llmobs/_evaluators/runner.py index 02fd2939dd7..7f08b258f62 100644 --- a/ddtrace/llmobs/_evaluators/runner.py +++ b/ddtrace/llmobs/_evaluators/runner.py @@ -3,12 +3,13 @@ import os from typing import Dict +from ddtrace import Span from ddtrace.internal import forksafe from ddtrace.internal import service from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService - -from .ragas.faithfulness import RagasFaithfulnessEvaluator +from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator +from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSampler logger = get_logger(__name__) @@ -28,11 +29,12 @@ class EvaluatorRunner(PeriodicService): def __init__(self, interval: float, llmobs_service=None, evaluators=None): super(EvaluatorRunner, self).__init__(interval=interval) self._lock = forksafe.RLock() - self._buffer = [] # type: list[Dict] + self._buffer = [] # type: list[tuple[Dict, Span]] self._buffer_limit = 1000 self.llmobs_service = llmobs_service self.executor = futures.ThreadPoolExecutor() + self.sampler = EvaluatorRunnerSampler() self.evaluators = [] if evaluators is None else evaluators if len(self.evaluators) > 0: @@ -70,27 +72,34 @@ def recreate(self) -> "EvaluatorRunner": def on_shutdown(self): self.executor.shutdown() - def enqueue(self, span_event: Dict) -> None: + def enqueue(self, span_event: Dict, span: Span) -> None: with self._lock: if len(self._buffer) >= self._buffer_limit: logger.warning( "%r event buffer full (limit is %d), dropping event", self.__class__.__name__, self._buffer_limit ) return - self._buffer.append(span_event) + self._buffer.append((span_event, span)) def periodic(self) -> None: with self._lock: if not self._buffer: return - events = self._buffer + span_events_and_spans = self._buffer # type: list[tuple[Dict, Span]] self._buffer = [] try: - self.run(events) + self.run(span_events_and_spans) except RuntimeError as e: logger.debug("failed to run evaluation: %s", e) - def run(self, spans): + def run(self, span_events_and_spans): for evaluator in self.evaluators: - self.executor.map(lambda span: evaluator.run_and_submit_evaluation(span), spans) + self.executor.map( + lambda span: evaluator.run_and_submit_evaluation(span), + [ + span_event + for span_event, span in span_events_and_spans + if self.sampler.sample(evaluator.LABEL, span) + ], + ) diff --git a/ddtrace/llmobs/_evaluators/sampler.py b/ddtrace/llmobs/_evaluators/sampler.py new file mode 100644 index 00000000000..a3298603656 --- /dev/null +++ b/ddtrace/llmobs/_evaluators/sampler.py @@ -0,0 +1,94 @@ +import json +from json.decoder import JSONDecodeError +import os +from typing import List +from typing import Optional +from typing import Union + +from ddtrace import config +from ddtrace.internal.logger import get_logger +from ddtrace.sampling_rule import SamplingRule + + +logger = get_logger(__name__) + + +class EvaluatorRunnerSamplingRule(SamplingRule): + SAMPLE_RATE_KEY = "sample_rate" + EVALUATOR_LABEL_KEY = "evaluator_label" + SPAN_NAME_KEY = "span_name" + + def __init__( + self, + sample_rate: float, + evaluator_label: Optional[Union[str, object]] = None, + span_name: Optional[object] = None, + ): + super(EvaluatorRunnerSamplingRule, self).__init__(sample_rate) + self.evaluator_label = evaluator_label + self.span_name = span_name + + def matches(self, evaluator_label, span_name): + for prop, pattern in [(span_name, self.span_name), (evaluator_label, self.evaluator_label)]: + if pattern != self.NO_RULE and prop != pattern: + return False + return True + + def __repr__(self): + return "EvaluatorRunnerSamplingRule(sample_rate={}, evaluator_label={}, span_name={})".format( + self.sample_rate, self.evaluator_label, self.span_name + ) + + __str__ = __repr__ + + +class EvaluatorRunnerSampler: + SAMPLING_RULES_ENV_VAR = "_DD_LLMOBS_EVALUATOR_SAMPLING_RULES" + + def __init__(self): + self.rules = self.parse_rules() + + def sample(self, evaluator_label, span): + for rule in self.rules: + if rule.matches(evaluator_label=evaluator_label, span_name=span.name): + return rule.sample(span) + return True + + def parse_rules(self) -> List[EvaluatorRunnerSamplingRule]: + rules = [] + sampling_rules_str = os.getenv(self.SAMPLING_RULES_ENV_VAR) + + def parsing_failed_because(msg, maybe_throw_this): + if config._raise: + raise maybe_throw_this(msg) + logger.warning(msg, exc_info=True) + + if not sampling_rules_str: + return [] + try: + json_rules = json.loads(sampling_rules_str) + except JSONDecodeError: + parsing_failed_because( + "Failed to parse evaluator sampling rules of: `{}`".format(sampling_rules_str), ValueError + ) + return [] + + if not isinstance(json_rules, list): + parsing_failed_because("Evaluator sampling rules must be a list of dictionaries", ValueError) + return [] + + for rule in json_rules: + if "sample_rate" not in rule: + parsing_failed_because( + "No sample_rate provided for sampling rule: {}".format(json.dumps(rule)), KeyError + ) + continue + try: + sample_rate = float(rule[EvaluatorRunnerSamplingRule.SAMPLE_RATE_KEY]) + except ValueError: + parsing_failed_because("sample_rate is not a float for rule: {}".format(json.dumps(rule)), KeyError) + continue + span_name = rule.get(EvaluatorRunnerSamplingRule.SPAN_NAME_KEY, SamplingRule.NO_RULE) + evaluator_label = rule.get(EvaluatorRunnerSamplingRule.EVALUATOR_LABEL_KEY, SamplingRule.NO_RULE) + rules.append(EvaluatorRunnerSamplingRule(sample_rate, evaluator_label, span_name)) + return rules diff --git a/ddtrace/llmobs/_trace_processor.py b/ddtrace/llmobs/_trace_processor.py index d0a7c28f2c3..beca9684c6f 100644 --- a/ddtrace/llmobs/_trace_processor.py +++ b/ddtrace/llmobs/_trace_processor.py @@ -68,7 +68,7 @@ def submit_llmobs_span(self, span: Span) -> None: if not span_event: return if self._evaluator_runner: - self._evaluator_runner.enqueue(span_event) + self._evaluator_runner.enqueue(span_event, span) def _llmobs_span_event(self, span: Span) -> Dict[str, Any]: """Span event object structure.""" diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index 308e420ddff..f29f9781721 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -438,3 +438,18 @@ def _oversized_retrieval_event(): }, "metrics": {"input_tokens": 64, "output_tokens": 128, "total_tokens": 192}, } + + +class DummyEvaluator: + LABEL = "dummy" + + def __init__(self, llmobs_service): + self.llmobs_service = llmobs_service + + def run_and_submit_evaluation(self, span): + self.llmobs_service.submit_evaluation( + span_context=span, + label=DummyEvaluator.LABEL, + value=1.0, + metric_type="score", + ) diff --git a/tests/llmobs/conftest.py b/tests/llmobs/conftest.py index dd331c726d4..fbe38232cd5 100644 --- a/tests/llmobs/conftest.py +++ b/tests/llmobs/conftest.py @@ -95,6 +95,12 @@ def mock_evaluator_logs(): yield m +@pytest.fixture +def mock_evaluator_sampler_logs(): + with mock.patch("ddtrace.llmobs._evaluators.sampler.logger") as m: + yield m + + @pytest.fixture def mock_http_writer_logs(): with mock.patch("ddtrace.internal.writer.writer.log") as m: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml new file mode 100644 index 00000000000..d715994c439 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml @@ -0,0 +1,37 @@ +interactions: +- request: + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": + "123", "trace_id": "1234", "label": "dummy", "metric_type": "score", "timestamp_ms": + 1728480443772, "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.14.0.dev196+g7cf7989ab", + "ml_app:unnamed-ml-app"]}]}}}' + headers: + Content-Type: + - application/json + DD-API-KEY: + - XXXXXX + method: POST + uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + response: + body: + string: '{"data":{"id":"ccf36d1a-6153-4042-ba2d-a5ec5896a6ac","type":"evaluation_metric","attributes":{"metrics":[{"id":"bYp4oTawxz","trace_id":"1234","span_id":"123","timestamp_ms":1728480443772,"ml_app":"unnamed-ml-app","metric_type":"score","label":"dummy","score_value":1,"tags":["ddtrace.version:2.14.0.dev196+g7cf7989ab","ml_app:unnamed-ml-app"]}]}}}' + headers: + content-length: + - '347' + content-security-policy: + - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com + content-type: + - application/vnd.api+json + date: + - Wed, 09 Oct 2024 13:27:25 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-frame-options: + - SAMEORIGIN + status: + code: 202 + message: Accepted +version: 1 diff --git a/tests/llmobs/test_llmobs_evaluator_runner.py b/tests/llmobs/test_llmobs_evaluator_runner.py index 88f336db99e..a846914b3ac 100644 --- a/tests/llmobs/test_llmobs_evaluator_runner.py +++ b/tests/llmobs/test_llmobs_evaluator_runner.py @@ -1,11 +1,21 @@ +import json +import os import time import mock import pytest import ddtrace +from ddtrace._trace.span import Span from ddtrace.llmobs._evaluators.runner import EvaluatorRunner +from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSampler +from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSamplingRule from ddtrace.llmobs._writer import LLMObsEvaluationMetricEvent +from tests.utils import override_env +from tests.utils import override_global_config + + +DUMMY_SPAN = Span("dummy_span") def _dummy_ragas_eval_metric_event(span_id, trace_id): @@ -31,7 +41,7 @@ def test_evaluator_runner_start(mock_evaluator_logs, mock_ragas_evaluator): def test_evaluator_runner_buffer_limit(mock_evaluator_logs): evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=mock.MagicMock()) for _ in range(1001): - evaluator_runner.enqueue({}) + evaluator_runner.enqueue({}, DUMMY_SPAN) mock_evaluator_logs.warning.assert_called_with( "%r event buffer full (limit is %d), dropping event", "EvaluatorRunner", 1000 ) @@ -40,7 +50,7 @@ def test_evaluator_runner_buffer_limit(mock_evaluator_logs): def test_evaluator_runner_periodic_enqueues_eval_metric(LLMObs, mock_llmobs_eval_metric_writer, mock_ragas_evaluator): evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=LLMObs) evaluator_runner.evaluators.append(mock_ragas_evaluator(llmobs_service=LLMObs)) - evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}) + evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN) evaluator_runner.periodic() mock_llmobs_eval_metric_writer.enqueue.assert_called_once_with( _dummy_ragas_eval_metric_event(span_id="123", trace_id="1234") @@ -53,7 +63,7 @@ def test_evaluator_runner_timed_enqueues_eval_metric(LLMObs, mock_llmobs_eval_me evaluator_runner.evaluators.append(mock_ragas_evaluator(llmobs_service=LLMObs)) evaluator_runner.start() - evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}) + evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN) time.sleep(0.1) @@ -63,37 +73,197 @@ def test_evaluator_runner_timed_enqueues_eval_metric(LLMObs, mock_llmobs_eval_me def test_evaluator_runner_on_exit(mock_writer_logs, run_python_code_in_subprocess): + env = os.environ.copy() + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] + if "PYTHONPATH" in env: + pypath.append(env["PYTHONPATH"]) + env.update( + { + "DD_API_KEY": "dummy-api-key", + "DD_SITE": "datad0g.com", + "PYTHONPATH": ":".join(pypath), + "DD_LLMOBS_ML_APP": "unnamed-ml-app", + "_DD_LLMOBS_WRITER_INTERVAL": "0.01", + } + ) out, err, status, pid = run_python_code_in_subprocess( """ import os import time -import mock - -from ddtrace.internal.utils.http import Response +import atexit from ddtrace.llmobs import LLMObs from ddtrace.llmobs._evaluators.runner import EvaluatorRunner -from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator - -with mock.patch( - "ddtrace.llmobs._evaluators.runner.EvaluatorRunner.periodic", - return_value=Response( - status=200, - body="{}", - ), -): - LLMObs.enable( - site="datad0g.com", - api_key=os.getenv("DD_API_KEY"), - ml_app="unnamed-ml-app", - ) - evaluator_runner = EvaluatorRunner( - interval=0.01, llmobs_service=LLMObs - ) - evaluator_runner.evaluators.append(RagasFaithfulnessEvaluator(llmobs_service=LLMObs)) - evaluator_runner.start() - evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}) +from tests.llmobs._utils import logs_vcr +from tests.llmobs._utils import DummyEvaluator + +ctx = logs_vcr.use_cassette("tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml") +ctx.__enter__() +atexit.register(lambda: ctx.__exit__()) +LLMObs.enable() +evaluator_runner = EvaluatorRunner( + interval=0.01, llmobs_service=LLMObs +) +evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=LLMObs)) +evaluator_runner.start() +evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, None) +evaluator_runner.periodic() """, + env=env, ) assert status == 0, err assert out == b"" assert err == b"" + + +def test_evaluator_runner_sampler_single_rule(monkeypatch): + monkeypatch.setenv( + EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, + json.dumps([{"sample_rate": 0.5, "evaluator_label": "ragas_faithfulness", "span_name": "dummy_span"}]), + ) + sampling_rules = EvaluatorRunnerSampler().rules + assert len(sampling_rules) == 1 + assert sampling_rules[0].sample_rate == 0.5 + assert sampling_rules[0].evaluator_label == "ragas_faithfulness" + assert sampling_rules[0].span_name == "dummy_span" + + +def test_evaluator_runner_sampler_multiple_rules(monkeypatch): + monkeypatch.setenv( + EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, + json.dumps( + [ + {"sample_rate": 0.5, "evaluator_label": "ragas_faithfulness", "span_name": "dummy_span"}, + {"sample_rate": 0.2, "evaluator_label": "ragas_faithfulness", "span_name": "dummy_span_2"}, + ] + ), + ) + sampling_rules = EvaluatorRunnerSampler().rules + assert len(sampling_rules) == 2 + assert sampling_rules[0].sample_rate == 0.5 + assert sampling_rules[0].evaluator_label == "ragas_faithfulness" + assert sampling_rules[0].span_name == "dummy_span" + + assert sampling_rules[1].sample_rate == 0.2 + assert sampling_rules[1].evaluator_label == "ragas_faithfulness" + assert sampling_rules[1].span_name == "dummy_span_2" + + +def test_evaluator_runner_sampler_no_rule_label_or_name(monkeypatch): + monkeypatch.setenv( + EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, + json.dumps([{"sample_rate": 0.5}]), + ) + sampling_rules = EvaluatorRunnerSampler().rules + assert len(sampling_rules) == 1 + assert sampling_rules[0].sample_rate == 0.5 + assert sampling_rules[0].evaluator_label == EvaluatorRunnerSamplingRule.NO_RULE + assert sampling_rules[0].span_name == EvaluatorRunnerSamplingRule.NO_RULE + + +def test_evaluator_sampler_invalid_json(monkeypatch, mock_evaluator_sampler_logs): + monkeypatch.setenv( + EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, + "not a json", + ) + + with override_global_config({"_raise": True}): + with pytest.raises(ValueError): + EvaluatorRunnerSampler().rules + + with override_global_config({"_raise": False}): + sampling_rules = EvaluatorRunnerSampler().rules + assert len(sampling_rules) == 0 + mock_evaluator_sampler_logs.warning.assert_called_once_with( + "Failed to parse evaluator sampling rules of: `not a json`", exc_info=True + ) + + +def test_evaluator_sampler_invalid_rule_not_a_list(monkeypatch, mock_evaluator_sampler_logs): + monkeypatch.setenv( + EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, + json.dumps({"sample_rate": 0.5, "evaluator_label": "ragas_faithfulness", "span_name": "dummy_span"}), + ) + + with override_global_config({"_raise": True}): + with pytest.raises(ValueError): + EvaluatorRunnerSampler().rules + + with override_global_config({"_raise": False}): + sampling_rules = EvaluatorRunnerSampler().rules + assert len(sampling_rules) == 0 + mock_evaluator_sampler_logs.warning.assert_called_once_with( + "Evaluator sampling rules must be a list of dictionaries", exc_info=True + ) + + +def test_evaluator_sampler_invalid_rule_missing_sample_rate(monkeypatch, mock_evaluator_sampler_logs): + monkeypatch.setenv( + EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, + json.dumps([{"sample_rate": 0.1, "span_name": "dummy"}, {"span_name": "dummy2"}]), + ) + + with override_global_config({"_raise": True}): + with pytest.raises(KeyError): + EvaluatorRunnerSampler().rules + + with override_global_config({"_raise": False}): + sampling_rules = EvaluatorRunnerSampler().rules + assert len(sampling_rules) == 1 + mock_evaluator_sampler_logs.warning.assert_called_once_with( + 'No sample_rate provided for sampling rule: {"span_name": "dummy2"}', exc_info=True + ) + + +def test_evaluator_runner_sampler_no_rules_samples_all(monkeypatch): + iterations = int(1e4) + + sampled = sum(EvaluatorRunnerSampler().sample("ragas_faithfulness", Span(name=str(i))) for i in range(iterations)) + + deviation = abs(sampled - (iterations)) / (iterations) + assert deviation < 0.05 + + +def test_evaluator_sampling_rule_matches(monkeypatch): + sample_rate = 0.5 + span_name_rule = "dummy_span" + evaluator_label_rule = "ragas_faithfulness" + + for rule in [ + {"evaluator_label": evaluator_label_rule}, + {"evaluator_label": evaluator_label_rule, "span_name": span_name_rule}, + {"span_name": span_name_rule}, + ]: + rule["sample_rate"] = sample_rate + with override_env({EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR: json.dumps([rule])}): + iterations = int(1e4 / sample_rate) + sampled = sum( + EvaluatorRunnerSampler().sample(evaluator_label_rule, Span(name=span_name_rule)) + for i in range(iterations) + ) + + deviation = abs(sampled - (iterations * sample_rate)) / (iterations * sample_rate) + assert deviation < 0.05 + + +def test_evaluator_sampling_does_not_match_samples_all(monkeypatch): + sample_rate = 0.5 + span_name_rule = "dummy_span" + evaluator_label_rule = "ragas_faithfulness" + + for rule in [ + {"evaluator_label": evaluator_label_rule}, + {"evaluator_label": evaluator_label_rule, "span_name": span_name_rule}, + {"span_name": span_name_rule}, + ]: + rule["sample_rate"] = sample_rate + with override_env({EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR: json.dumps([rule])}): + iterations = int(1e4 / sample_rate) + + label_and_span = "not a matching label", Span(name="not matching span name") + + assert EvaluatorRunnerSampler().rules[0].matches(*label_and_span) is False + + sampled = sum(EvaluatorRunnerSampler().sample(*label_and_span) for i in range(iterations)) + + deviation = abs(sampled - (iterations)) / (iterations) + assert deviation < 0.05 diff --git a/tests/llmobs/test_llmobs_service.py b/tests/llmobs/test_llmobs_service.py index 61f7f72b4f2..2672c8a8921 100644 --- a/tests/llmobs/test_llmobs_service.py +++ b/tests/llmobs/test_llmobs_service.py @@ -1567,10 +1567,10 @@ def test_llmobs_fork_evaluator_runner_run(monkeypatch): llmobs_service.enable(_tracer=DummyTracer(), ml_app="test_app", api_key="test_api_key") pid = os.fork() if pid: # parent - llmobs_service._instance._evaluator_runner.enqueue({"span_id": "123", "trace_id": "456"}) + llmobs_service._instance._evaluator_runner.enqueue({"span_id": "123", "trace_id": "456"}, None) assert len(llmobs_service._instance._evaluator_runner._buffer) == 1 else: # child - llmobs_service._instance._evaluator_runner.enqueue({"span_id": "123", "trace_id": "456"}) + llmobs_service._instance._evaluator_runner.enqueue({"span_id": "123", "trace_id": "456"}, None) assert len(llmobs_service._instance._evaluator_runner._buffer) == 1 llmobs_service.disable() os._exit(12) From 79ea26feec8176f80676565279d6da8c16928608 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Wed, 9 Oct 2024 18:31:50 +0100 Subject: [PATCH 8/9] refactor(di): migrate to the product interface (#10943) We migrate all the debugging products to the product plugin interface for automatic life-cycle management. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/bootstrap/preload.py | 24 -------------- ddtrace/debugging/_debugger.py | 3 -- ddtrace/debugging/_products/__init__.py | 0 .../_products/code_origin/__init__.py | 0 .../debugging/_products/code_origin/span.py | 31 +++++++++++++++++++ .../_products/dynamic_instrumentation.py | 31 +++++++++++++++++++ .../debugging/_products/exception_replay.py | 31 +++++++++++++++++++ ddtrace/internal/symbol_db/__init__.py | 9 +++--- ddtrace/internal/symbol_db/product.py | 31 +++++++++++++++++++ pyproject.toml | 4 +++ tests/debugging/exploration/debugger.py | 5 +++ tests/debugging/test_api.py | 1 + 12 files changed, 138 insertions(+), 32 deletions(-) create mode 100644 ddtrace/debugging/_products/__init__.py create mode 100644 ddtrace/debugging/_products/code_origin/__init__.py create mode 100644 ddtrace/debugging/_products/code_origin/span.py create mode 100644 ddtrace/debugging/_products/dynamic_instrumentation.py create mode 100644 ddtrace/debugging/_products/exception_replay.py create mode 100644 ddtrace/internal/symbol_db/product.py diff --git a/ddtrace/bootstrap/preload.py b/ddtrace/bootstrap/preload.py index 93eba3cc6e6..99481cee858 100644 --- a/ddtrace/bootstrap/preload.py +++ b/ddtrace/bootstrap/preload.py @@ -6,8 +6,6 @@ import os # noqa:I001 from ddtrace import config # noqa:F401 -from ddtrace.debugging._config import di_config # noqa:F401 -from ddtrace.debugging._config import er_config # noqa:F401 from ddtrace.settings.profiling import config as profiling_config # noqa:F401 from ddtrace.internal.logger import get_logger # noqa:F401 from ddtrace.internal.module import ModuleWatchdog # noqa:F401 @@ -17,9 +15,7 @@ from ddtrace.internal.utils.formats import asbool # noqa:F401 from ddtrace.internal.utils.formats import parse_tags_str # noqa:F401 from ddtrace.settings.asm import config as asm_config # noqa:F401 -from ddtrace.settings.code_origin import config as co_config # noqa:F401 from ddtrace.settings.crashtracker import config as crashtracker_config -from ddtrace.settings.symbol_db import config as symdb_config # noqa:F401 from ddtrace import tracer @@ -72,26 +68,6 @@ def register_post_preload(func: t.Callable) -> None: except Exception: log.error("failed to enable profiling", exc_info=True) -if symdb_config.enabled: - from ddtrace.internal import symbol_db - - symbol_db.bootstrap() - -if di_config.enabled: # Dynamic Instrumentation - from ddtrace.debugging import DynamicInstrumentation - - DynamicInstrumentation.enable() - -if co_config.span.enabled: - from ddtrace.debugging._origin.span import SpanCodeOriginProcessor - - SpanCodeOriginProcessor.enable() - -if er_config.enabled: # Exception Replay - from ddtrace.debugging._exception.replay import SpanExceptionHandler - - SpanExceptionHandler.enable() - if config._runtime_metrics_enabled: RuntimeWorker.enable() diff --git a/ddtrace/debugging/_debugger.py b/ddtrace/debugging/_debugger.py index 0a1163aa27e..abc5cf8796b 100644 --- a/ddtrace/debugging/_debugger.py +++ b/ddtrace/debugging/_debugger.py @@ -54,7 +54,6 @@ from ddtrace.debugging._signal.tracing import SpanDecoration from ddtrace.debugging._uploader import LogsIntakeUploaderV1 from ddtrace.debugging._uploader import UploaderProduct -from ddtrace.internal import atexit from ddtrace.internal import compat from ddtrace.internal.logger import get_logger from ddtrace.internal.metrics import Metrics @@ -305,7 +304,6 @@ def enable(cls, run_module: bool = False) -> None: debugger.start() - atexit.register(cls.disable) register_post_run_module_hook(cls._on_run_module) telemetry_writer.product_activated(TELEMETRY_APM_PRODUCT.DYNAMIC_INSTRUMENTATION, True) @@ -326,7 +324,6 @@ def disable(cls, join: bool = True) -> None: remoteconfig_poller.unregister("LIVE_DEBUGGING") - atexit.unregister(cls.disable) unregister_post_run_module_hook(cls._on_run_module) cls._instance.stop(join=join) diff --git a/ddtrace/debugging/_products/__init__.py b/ddtrace/debugging/_products/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/debugging/_products/code_origin/__init__.py b/ddtrace/debugging/_products/code_origin/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/debugging/_products/code_origin/span.py b/ddtrace/debugging/_products/code_origin/span.py new file mode 100644 index 00000000000..e373fbeae7d --- /dev/null +++ b/ddtrace/debugging/_products/code_origin/span.py @@ -0,0 +1,31 @@ +from ddtrace.settings.code_origin import config + + +# TODO[gab]: Uncomment this when the feature is ready +# requires = ["tracer"] + + +def post_preload(): + pass + + +def start(): + if config.span.enabled: + from ddtrace.debugging._origin.span import SpanCodeOriginProcessor + + SpanCodeOriginProcessor.enable() + + +def restart(join=False): + pass + + +def stop(join=False): + if config.span.enabled: + from ddtrace.debugging._origin.span import SpanCodeOriginProcessor + + SpanCodeOriginProcessor.disable() + + +def at_exit(join=False): + stop(join=join) diff --git a/ddtrace/debugging/_products/dynamic_instrumentation.py b/ddtrace/debugging/_products/dynamic_instrumentation.py new file mode 100644 index 00000000000..136d5692ec8 --- /dev/null +++ b/ddtrace/debugging/_products/dynamic_instrumentation.py @@ -0,0 +1,31 @@ +from ddtrace.settings.dynamic_instrumentation import config + + +requires = ["remote-configuration"] + + +def post_preload(): + pass + + +def start(): + if config.enabled: + from ddtrace.debugging import DynamicInstrumentation + + DynamicInstrumentation.enable() + + +def restart(join=False): + # Nothing to do + pass + + +def stop(join=False): + if config.enabled: + from ddtrace.debugging import DynamicInstrumentation + + DynamicInstrumentation.disable(join=join) + + +def at_exit(join=False): + stop(join=join) diff --git a/ddtrace/debugging/_products/exception_replay.py b/ddtrace/debugging/_products/exception_replay.py new file mode 100644 index 00000000000..c27f7ca5b5c --- /dev/null +++ b/ddtrace/debugging/_products/exception_replay.py @@ -0,0 +1,31 @@ +from ddtrace.debugging._config import er_config + + +# TODO[gab]: Uncomment this when the feature is ready +# requires = ["tracer"] + + +def post_preload(): + pass + + +def start(): + if er_config.enabled: + from ddtrace.debugging._exception.replay import SpanExceptionHandler + + SpanExceptionHandler.enable() + + +def restart(join=False): + pass + + +def stop(join=False): + if er_config.enabled: + from ddtrace.debugging._exception.replay import SpanExceptionHandler + + SpanExceptionHandler.disable() + + +def at_exit(join=False): + stop(join=join) diff --git a/ddtrace/internal/symbol_db/__init__.py b/ddtrace/internal/symbol_db/__init__.py index b78622ebee3..cfeefbdd849 100644 --- a/ddtrace/internal/symbol_db/__init__.py +++ b/ddtrace/internal/symbol_db/__init__.py @@ -1,4 +1,3 @@ -from ddtrace.internal import forksafe from ddtrace.internal.remoteconfig.worker import remoteconfig_poller from ddtrace.internal.symbol_db.remoteconfig import SymbolDatabaseAdapter from ddtrace.settings.symbol_db import config as symdb_config @@ -14,7 +13,7 @@ def bootstrap(): # Start the RCM subscriber to determine if and when to upload symbols. remoteconfig_poller.register("LIVE_DEBUGGING_SYMBOL_DB", SymbolDatabaseAdapter()) - @forksafe.register - def _(): - remoteconfig_poller.unregister("LIVE_DEBUGGING_SYMBOL_DB") - remoteconfig_poller.register("LIVE_DEBUGGING_SYMBOL_DB", SymbolDatabaseAdapter()) + +def restart(): + remoteconfig_poller.unregister("LIVE_DEBUGGING_SYMBOL_DB") + remoteconfig_poller.register("LIVE_DEBUGGING_SYMBOL_DB", SymbolDatabaseAdapter()) diff --git a/ddtrace/internal/symbol_db/product.py b/ddtrace/internal/symbol_db/product.py new file mode 100644 index 00000000000..c6c165e9577 --- /dev/null +++ b/ddtrace/internal/symbol_db/product.py @@ -0,0 +1,31 @@ +from ddtrace.settings.symbol_db import config + + +requires = ["remote-configuration"] + + +def post_preload(): + pass + + +def start(): + if config.enabled: + from ddtrace.internal import symbol_db + + symbol_db.bootstrap() + + +def restart(join=False): + if not config._force: + from ddtrace.internal import symbol_db + + symbol_db.restart() + + +def stop(join=False): + # Controlled via RC + pass + + +def at_exit(join=False): + stop(join=join) diff --git a/pyproject.toml b/pyproject.toml index 64adf321d43..40b97edd85c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,11 @@ ddtrace = "ddtrace.contrib.pytest.plugin" "ddtrace.pytest_benchmark" = "ddtrace.contrib.pytest_benchmark.plugin" [project.entry-points.'ddtrace.products'] +"code-origin-for-spans" = "ddtrace.debugging._products.code_origin.span" +"dynamic-instrumentation" = "ddtrace.debugging._products.dynamic_instrumentation" +"exception-replay" = "ddtrace.debugging._products.exception_replay" "remote-configuration" = "ddtrace.internal.remoteconfig.product" +"symbol-database" = "ddtrace.internal.symbol_db.product" [project.urls] "Bug Tracker" = "https://github.com/DataDog/dd-trace-py/issues" diff --git a/tests/debugging/exploration/debugger.py b/tests/debugging/exploration/debugger.py index 084c56cac07..a78de54efca 100644 --- a/tests/debugging/exploration/debugger.py +++ b/tests/debugging/exploration/debugger.py @@ -1,3 +1,4 @@ +import atexit import os import sys from types import ModuleType @@ -237,6 +238,10 @@ def enable(cls) -> None: cls._instance.__uploader__.get_collector().on_snapshot = cls.on_snapshot + # Register the debugger to be disabled at exit manually because we are + # not being managed by the product manager. + atexit.register(cls.disable) + @classmethod def disable(cls, join: bool = True) -> None: registry = cls._instance._probe_registry diff --git a/tests/debugging/test_api.py b/tests/debugging/test_api.py index f4af5285f15..a5d0f51c77c 100644 --- a/tests/debugging/test_api.py +++ b/tests/debugging/test_api.py @@ -36,6 +36,7 @@ def test_debugger_fork(): import os import sys + import ddtrace.auto # noqa: F401 from ddtrace.debugging import DynamicInstrumentation from ddtrace.internal.service import ServiceStatus From 857a619fb811647c6944b7fe54a763f86a3084d4 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 9 Oct 2024 19:45:14 +0200 Subject: [PATCH 9/9] ci: update iast-tdd-propagation suite (#10982) IAST: Update iast-tdd-propagation suite so it doesn't fail 1. Honor `FLASK_RUN_PORT` env var instead of `8000` 2. Update requests version in asserts ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .gitlab/tests/appsec.yml | 1 - tests/appsec/iast_tdd_propagation/flask_orm_app.py | 4 +++- tests/appsec/iast_tdd_propagation/flask_propagation_app.py | 6 +++++- tests/appsec/iast_tdd_propagation/flask_taint_sinks_app.py | 6 +++++- tests/appsec/iast_tdd_propagation/test_flask.py | 4 ++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.gitlab/tests/appsec.yml b/.gitlab/tests/appsec.yml index be67ac46e59..7b3667a954b 100644 --- a/.gitlab/tests/appsec.yml +++ b/.gitlab/tests/appsec.yml @@ -19,7 +19,6 @@ appsec iast: appsec iast tdd_propagation: extends: .test_base_riot_snapshot - allow_failure: true parallel: 2 variables: SUITE_NAME: "appsec_iast_tdd_propagation" diff --git a/tests/appsec/iast_tdd_propagation/flask_orm_app.py b/tests/appsec/iast_tdd_propagation/flask_orm_app.py index 19c516bebcf..670228f1880 100644 --- a/tests/appsec/iast_tdd_propagation/flask_orm_app.py +++ b/tests/appsec/iast_tdd_propagation/flask_orm_app.py @@ -25,6 +25,8 @@ orm = os.getenv("FLASK_ORM", "sqlite") +port = int(os.getenv("FLASK_RUN_PORT", 8000)) + orm_impl = importlib.import_module(f"{orm}_impl") @@ -94,4 +96,4 @@ def untainted_view(): if __name__ == "__main__": ddtrace_iast_flask_patch() - app.run(debug=False, port=8000) + app.run(debug=False, port=port) diff --git a/tests/appsec/iast_tdd_propagation/flask_propagation_app.py b/tests/appsec/iast_tdd_propagation/flask_propagation_app.py index 1c1b23f9fbb..53764dde655 100644 --- a/tests/appsec/iast_tdd_propagation/flask_propagation_app.py +++ b/tests/appsec/iast_tdd_propagation/flask_propagation_app.py @@ -1,9 +1,13 @@ +import os + from flask_propagation_views import create_app from ddtrace import auto # noqa: F401 +port = int(os.getenv("FLASK_RUN_PORT", 8000)) + app = create_app() if __name__ == "__main__": - app.run(debug=False, port=8000) + app.run(debug=False, port=port) diff --git a/tests/appsec/iast_tdd_propagation/flask_taint_sinks_app.py b/tests/appsec/iast_tdd_propagation/flask_taint_sinks_app.py index b140a953812..0b8536f1664 100644 --- a/tests/appsec/iast_tdd_propagation/flask_taint_sinks_app.py +++ b/tests/appsec/iast_tdd_propagation/flask_taint_sinks_app.py @@ -1,9 +1,13 @@ +import os + from flask_taint_sinks_views import create_app from ddtrace import auto # noqa: F401 +port = int(os.getenv("FLASK_RUN_PORT", 8000)) + app = create_app() if __name__ == "__main__": - app.run(debug=False, port=8000) + app.run(debug=False, port=port) diff --git a/tests/appsec/iast_tdd_propagation/test_flask.py b/tests/appsec/iast_tdd_propagation/test_flask.py index d973831928c..c4422300343 100644 --- a/tests/appsec/iast_tdd_propagation/test_flask.py +++ b/tests/appsec/iast_tdd_propagation/test_flask.py @@ -107,8 +107,8 @@ def test_iast_flask_headers(): assert tainted_response.status_code == 200 content = json.loads(tainted_response.content) assert content["param"] == [ - ["Host", "0.0.0.0:8000"], - ["User-Agent", "python-requests/2.31.0"], + ["Host", f"0.0.0.0:{_PORT}"], + ["User-Agent", "python-requests/2.32.3"], ["Accept-Encoding", "gzip, deflate, br"], ["Accept", "*/*"], ["Connection", "keep-alive"],