Merge branch 'main' into robertomonteromiguel/onboarding_rh9_ami_update

docs/edit/format.md

@@ -1,4 +1,4 @@
-System tests code is in python, and is linted/formated using [black](https://black.readthedocs.io/en/stable/) and [pylint](https://pylint.readthedocs.io/en/latest/).
+System tests code is in python, and is linted/formated using [mypy](https://mypy.readthedocs.io/en/stable/), [black](https://black.readthedocs.io/en/stable/) and [pylint](https://pylint.readthedocs.io/en/latest/).
 
 Ensure you meet the other pre-reqs in [README.md](../../README.md#requirements)
 Then, run the linter with:

docs/weblog/README.md

@@ -512,6 +512,23 @@ By default, the generated event has the following specification:
 
 Values can be changed with the query params called `event_name`.
 
+### GET '/inferred-proxy/span-creation'
+
+This endpoint is supposed to be hit with the necessary headers that are used to create inferred proxy
+spans for routers such as AWS API Gateway. Not including the headers means a span will not be created by the tracer
+if the feature exists.
+
+The endpoint supports the following query parameters:
+ - `status_code`: str containing status code to used in API response
+
+The headers necessary to create a span with example values:
+  `x-dd-proxy-request-time-ms`: start time in milliseconds
+  `x-dd-proxy-path`: "/api/data",
+  `x-dd-proxy-httpmethod`: "GET",
+  `x-dd-proxy-domain-name`: "system-tests-api-gateway.com",
+  `x-dd-proxy-stage`: "staging",
+  `x-dd-proxy`: "aws-apigateway",
+
 ### GET /users
 
 This endpoint calls the appsec blocking SDK functions used for blocking users. If the expected parameter matches one of

format.sh

@@ -38,11 +38,22 @@ source venv/bin/activate
 
 echo "Checking Python files..."
 if [ "$COMMAND" == "fix" ]; then
-  black .
+  black --quiet .
 else
   black --check --diff .
 fi
-pylint utils  # pylint does not have a fix mode
+
+echo "Running mypy type checks..."
+if ! mypy --config pyproject.toml; then
+  echo "Mypy type checks failed. Please fix the errors above. 💥 💔 💥"
+  exit 1
+fi
+
+echo "Running pylint checks..."
+if ! pylint utils; then
+  echo "Pylint checks failed. Please fix the errors above. 💥 💔 💥"
+  exit 1
+fi
 
 echo "Checking trailing whitespaces..."
 INCLUDE_PATTERN='.*\.(md|yml|yaml|sh|cs|Dockerfile|java|sql|ts|js|php)$'

manifests/cpp.yml

@@ -137,6 +137,8 @@ tests/:
       Test_DsmSQS: missing_feature
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   parametric/:

manifests/dotnet.yml

@@ -360,6 +360,8 @@ tests/:
       Test_DsmSQS: v2.48.0
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

manifests/golang.yml

@@ -482,6 +482,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         "*": irrelevant
         net-http: missing_feature (Endpoint not implemented)
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   parametric/:

manifests/java.yml

@@ -1426,6 +1426,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         "*": irrelevant
         spring-boot: bug (AIDM-325)
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_mongo.py:
       Test_Mongo: bug (APMAPI-729)
     test_otel_drop_in.py:

manifests/nodejs.yml

@@ -39,6 +39,7 @@ refs:
   - &ref_5_24_0 '>=5.24.0 || ^4.48.0'
   - &ref_5_25_0 '>=5.25.0 || ^4.49.0'
   - &ref_5_26_0 '>=5.26.0 || ^4.50.0'
+  - &ref_5_27_0 '>=5.27.0 || ^4.51.0'
 
 tests/:
   apm_tracing_e2e/:
@@ -628,6 +629,10 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         express4: *ref_5_20_0
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation:
+          '*': irrelevant
+          express4: *ref_5_26_0
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:
@@ -640,7 +645,8 @@ tests/:
       Test_Config_TraceEnabled: *ref_4_3_0
       Test_Config_TraceLogDirectory: missing_feature
       Test_Config_UnifiedServiceTagging: *ref_5_25_0
-    test_crashtracking.py: missing_feature
+    test_crashtracking.py:
+      Test_Crashtracking: *ref_5_27_0
     test_dynamic_configuration.py:
       TestDynamicConfigSamplingRules: *ref_5_16_0
       TestDynamicConfigTracingEnabled: *ref_5_4_0

manifests/php.yml

@@ -296,6 +296,8 @@ tests/:
       Test_DsmSQS: missing_feature
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   parametric/:

manifests/python.yml

@@ -723,6 +723,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         flask-poc: v2.8.0
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

manifests/ruby.yml

@@ -364,6 +364,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         rails70: missing_feature (Endpoint not implemented)
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

pyproject.toml

@@ -102,6 +102,13 @@ allow_no_jira_ticket_for_bugs = [
     "tests/parametric/test_config_consistency.py::Test_Config_TraceLogDirectory",
 ]
 
+[tool.mypy]
+files = ["utils/parametric", "tests/parametric"]
+ignore_missing_imports = true
+disable_error_code = ["no-redef"]
+exclude = 'utils/parametric/_library_client\.py|^(?!utils/parametric|tests/parametric).*$'
+follow_imports = "skip"
+
 [tool.pylint]
 init-hook='import sys; sys.path.append(".")'
 max-line-length = 120

requirements.txt

@@ -11,6 +11,7 @@ pylint==3.0.4
 python-dateutil==2.8.2
 msgpack==1.0.4
 watchdog==3.0.0
+mypy==1.0.0
 
 aiohttp==3.9.0
 yarl==1.9.4

tests/debugger/probes/pii_line.json

@@ -0,0 +1,18 @@
+[
+    {
+        "language": "",
+        "pii": "",
+        "id": "log170aa-acda-4453-9111-1478a600line",
+        "where": {
+           "typeName": null,
+           "sourceFile": "ACTUAL_SOURCE_FILE",
+            "lines": [
+                "33"
+            ]
+        },
+        "captureSnapshot": true,
+        "capture": {
+            "maxFieldCount": 200
+        }
+    }
+]

tests/debugger/test_debugger_pii.py

@@ -121,35 +121,38 @@ def filter(keys_to_filter):
 @features.debugger_pii_redaction
 @scenarios.debugger_pii_redaction
 class Test_Debugger_PII_Redaction(base._Base_Debugger_Test):
-    def _setup(self):
-        probes = base.read_probes("pii")
+    def _setup(self, probes_file):
+        probes = base.read_probes(probes_file)
         self.expected_probe_ids = base.extract_probe_ids(probes)
         self.rc_state = rc.send_debugger_command(probes, version=1)
 
         interfaces.agent.wait_for(self.wait_for_all_probes_installed, timeout=30)
 
         self.weblog_responses = [weblog.get("/debugger/pii")]
 
-    def _test(self, redacted_keys, redacted_types):
+    def _test(self, redacted_keys, redacted_types, line_probe=False):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()
         self.assert_all_weblog_responses_ok()
 
-        self._validate_pii_keyword_redaction(redacted_keys)
-        self._validate_pii_type_redaction(redacted_types)
+        self._validate_pii_keyword_redaction(redacted_keys, line_probe=line_probe)
+        self._validate_pii_type_redaction(redacted_types, line_probe=line_probe)
 
     def setup_pii_redaction_full(self):
-        self._setup()
+        self._setup("pii")
 
     @missing_feature(context.library < "[email protected]", reason="keywords are not fully redacted")
     @missing_feature(context.library < "[email protected]", reason="keywords are not fully redacted")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
+    # Ruby requires @irrelevant rather than @missing_feature to skip setup
+    # for this test (which will interfere with the line probe test).
+    @irrelevant(context.library == "ruby", reason="Local variable capture not implemented for method probes")
     def test_pii_redaction_full(self):
         self._test(REDACTED_KEYS, REDACTED_TYPES)
 
     def setup_pii_redaction_java_1_33(self):
-        self._setup()
+        self._setup("pii")
 
     @irrelevant(context.library != "[email protected]", reason="not relevant for other version")
     def test_pii_redaction_java_1_33(self):
@@ -170,7 +173,7 @@ def test_pii_redaction_java_1_33(self):
         )
 
     def setup_pii_redaction_dotnet_2_50(self):
-        self._setup()
+        self._setup("pii")
 
     @irrelevant(context.library != "[email protected]", reason="not relevant for other version")
     @bug(
@@ -179,7 +182,14 @@ def setup_pii_redaction_dotnet_2_50(self):
     def test_pii_redaction_dotnet_2_50(self):
         self._test(filter(["applicationkey", "connectionstring"]), REDACTED_TYPES)
 
-    def _validate_pii_keyword_redaction(self, should_redact_field_names):
+    def setup_pii_redaction_line(self):
+        self._setup("pii_line")
+
+    @irrelevant(context.library != "ruby", reason="Ruby needs to use line probes to capture variables")
+    def test_pii_redaction_line(self):
+        self._test(REDACTED_KEYS, REDACTED_TYPES, True)
+
+    def _validate_pii_keyword_redaction(self, should_redact_field_names, line_probe=False):
         agent_logs_endpoint_requests = list(interfaces.agent.get_data(path_filters="/api/v2/logs"))
         not_redacted = []
         not_found = list(set(should_redact_field_names))
@@ -193,12 +203,21 @@ def _validate_pii_keyword_redaction(self, should_redact_field_names):
 
                     if snapshot:
                         for field_name in should_redact_field_names:
-                            fields = snapshot["captures"]["return"]["locals"]["pii"]["fields"]
-
-                            if field_name in fields:
+                            if line_probe:
+                                fields = snapshot["captures"]["lines"]["33"]["locals"]["pii"]["fields"]
+                            else:
+                                fields = snapshot["captures"]["return"]["locals"]["pii"]["fields"]
+
+                            # Ruby prefixes instance variable names with @
+                            if context.library == "ruby":
+                                check_field_name = "@" + field_name
+                            else:
+                                check_field_name = field_name
+
+                            if check_field_name in fields:
                                 not_found.remove(field_name)
 
-                                if "value" in fields[field_name]:
+                                if "value" in fields[check_field_name]:
                                     not_redacted.append(field_name)
         error_message = ""
         if not_redacted:
@@ -212,7 +231,7 @@ def _validate_pii_keyword_redaction(self, should_redact_field_names):
         if error_message != "":
             raise ValueError(error_message)
 
-    def _validate_pii_type_redaction(self, should_redact_types):
+    def _validate_pii_type_redaction(self, should_redact_types, line_probe=False):
         agent_logs_endpoint_requests = list(interfaces.agent.get_data(path_filters="/api/v2/logs"))
         not_redacted = []
 
@@ -225,7 +244,10 @@ def _validate_pii_type_redaction(self, should_redact_types):
 
                     if snapshot:
                         for type_name in should_redact_types:
-                            type_info = snapshot["captures"]["return"]["locals"][type_name]
+                            if line_probe:
+                                type_info = snapshot["captures"]["lines"]["33"]["locals"][type_name]
+                            else:
+                                type_info = snapshot["captures"]["return"]["locals"][type_name]
 
                             if "fields" in type_info:
                                 not_redacted.append(type_name)

tests/debugger/test_debugger_probe_snapshot.py

@@ -4,7 +4,7 @@
 
 import tests.debugger.utils as base
 
-from utils import scenarios, interfaces, weblog, features, remote_config as rc, bug
+from utils import scenarios, interfaces, weblog, features, remote_config as rc, bug, missing_feature, context
 
 
 @features.debugger
@@ -41,6 +41,7 @@ def setup_span_method_probe_snaphots(self):
         ]
 
     @bug(library="python", reason="DEBUG-2708, DEBUG-2709")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_span_method_probe_snaphots(self):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()
@@ -61,6 +62,7 @@ def setup_span_decoration_method_probe_snaphots(self):
         ]
 
     @bug(library="python", reason="DEBUG-2708, DEBUG-2709")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_span_decoration_method_probe_snaphots(self):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()
@@ -105,6 +107,7 @@ def setup_span_decoration_line_probe_snaphots(self):
             weblog.get("/debugger/span-decoration/asd/1"),
         ]
 
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_span_decoration_line_probe_snaphots(self):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()

tests/debugger/test_debugger_probe_status.py

@@ -4,7 +4,7 @@
 
 import tests.debugger.utils as base
 
-from utils import scenarios, features, remote_config as rc, bug, context
+from utils import weblog, scenarios, features, remote_config as rc, bug, context, missing_feature
 
 
 @features.debugger
@@ -40,6 +40,7 @@ def setup_probe_status_metric(self):
 
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_probe_status_metric(self):
         self._assert()
 
@@ -49,6 +50,7 @@ def setup_probe_status_span(self):
 
         self._setup(probes)
 
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_probe_status_span(self):
         self._assert()
 
@@ -60,6 +62,7 @@ def setup_probe_status_spandecoration(self):
 
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_probe_status_spandecoration(self):
         self._assert()