From 97c5bd39c691f9bb8acdfc47f7ab195e20850759 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Wed, 20 Nov 2024 09:36:48 +0100 Subject: [PATCH 01/18] [python] Skip failing test for 2.16.3 release (#3498) --- tests/test_schemas.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index f29373d10f..b381418dde 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -28,13 +28,17 @@ def test_library_schema_full(self): ("/debugger/v1/diagnostics", "$[].content"), # DEBUG-2864 ] - if context.library == "python@2.16.2" and context.scenario is scenarios.debugger_expression_language: + if ( + context.library in ("python@2.16.2", "python@2.16.3") + and context.scenario is scenarios.debugger_expression_language + ): excluded_points.append(("/debugger/v1/input", "$[].debugger.snapshot.stack[].lineNumber")) interfaces.library.assert_schema_points(excluded_points) @bug( - context.library == "python@2.16.2" and context.scenario is scenarios.debugger_expression_language, + context.library in ("python@2.16.2", "python@2.16.3") + and context.scenario is scenarios.debugger_expression_language, reason="APMRP-360", ) def test_python_debugger_line_number(self): From c3a1f309c929e1ed2b89e6b1d73d8565c459a77d Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:49:08 +0100 Subject: [PATCH 02/18] Fix default network name (docker mode) (#3501) --- run.sh | 2 +- utils/_context/containers.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 861d46b2b5..a9b62e4137 100755 --- a/run.sh +++ b/run.sh @@ -142,7 +142,7 @@ function activate_venv() { } function network_name() { - perl -ne '/_NETWORK_NAME = "(.*)"/ and print "$1\n"' utils/_context/containers.py + perl -ne '/_DEFAULT_NETWORK_NAME = "(.*)"/ and print "$1\n"' utils/_context/containers.py } function ensure_network() { diff --git a/utils/_context/containers.py b/utils/_context/containers.py index dbdc36803c..a15f4c7071 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -45,7 +45,8 @@ def _get_client(): raise e -_NETWORK_NAME = "bridge" if "GITLAB_CI" in os.environ else "system-tests_default" +_DEFAULT_NETWORK_NAME = "system-tests_default" +_NETWORK_NAME = "bridge" if "GITLAB_CI" in os.environ else _DEFAULT_NETWORK_NAME def create_network(): From 9baf96526019d133b1c3a840116d3557ad9584f4 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Wed, 20 Nov 2024 10:21:08 +0100 Subject: [PATCH 03/18] Fix typo --- .github/workflows/python-temporary-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-temporary-test.yml b/.github/workflows/python-temporary-test.yml index 086b4f9fba..09143003d5 100644 --- a/.github/workflows/python-temporary-test.yml +++ b/.github/workflows/python-temporary-test.yml @@ -49,5 +49,5 @@ jobs: uses: actions/upload-artifact@v4 with: # log name convention to respect : logs_$SCENARIO-FAMILY_$LIBRARY_$WEBLOG_$CI-ENVIRONMENT - name: logs_endtoend_{{ matrix.attempt }} + name: logs_endtoend_${{ matrix.attempt }} path: artifact.tar.gz From 8f4d244a87157c48bae720bd6fbef22daa09ced9 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Wed, 20 Nov 2024 10:40:59 +0100 Subject: [PATCH 04/18] Delet temp test --- .github/workflows/python-temporary-test.yml | 53 --------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/workflows/python-temporary-test.yml diff --git a/.github/workflows/python-temporary-test.yml b/.github/workflows/python-temporary-test.yml deleted file mode 100644 index 09143003d5..0000000000 --- a/.github/workflows/python-temporary-test.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: python-temporary-test - -on: - workflow_dispatch: {} - schedule: - - cron: '00 05 * * 2-6' - -env: - REGISTRY: ghcr.io - -jobs: - end-to-end: - runs-on: ubuntu-latest - strategy: - matrix: - attempt: [1, 2, 3, 4 ,5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - fail-fast: false - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - repository: 'DataDog/system-tests' - ref: 'christophe-papazian/add_stderr_observability_for_nightly_on_standalone' - - name: Install runner - uses: ./.github/actions/install_runner - - name: Pull images - uses: ./.github/actions/pull_images - with: - library: python - weblog: uwsgi-poc - scenarios: '"APPSEC_STANDALONE"' - dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build agent - run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh -i agent - - name: Build weblog - id: build - run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh python -i weblog -w uwsgi-poc - - name: Run APPSEC_STANDALONE scenario - if: always() && steps.build.outcome == 'success' - run: ./run.sh APPSEC_STANDALONE - - - name: Compress logs - id: compress_logs - if: always() && steps.build.outcome == 'success' - run: tar -czvf artifact.tar.gz $(ls | grep logs) - - name: Upload artifact - if: always() && steps.compress_logs.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - # log name convention to respect : logs_$SCENARIO-FAMILY_$LIBRARY_$WEBLOG_$CI-ENVIRONMENT - name: logs_endtoend_${{ matrix.attempt }} - path: artifact.tar.gz From a1b9c54cc9035954ace839bf5ad703dd785301f9 Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:50:26 +0100 Subject: [PATCH 05/18] Onboarding: Extract Java version (#3503) --- .../provisions/auto-inject/auto-inject-tested_components.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject-tested_components.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject-tested_components.yml index fe017fe9b6..8ca6f02444 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject-tested_components.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject-tested_components.yml @@ -47,6 +47,8 @@ runtime_version="${runtime_version:1}" elif [ "$DD_LANG" == "python" ]; then export runtime_version=$(python --version 2>&1 | awk '{print $2}' | cut -d '.' -f 1,2) + elif [ "$DD_LANG" == "java" ]; then + export runtime_version=$(java -version 2>&1 | awk 'NR==1{ gsub(/"/,""); print $3 }') else export runtime_version="" fi @@ -104,6 +106,8 @@ runtime_version="${runtime_version:1}" elif [ "$DD_LANG" == "python" ]; then export runtime_version=$(python --version 2>&1 | awk '{print $2}' | cut -d '.' -f 1,2) + elif [ "$DD_LANG" == "java" ]; then + export runtime_version=$(java -version 2>&1 | awk 'NR==1{ gsub(/"/,""); print $3 }') else export runtime_version="" fi From 27c15a3c3c78aa70e440070a48c3eebbf3dbca9b Mon Sep 17 00:00:00 2001 From: Oleg Pudeyev <156273877+p-datadog@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:16:23 -0500 Subject: [PATCH 06/18] DEBUG-2334 Split debugger snapshot tests (#3466) Co-authored-by: Oleg Pudeyev --- .../probes/probe_snapshot_log_line.json | 15 ++++ .../probes/probe_snapshot_log_method.json | 13 ++++ ...log.json => probe_snapshot_log_mixed.json} | 0 ... probe_snapshot_span_decoration_line.json} | 12 --- ...robe_snapshot_span_decoration_method.json} | 22 ------ .../probes/probe_snapshot_span_method.json | 13 ++++ .../debugger/test_debugger_probe_snapshot.py | 77 ++++++++++++++++--- 7 files changed, 106 insertions(+), 46 deletions(-) create mode 100644 tests/debugger/probes/probe_snapshot_log_line.json create mode 100644 tests/debugger/probes/probe_snapshot_log_method.json rename tests/debugger/probes/{probe_snapshot_mix_log.json => probe_snapshot_log_mixed.json} (100%) rename tests/debugger/probes/{probe_snapshot_line.json => probe_snapshot_span_decoration_line.json} (83%) rename tests/debugger/probes/{probe_snapshot_method.json => probe_snapshot_span_decoration_method.json} (70%) create mode 100644 tests/debugger/probes/probe_snapshot_span_method.json diff --git a/tests/debugger/probes/probe_snapshot_log_line.json b/tests/debugger/probes/probe_snapshot_log_line.json new file mode 100644 index 0000000000..23ecbc8661 --- /dev/null +++ b/tests/debugger/probes/probe_snapshot_log_line.json @@ -0,0 +1,15 @@ +[ + { + "language": "", + "type": "", + "id": "log170aa-acda-4453-9111-1478a697line", + "where": { + "typeName": null, + "sourceFile": "ACTUAL_SOURCE_FILE", + "lines": [ + "20" + ] + } + } + +] \ No newline at end of file diff --git a/tests/debugger/probes/probe_snapshot_log_method.json b/tests/debugger/probes/probe_snapshot_log_method.json new file mode 100644 index 0000000000..40a8f7a55c --- /dev/null +++ b/tests/debugger/probes/probe_snapshot_log_method.json @@ -0,0 +1,13 @@ +[ + { + "language": "", + "type": "", + "id": "log170aa-acda-4453-9111-1478a6method", + "where": { + "typeName": "ACTUAL_TYPE_NAME", + "methodName": "LogProbe", + "sourceFile": null + }, + "evaluateAt": "EXIT" + } +] \ No newline at end of file diff --git a/tests/debugger/probes/probe_snapshot_mix_log.json b/tests/debugger/probes/probe_snapshot_log_mixed.json similarity index 100% rename from tests/debugger/probes/probe_snapshot_mix_log.json rename to tests/debugger/probes/probe_snapshot_log_mixed.json diff --git a/tests/debugger/probes/probe_snapshot_line.json b/tests/debugger/probes/probe_snapshot_span_decoration_line.json similarity index 83% rename from tests/debugger/probes/probe_snapshot_line.json rename to tests/debugger/probes/probe_snapshot_span_decoration_line.json index 4d2a819fd2..07f240a4fd 100644 --- a/tests/debugger/probes/probe_snapshot_line.json +++ b/tests/debugger/probes/probe_snapshot_span_decoration_line.json @@ -1,16 +1,4 @@ [ - { - "language": "", - "type": "", - "id": "log170aa-acda-4453-9111-1478a697line", - "where": { - "typeName": null, - "sourceFile": "ACTUAL_SOURCE_FILE", - "lines": [ - "20" - ] - } - }, { "language": "", "type": "", diff --git a/tests/debugger/probes/probe_snapshot_method.json b/tests/debugger/probes/probe_snapshot_span_decoration_method.json similarity index 70% rename from tests/debugger/probes/probe_snapshot_method.json rename to tests/debugger/probes/probe_snapshot_span_decoration_method.json index 583b254045..350c95a41e 100644 --- a/tests/debugger/probes/probe_snapshot_method.json +++ b/tests/debugger/probes/probe_snapshot_span_decoration_method.json @@ -1,26 +1,4 @@ [ - { - "language": "", - "type": "", - "id": "log170aa-acda-4453-9111-1478a6method", - "where": { - "typeName": "ACTUAL_TYPE_NAME", - "methodName": "LogProbe", - "sourceFile": null - }, - "evaluateAt": "EXIT" - }, - { - "language": "", - "type": "", - "id": "span70aa-acda-4453-9111-1478a6method", - "where": { - "typeName": "ACTUAL_TYPE_NAME", - "methodName": "SpanProbe", - "sourceFile": null - }, - "evaluateAt": "EXIT" - }, { "language": "", "type": "", diff --git a/tests/debugger/probes/probe_snapshot_span_method.json b/tests/debugger/probes/probe_snapshot_span_method.json new file mode 100644 index 0000000000..6b57f2917a --- /dev/null +++ b/tests/debugger/probes/probe_snapshot_span_method.json @@ -0,0 +1,13 @@ +[ + { + "language": "", + "type": "", + "id": "span70aa-acda-4453-9111-1478a6method", + "where": { + "typeName": "ACTUAL_TYPE_NAME", + "methodName": "SpanProbe", + "sourceFile": null + }, + "evaluateAt": "EXIT" + } +] \ No newline at end of file diff --git a/tests/debugger/test_debugger_probe_snapshot.py b/tests/debugger/test_debugger_probe_snapshot.py index 1677d5ce84..b279f2e129 100644 --- a/tests/debugger/test_debugger_probe_snapshot.py +++ b/tests/debugger/test_debugger_probe_snapshot.py @@ -10,36 +10,72 @@ @features.debugger @scenarios.debugger_method_probes_snapshot class Test_Debugger_Method_Probe_Snaphots(base._Base_Debugger_Test): - def setup_method_probe_snaphots(self): - probes = base.read_probes("probe_snapshot_method") + def setup_log_method_probe_snaphots(self): + probes = base.read_probes("probe_snapshot_log_method") self.expected_probe_ids = base.extract_probe_ids(probes) self.rc_state = rc.send_debugger_command(probes, version=1) interfaces.agent.wait_for(self.wait_for_all_probes_installed, timeout=30) self.weblog_responses = [ weblog.get("/debugger/log"), - weblog.get("/debugger/span"), - weblog.get("/debugger/span-decoration/asd/1"), ] @bug(library="python", reason="DEBUG-2708, DEBUG-2709") - def test_method_probe_snaphots(self): + def test_log_method_probe_snaphots(self): self.assert_all_states_not_error() self.assert_all_probes_are_installed() self.assert_all_weblog_responses_ok() expected_snapshots = ["log170aa-acda-4453-9111-1478a6method"] - expected_spans = ["span70aa-acda-4453-9111-1478a6method", "decor0aa-acda-4453-9111-1478a6method"] _validate_snapshots(expected_snapshots) + + def setup_span_method_probe_snaphots(self): + probes = base.read_probes("probe_snapshot_span_method") + self.expected_probe_ids = base.extract_probe_ids(probes) + self.rc_state = rc.send_debugger_command(probes, version=1) + + interfaces.agent.wait_for(self.wait_for_all_probes_installed, timeout=30) + self.weblog_responses = [ + weblog.get("/debugger/span"), + ] + + @bug(library="python", reason="DEBUG-2708, DEBUG-2709") + def test_span_method_probe_snaphots(self): + self.assert_all_states_not_error() + self.assert_all_probes_are_installed() + self.assert_all_weblog_responses_ok() + + expected_spans = ["span70aa-acda-4453-9111-1478a6method"] + + _validate_spans(expected_spans) + + def setup_span_decoration_method_probe_snaphots(self): + probes = base.read_probes("probe_snapshot_span_decoration_method") + self.expected_probe_ids = base.extract_probe_ids(probes) + self.rc_state = rc.send_debugger_command(probes, version=1) + + interfaces.agent.wait_for(self.wait_for_all_probes_installed, timeout=30) + self.weblog_responses = [ + weblog.get("/debugger/span-decoration/asd/1"), + ] + + @bug(library="python", reason="DEBUG-2708, DEBUG-2709") + def test_span_decoration_method_probe_snaphots(self): + self.assert_all_states_not_error() + self.assert_all_probes_are_installed() + self.assert_all_weblog_responses_ok() + + expected_spans = ["decor0aa-acda-4453-9111-1478a6method"] + _validate_spans(expected_spans) @features.debugger @scenarios.debugger_line_probes_snapshot class Test_Debugger_Line_Probe_Snaphots(base._Base_Debugger_Test): - def setup_line_probe_snaphots(self): - probes = base.read_probes("probe_snapshot_line") + def setup_log_line_probe_snaphots(self): + probes = base.read_probes("probe_snapshot_log_line") self.expected_probe_ids = base.extract_probe_ids(probes) self.rc_state = rc.send_debugger_command(probes, version=1) @@ -47,18 +83,35 @@ def setup_line_probe_snaphots(self): self.weblog_responses = [ weblog.get("/debugger/log"), - weblog.get("/debugger/span-decoration/asd/1"), ] - def test_line_probe_snaphots(self): + def test_log_line_probe_snaphots(self): self.assert_all_states_not_error() self.assert_all_probes_are_installed() self.assert_all_weblog_responses_ok() expected_snapshots = ["log170aa-acda-4453-9111-1478a697line"] - expected_spans = ["decor0aa-acda-4453-9111-1478a697line"] _validate_snapshots(expected_snapshots) + + def setup_span_decoration_line_probe_snaphots(self): + probes = base.read_probes("probe_snapshot_span_decoration_line") + self.expected_probe_ids = base.extract_probe_ids(probes) + self.rc_state = rc.send_debugger_command(probes, version=1) + + interfaces.agent.wait_for(self.wait_for_all_probes_installed, timeout=30) + + self.weblog_responses = [ + weblog.get("/debugger/span-decoration/asd/1"), + ] + + def test_span_decoration_line_probe_snaphots(self): + self.assert_all_states_not_error() + self.assert_all_probes_are_installed() + self.assert_all_weblog_responses_ok() + + expected_spans = ["decor0aa-acda-4453-9111-1478a697line"] + _validate_spans(expected_spans) @@ -66,7 +119,7 @@ def test_line_probe_snaphots(self): @scenarios.debugger_mix_log_probe class Test_Debugger_Mix_Log_Probe(base._Base_Debugger_Test): def setup_mix_probe(self): - probes = base.read_probes("probe_snapshot_mix_log") + probes = base.read_probes("probe_snapshot_log_mixed") self.expected_probe_ids = base.extract_probe_ids(probes) self.rc_state = rc.send_debugger_command(probes, version=1) From a31e3069dd59a1c818c2cda92100e21ab020affc Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:52:13 -0800 Subject: [PATCH 07/18] re-enable ruby container SSI tests for armd64/glibc [INPLAT-216] (#3482) Co-authored-by: roberto montero --- .gitlab-ci.yml | 6 +++--- .../ruby/provision_test-app-ruby-multicontainer.yml | 1 + .../ruby/test-app-ruby-multicontainer/docker-compose.yml | 8 ++++---- utils/virtual_machine/virtual_machine_provisioner.py | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5929e6e2cc..281dedc3bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -219,9 +219,9 @@ onboarding_ruby: - ONBOARDING_FILTER_WEBLOG: [test-app-ruby] SCENARIO: [CHAOS_INSTALLER_AUTO_INJECTION] DEFAULT_VMS: ["True", "False"] - #- ONBOARDING_FILTER_WEBLOG: [test-app-ruby-multicontainer,test-app-ruby-multialpine] - # SCENARIO: [SIMPLE_INSTALLER_AUTO_INJECTION] - # DEFAULT_VMS: ["True", "False"] + - ONBOARDING_FILTER_WEBLOG: [test-app-ruby-multicontainer] + SCENARIO: [SIMPLE_INSTALLER_AUTO_INJECTION] + DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} diff --git a/utils/build/virtual_machine/weblogs/ruby/provision_test-app-ruby-multicontainer.yml b/utils/build/virtual_machine/weblogs/ruby/provision_test-app-ruby-multicontainer.yml index 7dfa500c45..2f0525d56b 100644 --- a/utils/build/virtual_machine/weblogs/ruby/provision_test-app-ruby-multicontainer.yml +++ b/utils/build/virtual_machine/weblogs/ruby/provision_test-app-ruby-multicontainer.yml @@ -4,6 +4,7 @@ weblog: exact_os_branches: [ubuntu24] install: - os_type: linux + os_cpu: arm64 copy_files: - name: copy-multicontainer-run-script local_path: utils/build/virtual_machine/weblogs/common/create_and_run_app_multicontainer.sh diff --git a/utils/build/virtual_machine/weblogs/ruby/test-app-ruby-multicontainer/docker-compose.yml b/utils/build/virtual_machine/weblogs/ruby/test-app-ruby-multicontainer/docker-compose.yml index 4f451847ff..bd4d175210 100644 --- a/utils/build/virtual_machine/weblogs/ruby/test-app-ruby-multicontainer/docker-compose.yml +++ b/utils/build/virtual_machine/weblogs/ruby/test-app-ruby-multicontainer/docker-compose.yml @@ -20,7 +20,7 @@ services: context: . dockerfile: Dockerfile.ruby_3_2 healthcheck: - test: "true" + test: "curl -f http://localhost:18080/" ruby_3_1: env_file: "scenario_app.env" @@ -30,7 +30,7 @@ services: context: . dockerfile: Dockerfile.ruby_3_1 healthcheck: - test: "true" + test: "curl -f http://localhost:18080/" ruby_3_0: env_file: "scenario_app.env" @@ -40,7 +40,7 @@ services: context: . dockerfile: Dockerfile.ruby_3_0 healthcheck: - test: "true" + test: "curl -f http://localhost:18080/" ruby_2_7: env_file: "scenario_app.env" @@ -50,4 +50,4 @@ services: context: . dockerfile: Dockerfile.ruby_2_7 healthcheck: - test: "true" + test: "curl -f http://localhost:18080/" diff --git a/utils/virtual_machine/virtual_machine_provisioner.py b/utils/virtual_machine/virtual_machine_provisioner.py index c2a239f7b7..9b4bf963af 100644 --- a/utils/virtual_machine/virtual_machine_provisioner.py +++ b/utils/virtual_machine/virtual_machine_provisioner.py @@ -102,7 +102,8 @@ def remove_unsupported_machines( vms_to_remove.append(vm) # Ok remove the vms for vm in vms_to_remove: - required_vms.remove(vm) + if vm in required_vms: + required_vms.remove(vm) def get_provision(self, library_name, env, weblog, vm_provision_name, os_type, os_distro, os_branch, os_cpu): """ Parse the provision files (main provision file and weblog provision file) and return a Provision object""" From 05c494c761b2353bb3d6580f15fe5025606de814 Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Wed, 20 Nov 2024 06:52:38 -0800 Subject: [PATCH 08/18] [python] add test-app-python-27 app for INSTALLER_NOT_SUPPORTED_AUTO_INJECTION (#3455) Co-authored-by: brettlangdon Co-authored-by: Brett Langdon Co-authored-by: roberto montero --- .gitlab-ci.yml | 2 +- .../Dockerfile | 12 +++++ .../build.sh | 6 +++ .../django_app.py | 29 ++++++++++++ utils/_context/virtual_machines.py | 4 +- .../python/provision_test-app-python-27.yml | 46 +++++++++++++++++++ .../test-app-python-27/python_install.sh | 26 +++++++++++ .../test-app-python-27/test-app-python_run.sh | 16 +++++++ 8 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 lib-injection/build/docker/python/dd-lib-python-init-test-django-27/Dockerfile create mode 100755 lib-injection/build/docker/python/dd-lib-python-init-test-django-27/build.sh create mode 100644 lib-injection/build/docker/python/dd-lib-python-init-test-django-27/django_app.py create mode 100644 utils/build/virtual_machine/weblogs/python/provision_test-app-python-27.yml create mode 100755 utils/build/virtual_machine/weblogs/python/test-app-python-27/python_install.sh create mode 100755 utils/build/virtual_machine/weblogs/python/test-app-python-27/test-app-python_run.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 281dedc3bb..1a805e7c60 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -151,7 +151,7 @@ onboarding_python: - ONBOARDING_FILTER_WEBLOG: [test-app-python-multicontainer,test-app-python-multialpine] SCENARIO: [SIMPLE_INSTALLER_AUTO_INJECTION] DEFAULT_VMS: ["True", "False"] - - ONBOARDING_FILTER_WEBLOG: [test-app-python-unsupported-defaults] + - ONBOARDING_FILTER_WEBLOG: [test-app-python-unsupported-defaults,test-app-python-27] SCENARIO: [INSTALLER_NOT_SUPPORTED_AUTO_INJECTION] DEFAULT_VMS: ["True", "False"] script: diff --git a/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/Dockerfile b/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/Dockerfile new file mode 100644 index 0000000000..eea7cec416 --- /dev/null +++ b/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/Dockerfile @@ -0,0 +1,12 @@ +FROM python:2.7 + +ENV PYTHONUNBUFFERED 1 +ENV DJANGO_SETTINGS_MODULE django_app +WORKDIR /src +ADD . /src +RUN pip install django +EXPOSE 18080 + +# Many users run a non-root user, ensure this is supported by the injection mechanism +USER 1000 +CMD python -m django runserver 0.0.0.0:18080 diff --git a/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/build.sh b/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/build.sh new file mode 100755 index 0000000000..9340d6f6df --- /dev/null +++ b/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ -z "${BUILDX_PLATFORMS}" ] ; then + BUILDX_PLATFORMS=`docker buildx imagetools inspect --raw python:3.12 | jq -r 'reduce (.manifests[] | [ .platform.os, .platform.architecture, .platform.variant ] | join("/") | sub("\\/$"; "")) as $item (""; . + "," + $item)' | sed 's/,//'` +fi +docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file diff --git a/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/django_app.py b/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/django_app.py new file mode 100644 index 0000000000..94bc227546 --- /dev/null +++ b/lib-injection/build/docker/python/dd-lib-python-init-test-django-27/django_app.py @@ -0,0 +1,29 @@ +import os +import signal +import sys + +from django.http import HttpResponse +from django.conf.urls import url + + +def handle_sigterm(signo, sf): + sys.exit(0) + + +signal.signal(signal.SIGTERM, handle_sigterm) + + +filepath, extension = os.path.splitext(__file__) +ROOT_URLCONF = os.path.basename(filepath) +DEBUG = False +SECRET_KEY = "fdsfdasfa" +ALLOWED_HOSTS = ["*"] + + +def index(request): + return HttpResponse("test") + + +urlpatterns = [ + url("", index), +] diff --git a/utils/_context/virtual_machines.py b/utils/_context/virtual_machines.py index 5db82051bc..362aaa3c26 100644 --- a/utils/_context/virtual_machines.py +++ b/utils/_context/virtual_machines.py @@ -207,8 +207,8 @@ def set_vm_logs(self, vm_logs): def get_cache_name(self): """ Generate a unique name for the cache. - use: vm name + provision name + weblog id + hash of the cacheable installations - We geneate the hash from cacheable steps content. If we modify the step scripts + use: vm name + provision name + weblog id + hash of the cacheable installations + We geneate the hash from cacheable steps content. If we modify the step scripts the hash will change and the cache will be regenerated. If we use the AWS provider: The AWS AMI is limited to 128 characters, so we need to keep the name short """ diff --git a/utils/build/virtual_machine/weblogs/python/provision_test-app-python-27.yml b/utils/build/virtual_machine/weblogs/python/provision_test-app-python-27.yml new file mode 100644 index 0000000000..1c70a46641 --- /dev/null +++ b/utils/build/virtual_machine/weblogs/python/provision_test-app-python-27.yml @@ -0,0 +1,46 @@ +lang_variant: + name: Python_2.7.18 + excluded_os_branches: [amazon_linux2, rhel_7_amd64, redhat] + version: 2.7.18 + cache: true + install: + - os_type: linux + os_distro: deb + copy_files: + - name: copy-auto-install-script + local_path: utils/build/virtual_machine/weblogs/python/test-app-python-27/python_install.sh + remote-command: sudo apt-get -y update && sudo apt-get -y install patch && sudo sh python_install.sh deb 2.7.18 + + - os_type: linux + os_distro: rpm + copy_files: + - name: copy-auto-install-script + local_path: utils/build/virtual_machine/weblogs/python/test-app-python-27/python_install.sh + remote-command: | + if ! command -v make &> /dev/null + then + sudo yum groupinstall "Development Tools" -y + fi + sudo yum -y install patch + sudo sh python_install.sh rpm 2.7.18 + +weblog: + name: test-app-python-27 + excluded_os_branches: [amazon_linux2, rhel_7_amd64, redhat, ubuntu23] + install: + - os_type: linux + + copy_files: + - name: copy-service + local_path: utils/build/virtual_machine/weblogs/common/test-app.service + + - name: copy-service-run-script + local_path: utils/build/virtual_machine/weblogs/common/create_and_run_app_service.sh + + - name: copy-run-weblog-script + local_path: utils/build/virtual_machine/weblogs/python/test-app-python-27/test-app-python_run.sh + + - name: copy-python-app + local_path: lib-injection/build/docker/python/dd-lib-python-init-test-django-27 + + remote-command: export PATH="/home/datadog/.pyenv/bin:$PATH" && eval "$(pyenv init -)" && sudo sh test-app-python_run.sh diff --git a/utils/build/virtual_machine/weblogs/python/test-app-python-27/python_install.sh b/utils/build/virtual_machine/weblogs/python/test-app-python-27/python_install.sh new file mode 100755 index 0000000000..ac43611315 --- /dev/null +++ b/utils/build/virtual_machine/weblogs/python/test-app-python-27/python_install.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +DISTRO=$1 +PY_VERSION=$2 + +#Workaround: if python install pyenv +if [ "$DISTRO" = "deb" ]; then + #Install pyenv + packages_install="make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev git curl llvm libncursesw5-dev xz-utils tk-dev tzdata libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev" + # shellcheck disable=SC2086 + sudo apt-get install -y $packages_install || ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $packages_install + export PYENV_ROOT="/home/datadog/.pyenv" + sudo curl https://pyenv.run | bash + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init -)" + else + #Install pyenv + sudo yum install -y gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel tk-devel libffi-devel git + git clone https://github.com/pyenv/pyenv.git /home/datadog/.pyenv + export PYENV_ROOT="/home/datadog/.pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init -)" + fi + + +export PATH="/home/datadog/.pyenv/bin:$PATH" && eval "$(pyenv init -)" && pyenv install "$PY_VERSION" && pyenv global "$PY_VERSION" diff --git a/utils/build/virtual_machine/weblogs/python/test-app-python-27/test-app-python_run.sh b/utils/build/virtual_machine/weblogs/python/test-app-python-27/test-app-python_run.sh new file mode 100755 index 0000000000..9dc881ae53 --- /dev/null +++ b/utils/build/virtual_machine/weblogs/python/test-app-python-27/test-app-python_run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +echo "START python APP" + +set -e + +# shellcheck disable=SC2035 +sudo chmod -R 755 * + +sudo cp django_app.py /home/datadog/ +sudo /home/datadog/.pyenv/shims/pip install django +echo "Testing weblog with python version:" +sudo /home/datadog/.pyenv/shims/python --version +./create_and_run_app_service.sh "/home/datadog/.pyenv/shims/python -m django runserver 0.0.0.0:5985" "PYTHONPATH=/home/datadog/:$PYTHONPATH PYTHONUNBUFFERED=1 DJANGO_SETTINGS_MODULE=django_app" +echo "RUN AFTER THE SERVICE" +cat test-app.service +echo "RUN python DONE" From 555dbce3e21e72f7bd3ab53c5fa40c5ff83a896c Mon Sep 17 00:00:00 2001 From: ValentinZakharov Date: Wed, 20 Nov 2024 15:58:25 +0100 Subject: [PATCH 09/18] Enabled xpassed tests for Java (#3484) --- manifests/java.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/manifests/java.yml b/manifests/java.yml index 43dea63d71..cc225fc0f7 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -590,7 +590,9 @@ tests/: '*': v1.40.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) Test_Lfi_RC_CustomAction: missing_feature (APPSEC-54930) - Test_Lfi_Rules_Version: missing_feature + Test_Lfi_Rules_Version: + '*': v1.43.0 + spring-boot-3-native: missing_feature (GraalVM. Tracing support only) Test_Lfi_StackTrace: '*': v1.40.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) @@ -603,8 +605,11 @@ tests/: '*': v1.40.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) spring-boot-payara: missing_feature (APPSEC-54966) - test_libddwaf.py: missing_feature - test_shi.py: missing_feature (Not support in Java) + test_libddwaf.py: + Test_Libddwaf_Version: + '*': v1.40.0 + spring-boot-3-native: missing_feature (GraalVM. Tracing support only) + test_shi.py: irrelevant (Not support in Java) # SQLi was introduced in v1.38.0 (with RASP disabled by default, but was flaky) test_sqli.py: Test_Sqli_BodyJson: @@ -638,7 +643,9 @@ tests/: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) vertx3: v1.40.0 # issue in context propagation in 1.39.0 vertx4: v1.40.0 # issue in context propagation in 1.39.0 - Test_Sqli_Rules_Version: missing_feature + Test_Sqli_Rules_Version: + '*': v1.43.0 + spring-boot-3-native: missing_feature (GraalVM. Tracing support only) Test_Sqli_StackTrace: '*': v1.39.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) @@ -683,14 +690,14 @@ tests/: Test_Ssrf_Mandatory_SpanTags: '*': v1.39.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) - spring-boot-payara: missing_feature (APPSEC-54966) vertx4: missing_feature (APPSEC-55781) Test_Ssrf_Optional_SpanTags: '*': v1.39.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) - spring-boot-payara: missing_feature (APPSEC-54966) vertx4: missing_feature (APPSEC-55781) - Test_Ssrf_Rules_Version: missing_feature + Test_Ssrf_Rules_Version: + '*': v1.43.0 + spring-boot-3-native: missing_feature (GraalVM. Tracing support only) Test_Ssrf_StackTrace: '*': v1.39.0 spring-boot-3-native: missing_feature (GraalVM. Tracing support only) @@ -1618,4 +1625,3 @@ tests/: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) Test_TelemetrySCAEnvVar: missing_feature Test_TelemetryV2: v1.23.0 - From d1e67f6d53ec96e50b3d5468597792e3dc5b1120 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 20 Nov 2024 10:09:21 -0500 Subject: [PATCH 10/18] parametric: remove http_headers from otel start span creation (#3478) --- manifests/dotnet.yml | 2 - manifests/php.yml | 2 - manifests/python.yml | 2 - manifests/ruby.yml | 2 - .../test_otel_api_interoperability.py | 23 +--- tests/parametric/test_otel_span_methods.py | 122 ++---------------- tests/parametric/test_otel_span_with_w3c.py | 62 --------- .../parametric/Endpoints/ApmTestApiOtel.cs | 61 +-------- .../build/docker/golang/parametric/helpers.go | 3 - utils/build/docker/golang/parametric/otel.go | 56 +------- .../controller/OpenTelemetryController.java | 50 +------ .../trace/opentelemetry/dto/SpanLink.java | 3 +- .../opentelemetry/dto/StartSpanArgs.java | 1 - .../build/docker/nodejs/parametric/server.js | 23 +--- utils/build/docker/php/parametric/server.php | 55 +------- .../parametric/apm_test_client/server.py | 42 +----- utils/build/docker/ruby/parametric/server.rb | 29 ++--- utils/parametric/_library_client.py | 24 +--- 18 files changed, 50 insertions(+), 512 deletions(-) delete mode 100644 tests/parametric/test_otel_span_with_w3c.py diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 13e8d85c37..c9c07d622b 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -387,8 +387,6 @@ tests/: Test_Otel_Env_Vars: v2.53.0 test_otel_span_with_baggage.py: Test_Otel_Span_With_Baggage: missing_feature - test_otel_span_with_w3c.py: - Test_Otel_Span_With_W3c: v2.42.0 test_otel_tracer.py: Test_Otel_Tracer: v2.8.0 test_parametric_endpoints.py: diff --git a/manifests/php.yml b/manifests/php.yml index c20c886d37..7e3b9670c3 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -325,8 +325,6 @@ tests/: Test_Otel_Span_Methods: v0.94.0 test_otel_span_with_baggage.py: Test_Otel_Span_With_Baggage: missing_feature - test_otel_span_with_w3c.py: - Test_Otel_Span_With_W3c: v0.94.0 test_otel_tracer.py: Test_Otel_Tracer: v0.94.0 test_parametric_endpoints.py: diff --git a/manifests/python.yml b/manifests/python.yml index 864f75dc8e..473e64f1fd 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -771,8 +771,6 @@ tests/: Test_Otel_Span_Methods: v2.8.0 test_otel_span_with_baggage.py: Test_Otel_Span_With_Baggage: missing_feature - test_otel_span_with_w3c.py: - Test_Otel_Span_With_W3c: v2.8.0 test_otel_tracer.py: Test_Otel_Tracer: v2.8.0 test_parametric_endpoints.py: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 3a89825a9a..26b7e80899 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -393,8 +393,6 @@ tests/: Test_Otel_Span_Methods: v1.17.0 test_otel_span_with_baggage.py: Test_Otel_Span_With_Baggage: missing_feature - test_otel_span_with_w3c.py: - Test_Otel_Span_With_W3c: v1.17.0 test_parametric_endpoints.py: Test_Parametric_DDSpan_Set_Resource: missing_feature (set_resource endpoint is not supported) Test_Parametric_DDTrace_Baggage: missing_feature (baggage is not supported) diff --git a/tests/parametric/test_otel_api_interoperability.py b/tests/parametric/test_otel_api_interoperability.py index 75d65614f6..8211e89f1a 100644 --- a/tests/parametric/test_otel_api_interoperability.py +++ b/tests/parametric/test_otel_api_interoperability.py @@ -227,21 +227,17 @@ def test_span_links_add(self, test_agent, test_library): - Test that links can be added with the Datadog API on a span created with the OTel API """ with test_library: - with test_library.otel_start_span("otel.span") as otel_span: - current_span = test_library.current_span() + with test_library.start_span("dd_root") as dd_span: + pass + with test_library.otel_start_span("otel_root") as otel_span: + current_span = test_library.current_span() current_span.add_link( - parent_id=0, - attributes=TEST_ATTRIBUTES, - http_headers=[ - ("traceparent", f"00-{TEST_TRACE_ID}-{TEST_SPAN_ID}-01"), - ("tracestate", TEST_TRACESTATE), - ], + parent_id=dd_span.span_id, attributes=TEST_ATTRIBUTES, ) - otel_span.end_span() - traces = test_agent.wait_for_num_traces(1, sort_by_start=False) + traces = test_agent.wait_for_num_traces(2, sort_by_start=False) trace = find_trace(traces, otel_span.trace_id) assert len(trace) == 1 @@ -249,13 +245,6 @@ def test_span_links_add(self, test_agent, test_library): span_links = retrieve_span_links(root) assert len(span_links) == 1 - link = span_links[0] - assert link["trace_id"] == TEST_TRACE_ID_LOW - assert link["trace_id_high"] == TEST_TRACE_ID_HIGH - assert link["span_id"] == TEST_SPAN_ID_INT - assert "t.dm:-0" in link["tracestate"] - assert link["attributes"]["arg1"] == "val1" - def test_concurrent_traces_in_order(self, test_agent, test_library): """ - Basic concurrent traces and spans diff --git a/tests/parametric/test_otel_span_methods.py b/tests/parametric/test_otel_span_methods.py index 7792500a36..561982c661 100644 --- a/tests/parametric/test_otel_span_methods.py +++ b/tests/parametric/test_otel_span_methods.py @@ -512,107 +512,6 @@ def test_otel_span_started_with_link_from_another_span(self, test_agent, test_li root_tid = root["meta"].get("_dd.p.tid", "0") assert link.get("trace_id_high") == int(root_tid, 16) - @missing_feature(context.library < "dotnet@2.53.0", reason="Will be released in 2.53.0") - @missing_feature(context.library < "java@1.26.0", reason="Implemented in 1.26.0") - @missing_feature(context.library < "nodejs@5.3.0", reason="Implemented in 3.48.0, 4.27.0, and 5.3.0") - @missing_feature(context.library < "golang@1.61.0", reason="Implemented in 1.61.0") - @missing_feature(context.library < "ruby@2.0.0", reason="Not implemented") - @missing_feature(context.library == "php", reason="Implemented in 0.97.0 but link.flags are not natively supported") - def test_otel_span_started_with_link_from_datadog_headers(self, test_agent, test_library): - """Properly inject datadog distributed tracing information into span links. - """ - with test_library: - with test_library.otel_start_span( - "root", - links=[ - Link( - http_headers=[ - ["x-datadog-trace-id", "1234567890"], - ["x-datadog-parent-id", "9876543210"], - ["x-datadog-sampling-priority", "2"], - ["x-datadog-origin", "synthetics"], - ["x-datadog-tags", "_dd.p.dm=-4,_dd.p.tid=0000000000000010"], - ], - attributes={"foo": "bar"}, - ) - ], - ) as span: - span.end_span() - - traces = test_agent.wait_for_num_traces(1) - trace = find_trace(traces, span.trace_id) - span = find_span(trace, span.span_id) - span_links = retrieve_span_links(span) - assert span_links is not None - assert len(span_links) == 1 - - link = span_links[0] - assert link.get("span_id") == 9876543210 - assert link.get("trace_id") == 1234567890 - assert link.get("trace_id_high") == 16 - - # Tracestate is not required, but if it is present, it must be valid - if link.get("tracestate"): - tracestateArr = link["tracestate"].split(",") - assert len(tracestateArr) == 1 and tracestateArr[0].startswith("dd=") - tracestateDD = tracestateArr[0][3:].split(";") - assert "o:synthetics" in tracestateDD - assert "s:2" in tracestateDD - assert "t.dm:-4" in tracestateDD - # Sampled flag should be set to match the existing tracestate - assert link.get("flags") == 1 | TRACECONTEXT_FLAGS_SET - - @missing_feature(context.library < "dotnet@2.53.0", reason="Will be released in 2.53.0") - @missing_feature(context.library < "java@1.28.0", reason="Implemented in 1.28.0") - @missing_feature(context.library < "nodejs@5.3.0", reason="Implemented in 3.48.0, 4.27.0, and 5.3.0") - @missing_feature(context.library < "golang@1.61.0", reason="Implemented in 1.61.0") - @bug(context.library < "ruby@2.3.1-dev", reason="APMRP-360") - @missing_feature(context.library == "php", reason="Implemented in 0.97.0 but link.flags are not natively supported") - def test_otel_span_started_with_link_from_w3c_headers(self, test_agent, test_library): - """Properly inject w3c distributed tracing information into span links. - This mostly tests that the injected tracestate and flags are accurate. - """ - with test_library: - with test_library.otel_start_span( - "root", - links=[ - Link( - http_headers=[ - ["traceparent", "00-12345678901234567890123456789012-1234567890123456-01"], - ["tracestate", "foo=1,dd=t.dm:-4;s:2,bar=baz"], - ] - ) - ], - ) as span: - span.end_span() - - traces = test_agent.wait_for_num_traces(1) - trace = find_trace(traces, span.trace_id) - span = find_span(trace, span.span_id) - span_links = retrieve_span_links(span) - assert span_links is not None - assert len(span_links) == 1 - - link = span_links[0] - assert link.get("span_id") == 1311768467284833366 - assert link.get("trace_id") == 8687463697196027922 - assert link.get("trace_id_high") == 1311768467284833366 - - assert link.get("tracestate") is not None - tracestateArr = link["tracestate"].split(",") - dd_member = next(iter([x for x in tracestateArr if x.startswith("dd=")]), None) - foo_member = next(iter([x for x in tracestateArr if x.startswith("foo=")]), None) - bar_member = next(iter([x for x in tracestateArr if x.startswith("bar=")]), None) - # ruby removes the dd member from the tracestate while python does not - if dd_member: - assert "s:2" in dd_member - assert "t.dm:-4" in dd_member - assert foo_member == "foo=1" - assert bar_member == "bar=baz" - - assert (link.get("flags") == 1 | TRACECONTEXT_FLAGS_SET) or (link.get("flags") == TRACECONTEXT_FLAGS_SET) - assert link.get("attributes") is None or len(link.get("attributes")) == 0 - @missing_feature(context.library < "dotnet@2.53.0", reason="Will be released in 2.53.0") @missing_feature(context.library < "java@1.26.0", reason="Implemented in 1.26.0") @missing_feature(context.library == "golang", reason="Not implemented") @@ -623,26 +522,23 @@ def test_otel_span_link_attribute_handling(self, test_agent, test_library): """Test that span links implementations correctly handle attributes according to spec. """ with test_library: + with test_library.otel_start_span("span1") as s1: + s1.end_span() + with test_library.otel_start_span( "root", links=[ Link( - http_headers=[ - ["x-datadog-trace-id", "1234567890"], - ["x-datadog-parent-id", "9876543210"], - ["x-datadog-sampling-priority", "2"], - ["x-datadog-origin", "synthetics"], - ["x-datadog-tags", "_dd.p.dm=-4,_dd.p.tid=0000000000000010"], - ], + parent_id=s1.span_id, attributes={"foo": "bar", "array": ["a", "b", "c"], "bools": [True, False], "nested": [1, 2]}, ) ], - ) as span: - span.end_span() + ) as s2: + s2.end_span() - traces = test_agent.wait_for_num_traces(1) - trace = find_trace(traces, span.trace_id) - span = find_span(trace, span.span_id) + traces = test_agent.wait_for_num_traces(2) + trace = find_trace(traces, s2.trace_id) + span = find_span(trace, s2.span_id) span_links = retrieve_span_links(span) assert span_links is not None assert len(span_links) == 1 diff --git a/tests/parametric/test_otel_span_with_w3c.py b/tests/parametric/test_otel_span_with_w3c.py deleted file mode 100644 index b81d9adcbe..0000000000 --- a/tests/parametric/test_otel_span_with_w3c.py +++ /dev/null @@ -1,62 +0,0 @@ -import time - -import pytest - -from utils.dd_constants import SpanKind -from utils.parametric.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN -from utils.parametric.spec.trace import find_only_span -from utils import missing_feature, irrelevant, context, scenarios, features - -# this global mark applies to all tests in this file. -# DD_TRACE_OTEL_ENABLED=true is required in some tracers (.NET, Python?) -# CORECLR_ENABLE_PROFILING=1 is required in .NET to enable auto-instrumentation -pytestmark = pytest.mark.parametrize( - "library_env", [{"DD_TRACE_OTEL_ENABLED": "true", "CORECLR_ENABLE_PROFILING": "1"}], -) - - -@scenarios.parametric -@features.open_tracing_api -class Test_Otel_Span_With_W3c: - @irrelevant(context.library == "cpp", reason="library does not implement OpenTelemetry") - @missing_feature(context.library == "python", reason="Not implemented") - @missing_feature(context.library <= "java@1.23.0", reason="OTel resource naming implemented in 1.24.0") - @missing_feature(context.library == "nodejs", reason="Not implemented") - def test_otel_start_span_with_w3c(self, test_agent, test_library): - """ - - Start/end a span with start and end options - """ - with test_library: - duration_us = int(2 * 1_000_000) - start_time = int(time.time()) - with test_library.otel_start_span( - "operation", - span_kind=SpanKind.PRODUCER, - timestamp=start_time, - attributes={"start_attr_key": "start_attr_val"}, - ) as parent: - parent.end_span(timestamp=start_time + duration_us) - duration_ns = int(duration_us * 1_000) # OTEL durations are microseconds, must convert to ns for dd - - root_span = find_only_span(test_agent.wait_for_num_traces(1)) - assert root_span["name"] == "producer" - assert root_span["resource"] == "operation" - assert root_span["meta"]["start_attr_key"] == "start_attr_val" - assert root_span["duration"] == duration_ns - - @irrelevant(context.library == "cpp", reason="library does not implement OpenTelemetry") - def test_otel_span_with_w3c_headers(self, test_agent, test_library): - with test_library: - with test_library.otel_start_span( - name="name", http_headers=[["traceparent", "00-00000000000000001111111111111111-2222222222222222-01"]], - ) as span: - context = span.span_context() - assert context.get("trace_flags") == "01" - assert context.get("trace_id") == "00000000000000001111111111111111" - span.end_span() - - span = find_only_span(test_agent.wait_for_num_traces(1)) - assert span.get("trace_id") == 1229782938247303441 - assert span.get("parent_id") == 2459565876494606882 - assert span["metrics"].get(SAMPLING_PRIORITY_KEY) == 1 - assert span["meta"].get(ORIGIN) is None diff --git a/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs b/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs index c7b3a033bc..4ffaf6ab90 100644 --- a/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs +++ b/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs @@ -47,34 +47,6 @@ private static async Task OtelStartSpan(HttpRequest request) } } - // try extracting parent context from headers (remote parent) - if (requestBodyObject.TryGetValue("http_headers", out var headersList)) - { - var manualExtractedContext = _spanContextExtractor.Extract( - ((Newtonsoft.Json.Linq.JArray)headersList).ToObject(), - getter: GetHeaderValues!); - - _logger?.LogInformation("Extracted SpanContext: {ExtractedContext}", manualExtractedContext); - - if (manualExtractedContext is not null) - { - // This implementation is .NET v3 specific, and assumes that the span returned by StartActive is a DuckType - var extractedContext = manualExtractedContext.GetType() - .GetProperty("Instance", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public) - ?.GetValue(manualExtractedContext); - var parentTraceId = ActivityTraceId.CreateFromString(RawTraceId.GetValue(extractedContext) as string); - var parentSpanId = ActivitySpanId.CreateFromString(RawSpanId.GetValue(extractedContext) as string); - var flags = (SamplingPriority.GetValue(extractedContext) as int?) > 0 ? ActivityTraceFlags.Recorded : ActivityTraceFlags.None; - - remoteParentContext = new ActivityContext( - parentTraceId, - parentSpanId, - flags, - AdditionalW3CTraceState.GetValue(extractedContext) as string, - isRemote: true); - } - } - // sanity check that we didn't receive both a local and remote parent if (localParentContext != null && remoteParentContext != null) { @@ -132,38 +104,7 @@ private static async Task OtelStartSpan(HttpRequest request) tags = ToActivityTagsCollection(((Newtonsoft.Json.Linq.JObject?)spanLink["attributes"])?.ToObject>()); } - ActivityContext contextToLink = new ActivityContext(); - - if (parentSpanLink > 0) - { - contextToLink = FindActivity(parentSpanLink).Context; - } - else - { - var httpHeadersToken = (JArray)spanLink["http_headers"]!; - - var manualExtractedContext = _spanContextExtractor.Extract( - httpHeadersToken.ToObject(), - getter: GetHeaderValues!); - - // This implementation is .NET v3 specific, and assumes that the span returned by StartActive is a DuckType - var extractedContext = manualExtractedContext?.GetType() - .GetProperty("Instance", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public) - ?.GetValue(manualExtractedContext); - var parentTraceId = ActivityTraceId.CreateFromString(RawTraceId.GetValue(extractedContext) as string); - var parentSpanId = ActivitySpanId.CreateFromString(RawSpanId.GetValue(extractedContext) as string); - var flags = (SamplingPriority.GetValue(extractedContext) as int?) > 0 ? ActivityTraceFlags.Recorded : ActivityTraceFlags.None; - var datadogHeadersTracestate = W3CTraceContextCreateTraceStateHeader.Invoke(null, new object[] { extractedContext! }); - var tracestate = (string?)httpHeadersToken[1][0] == "tracestate" ? (string?)httpHeadersToken[1][1] : datadogHeadersTracestate; - - contextToLink = new ActivityContext( - parentTraceId, - parentSpanId, - flags, - (string?)tracestate, - isRemote: true); - } - + ActivityContext contextToLink = FindActivity(parentSpanLink).Context; linksList.Add(new ActivityLink(contextToLink, tags)); } } diff --git a/utils/build/docker/golang/parametric/helpers.go b/utils/build/docker/golang/parametric/helpers.go index 05f110b3d7..fbcfc0884e 100644 --- a/utils/build/docker/golang/parametric/helpers.go +++ b/utils/build/docker/golang/parametric/helpers.go @@ -19,7 +19,6 @@ type StartSpanArgs struct { Resource string `json:"resource,omitempty"` Type string `json:"type,omitempty"` Origin string `json:"origin,omitempty"` - HttpHeaders []Tuple `json:"http_headers,omitempty"` SpanTags []Tuple `json:"span_tags,omitempty"` SpanLinks []SpanLink `json:"span_links,omitempty"` } @@ -28,7 +27,6 @@ type Tuple []string type SpanLink struct { ParentId uint64 `json:"parent_id"` - HttpHeaders []Tuple `json:"http_headers"` Attributes AttributeKeyVals `json:"attributes,omitempty"` } @@ -98,7 +96,6 @@ type OtelStartSpanArgs struct { Type string `json:"type"` Timestamp int64 `json:"timestamp"` SpanLinks []SpanLink `json:"links"` - HttpHeaders []Tuple `json:"http_headers"` Attributes AttributeKeyVals `json:"attributes"` } diff --git a/utils/build/docker/golang/parametric/otel.go b/utils/build/docker/golang/parametric/otel.go index 6fed45fa9a..a87f714e37 100644 --- a/utils/build/docker/golang/parametric/otel.go +++ b/utils/build/docker/golang/parametric/otel.go @@ -12,7 +12,6 @@ import ( "go.opentelemetry.io/otel/codes" otel_trace "go.opentelemetry.io/otel/trace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" ddotel "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentelemetry" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) @@ -59,60 +58,13 @@ func (s *apmClientServer) OtelStartSpan(args OtelStartSpanArgs) (OtelStartSpanRe if a := args.Attributes; len(a) > 0 { otelOpts = append(otelOpts, otel_trace.WithAttributes(a.ConvertToAttributes()...)) } - if h := args.HttpHeaders; len(h) > 0 { - headers := map[string]string{} - for _, headerTuple := range h { - k := headerTuple.Key() - v := headerTuple.Value() - if k != "" && v != "" { - headers[k] = v - } - } - sctx, err := tracer.NewPropagator(nil).Extract(tracer.TextMapCarrier(headers)) - if err != nil { - fmt.Println("failed to extract span context from headers:", err, args.HttpHeaders) - } else { - ddOpts = append(ddOpts, tracer.ChildOf(sctx)) - } - } if links := args.SpanLinks; links != nil { for _, link := range links { - if p := link.ParentId; p != 0 { - if _, ok := s.otelSpans[p]; ok { - otelOpts = append(otelOpts, otel_trace.WithLinks(otel_trace.Link{SpanContext: s.otelSpans[p].span.SpanContext(), Attributes: link.Attributes.ConvertToAttributesStringified()})) - } - } else if h := link.HttpHeaders; h != nil { - headers := map[string]string{} - for _, headerTuple := range h { - k := headerTuple.Key() - v := headerTuple.Value() - if k != "" && v != "" { - headers[k] = v - } - } - extractedContext, _ := tracer.NewPropagator(nil).Extract(tracer.TextMapCarrier(headers)) - state, _ := otel_trace.ParseTraceState(headers["tracestate"]) - - var traceID otel_trace.TraceID - var spanID otel_trace.SpanID - if w3cCtx, ok := extractedContext.(ddtrace.SpanContextW3C); ok { - traceID = w3cCtx.TraceID128Bytes() - } else { - fmt.Printf("Non-W3C context found in span, unable to get full 128 bit trace id") - uint64ToByte(extractedContext.TraceID(), traceID[:]) - } - uint64ToByte(extractedContext.SpanID(), spanID[:]) - config := otel_trace.SpanContextConfig{ - TraceID: traceID, - SpanID: spanID, - TraceState: state, - } - var newCtx = otel_trace.NewSpanContext(config) - otelOpts = append(otelOpts, otel_trace.WithLinks(otel_trace.Link{ - SpanContext: newCtx, - Attributes: link.Attributes.ConvertToAttributesStringified(), - })) + if pSpan, ok := s.otelSpans[link.ParentId]; ok { + otelOpts = append(otelOpts, otel_trace.WithLinks(otel_trace.Link{SpanContext: pSpan.span.SpanContext(), Attributes: link.Attributes.ConvertToAttributesStringified()})) + } else { + return OtelStartSpanReturn{}, fmt.Errorf("OtelStartSpan call failed. Failed to generate a link to span with id=%d", link.ParentId) } } } diff --git a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/controller/OpenTelemetryController.java b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/controller/OpenTelemetryController.java index 23390586c8..a69a0c1369 100644 --- a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/controller/OpenTelemetryController.java +++ b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/controller/OpenTelemetryController.java @@ -171,15 +171,6 @@ public StartSpanResult startSpan(@RequestBody StartSpanArgs args) { builder.setParent(contextWithParentSpan); } } - // Check HTTP headers to extract propagated context from - if (args.httpHeaders() != null && !args.httpHeaders().isEmpty()) { - Context extractedContext = this.propagator.extract( - Context.root(), - args.httpHeaders(), - HeadersTextMapGetter.INSTANCE - ); - builder.setParent(extractedContext); - } // Add other span information builder.setSpanKind(parseSpanKindNumber(args.spanKind())); if (args.timestamp() > 0) { @@ -187,22 +178,12 @@ public StartSpanResult startSpan(@RequestBody StartSpanArgs args) { } if (args.links() != null && !args.links().isEmpty()) { for (SpanLink spanLink : args.links()) { - SpanContext spanContext = null; LOGGER.debug("Span link: {}", spanLink); - if (spanLink.parentId() > 0) { - Span span = getSpan(spanLink.parentId()); - if (span == null) { - return StartSpanResult.error(); - } - spanContext = span.getSpanContext(); - } else if (spanLink.httpHeaders() != null && !spanLink.httpHeaders().isEmpty()) { - Context extractedContext = this.propagator.extract( - Context.root(), - spanLink.httpHeaders(), - HeadersTextMapGetter.INSTANCE - ); - spanContext = Span.fromContext(extractedContext).getSpanContext(); + Span span = getSpan(spanLink.parentId()); + if (span == null) { + return StartSpanResult.error(); } + SpanContext spanContext = span.getSpanContext(); if (spanContext != null && spanContext.isValid()) { LOGGER.debug("Adding links from context {}", spanContext); builder.addLink(spanContext, parseAttributes(spanLink.attributes())); @@ -342,27 +323,4 @@ private Span getSpan(long spanId) { } return span; } - - private static class HeadersTextMapGetter implements TextMapGetter> { - private static final HeadersTextMapGetter INSTANCE = new HeadersTextMapGetter(); - - @Override - public Iterable keys(List headers) { - return headers.stream() - .map(KeyValue::key) - .toList(); - } - - @Override - public String get(List headers, String key) { - if (headers == null || headers.isEmpty()) { - return null; - } - return headers.stream() - .filter(kv -> Objects.equals(key, kv.key())) - .map(KeyValue::value) - .findFirst() - .orElse(null); - } - } } diff --git a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/SpanLink.java b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/SpanLink.java index 501e959ea0..281434fcf2 100644 --- a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/SpanLink.java +++ b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/SpanLink.java @@ -8,6 +8,5 @@ public record SpanLink( @JsonProperty("parent_id") long parentId, - Map attributes, - @JsonProperty("http_headers") @JsonDeserialize(using = KeyValueListDeserializer.class) List httpHeaders) { + Map attributes) { } diff --git a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/StartSpanArgs.java b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/StartSpanArgs.java index fbeb0282b5..64696cf837 100644 --- a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/StartSpanArgs.java +++ b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/opentelemetry/dto/StartSpanArgs.java @@ -11,7 +11,6 @@ public record StartSpanArgs( String name, @JsonProperty("span_kind") int spanKind, long timestamp, - @JsonProperty("http_headers") @JsonDeserialize(using = KeyValueListDeserializer.class) List httpHeaders, List links, Map attributes) { } diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index c461de5d44..4af103152a 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -159,14 +159,7 @@ app.post('/trace/otel/start_span', (req, res) => { const makeSpan = (parentContext) => { const links = (request.links || []).map(link => { - let spanContext; - if (link.parent_id && link.parent_id !== 0) { - spanContext = otelSpans[link.parent_id].spanContext(); - } else { - const linkHeaders = Object.fromEntries(link.http_headers.map(([k, v]) => [k.toLowerCase(), v])); - const extractedContext = tracer.extract('http_headers', linkHeaders) - spanContext = new OtelSpanContext(extractedContext) - } + let spanContext = otelSpans[link.parent_id].spanContext() return {context: spanContext, attributes: link.attributes} }); @@ -189,20 +182,6 @@ app.post('/trace/otel/start_span', (req, res) => { const parentContext = trace.setSpan(ROOT_CONTEXT, parentSpan) return makeSpan(parentContext) } - if (request.http_headers) { - const http_headers = request.http_headers || [] - // Node.js HTTP headers are automatically lower-cased, simulate that here. - const convertedHeaders = {} - for (const [ key, value ] of http_headers) { - convertedHeaders[key.toLowerCase()] = value - } - const extracted = tracer.extract('http_headers', convertedHeaders) - if (extracted) { - const parentSpan = trace.wrapSpanContext(new OtelSpanContext(extracted)) - const parentContext = trace.setSpan(ROOT_CONTEXT, parentSpan) - return makeSpan(parentContext) - } - } makeSpan() }); diff --git a/utils/build/docker/php/parametric/server.php b/utils/build/docker/php/parametric/server.php index b016b3dc3d..068f184f4c 100644 --- a/utils/build/docker/php/parametric/server.php +++ b/utils/build/docker/php/parametric/server.php @@ -215,23 +215,8 @@ function remappedSpanKind($spanKind) { $router->addRoute('POST', '/trace/span/add_link', new ClosureRequestHandler(function (Request $req) use (&$spans, &$closed_spans) { $span = $spans[arg($req, 'span_id')]; $parent_id = arg($req, 'parent_id'); - $httpHeaders = arg($req, 'http_headers'); - if (isset($spans[$parent_id]) || isset($closed_spans[$parent_id])) { - $link = ($spans[$parent_id] ?? $closed_spans[$parent_id])->getLink(); - $link->attributes += arg($req, "attributes") ?? []; - } elseif ($httpHeaders) { - $httpHeaders = array_merge(...array_map(fn($h) => [strtolower($h[0]) => $h[1]], $httpHeaders)); - $callback = function ($headername) use ($httpHeaders) { - return $httpHeaders[$headername] ?? null; - }; - $link = \DDTrace\SpanLink::fromHeaders($callback); - $link->attributes += arg($req, "attributes") ?? []; - } else { - $link = new \DDTrace\SpanLink(); - $link->spanId = arg($req, 'parent_id'); - $link->attributes = arg($req, 'attributes') ?? []; - } - + $link = ($spans[$parent_id] ?? $closed_spans[$parent_id])->getLink(); + $link->attributes = ($link->attributes ?? []) + (arg($req, "attributes") ?? []); $span->links[] = $link; return jsonResponse([]); })); @@ -286,7 +271,6 @@ function remappedSpanKind($spanKind) { $timestamp = arg($req, 'timestamp'); $spanKind = arg($req, 'span_kind'); $parentId = arg($req, 'parent_id'); - $httpHeaders = arg($req, 'http_headers'); $attributes = arg($req, 'attributes'); $tracer = (new TracerProvider())->getTracer('OpenTelemetry.PHPTestTracer'); @@ -324,39 +308,10 @@ function remappedSpanKind($spanKind) { if ($span_links = arg($req, 'links')) { foreach ($span_links as $span_link) { $span_link_attributes = isset($span_link["attributes"]) ? $span_link["attributes"] : []; - if ($span_link_parent_id = $span_link["parent_id"]) { - $span_context = $otelSpans[$span_link_parent_id]->getContext(); - $spanBuilder->addLink($span_context, $span_link_attributes); - } else if ($span_link_http_headers = $span_link["http_headers"]) { - $carrier = []; - foreach ($span_link_http_headers as $span_link_http_header) { - $carrier[$span_link_http_header[0]] = $span_link_http_header[1]; - } - - $callback = function ($headername) use ($carrier) { - return $carrier[$headername] ?? null; - }; - $headers_link = \DDTrace\SpanLink::fromHeaders($callback); - - $linkSpanContext = \OpenTelemetry\API\Trace\SpanContext::create( - $headers_link->traceId, - $headers_link->spanId, - \OpenTelemetry\API\Trace\TraceFlags::DEFAULT, // trace flags are not currently embedded into the native span link - new \OpenTelemetry\API\Trace\TraceState($headers_link->traceState ?? null), - ); - - $spanBuilder->addLink($linkSpanContext, $span_link_attributes); - } - } - } - - if ($httpHeaders) { - $carrier = []; - foreach ($httpHeaders as $headers) { - $carrier[$headers[0]] = $headers[1]; + $span_link_parent_id = $span_link["parent_id"]; + $span_context = $otelSpans[$span_link_parent_id]->getContext(); + $spanBuilder->addLink($span_context, $span_link_attributes); } - $remoteContext = TraceContextPropagator::getInstance()->extract($carrier); - $spanBuilder->setParent($remoteContext); } if ($attributes) { diff --git a/utils/build/docker/python/parametric/apm_test_client/server.py b/utils/build/docker/python/parametric/apm_test_client/server.py index 69ad62a139..a305689d0d 100644 --- a/utils/build/docker/python/parametric/apm_test_client/server.py +++ b/utils/build/docker/python/parametric/apm_test_client/server.py @@ -401,7 +401,6 @@ class OtelStartSpanArgs(BaseModel): type: str = "" links: List[Dict] = [] timestamp: int - http_headers: List[Tuple[str, str]] attributes: dict @@ -414,46 +413,11 @@ class OtelStartSpanReturn(BaseModel): def otel_start_span(args: OtelStartSpanArgs): otel_tracer = opentelemetry.trace.get_tracer(__name__) - if args.parent_id: - parent_span = otel_spans[args.parent_id] - elif args.http_headers: - headers = {k: v for k, v in args.http_headers} - ddcontext = HTTPPropagator.extract(headers) - parent_span = OtelNonRecordingSpan( - OtelSpanContext( - ddcontext.trace_id, - ddcontext.span_id, - True, - ( - TraceFlags.SAMPLED - if ddcontext.sampling_priority and ddcontext.sampling_priority > 0 - else TraceFlags.DEFAULT - ), - TraceState.from_header([ddcontext._tracestate]), - ) - ) - else: - parent_span = None - + parent_span = otel_spans.get(args.parent_id) links = [] for link in args.links: - parent_id = link.get("parent_id", 0) - if parent_id > 0: - span_context = otel_spans[parent_id].get_span_context() - else: - headers = {k: v for k, v in link["http_headers"]} - ddcontext = HTTPPropagator.extract(headers) - span_context = OtelSpanContext( - ddcontext.trace_id, - ddcontext.span_id, - True, - ( - TraceFlags.SAMPLED - if ddcontext.sampling_priority and ddcontext.sampling_priority > 0 - else TraceFlags.DEFAULT - ), - TraceState.from_header([ddcontext._tracestate]), - ) + parent_id = link["parent_id"] + span_context = otel_spans[parent_id].get_span_context() links.append(opentelemetry.trace.Link(span_context, link.get("attributes"))) # parametric tests expect span kind to be 0 for internal, 1 for server, 2 for client, .... diff --git a/utils/build/docker/ruby/parametric/server.rb b/utils/build/docker/ruby/parametric/server.rb index de54fb342e..eb75b15566 100644 --- a/utils/build/docker/ruby/parametric/server.rb +++ b/utils/build/docker/ruby/parametric/server.rb @@ -266,7 +266,7 @@ def to_json(*_args) end class OtelStartSpanArgs - attr_accessor :name, :parent_id, :span_kind, :service, :resource, :type, :links, :timestamp, :http_headers, + attr_accessor :name, :parent_id, :span_kind, :service, :resource, :type, :links, :timestamp, :attributes def initialize(params) @@ -278,7 +278,6 @@ def initialize(params) @type = params['type'] @links = params['links'] @timestamp = params['timestamp'] - @http_headers = params['http_headers'] @attributes = params['attributes'] end end @@ -509,18 +508,15 @@ def get_digest(span_id) end def parse_otel_link(link) - link_context = if !link['http_headers'].nil? && !link['http_headers'].size.nil? - digest = extract_http_headers(link['http_headers']) - digest_to_spancontext(digest) - elsif OTEL_SPANS.key?(link['parent_id']) - OTEL_SPANS[link['parent_id']].context - else - raise "Span id in #{link} not found in span list: #{OTEL_SPANS}" - end - OpenTelemetry::Trace::Link.new( - link_context, - link['attributes'] - ) + if OTEL_SPANS.key?(link['parent_id']) + link_context = OTEL_SPANS[link['parent_id']].context + OpenTelemetry::Trace::Link.new( + link_context, + link['attributes'] + ) + else + raise "Parent id in #{link} not found in span list: #{OTEL_SPANS}" + end end def digest_to_spancontext(digest) @@ -724,10 +720,7 @@ def handle_trace_otel_start_span(req, res) js = JSON.parse(req.body.read) args = OtelStartSpanArgs.new(js) - headers = args.http_headers.to_h - if !headers.empty? - parent_context = OpenTelemetry.propagation.extract(headers) - elsif args.parent_id != 0 + if args.parent_id != 0 parent_span = OTEL_SPANS[args.parent_id] parent_context = OpenTelemetry::Trace.context_with_span(parent_span) end diff --git a/utils/parametric/_library_client.py b/utils/parametric/_library_client.py index 5fbdff47fe..b8d9df4f8b 100644 --- a/utils/parametric/_library_client.py +++ b/utils/parametric/_library_client.py @@ -32,9 +32,8 @@ class SpanResponse(TypedDict): class Link(TypedDict): - parent_id: int # 0 to extract from headers + parent_id: int attributes: dict - http_headers: List[Tuple[str, str]] class APMLibraryClient: @@ -188,19 +187,10 @@ def span_set_error(self, span_id: int, typestr: str, message: str, stack: str) - json={"span_id": span_id, "type": typestr, "message": message, "stack": stack}, ) - def span_add_link( - self, span_id: int, parent_id: int, attributes: dict = None, http_headers: List[Tuple[str, str]] = None - ): - # Avoid using http_headers when creating a span link in the parametric apps - # Alternative endpoints will be provided to set these values. This will be documented in a future PR. + def span_add_link(self, span_id: int, parent_id: int, attributes: dict = None): self._session.post( self._url("/trace/span/add_link"), - json={ - "span_id": span_id, - "parent_id": parent_id, - "attributes": attributes or {}, - "http_headers": http_headers or [], - }, + json={"span_id": span_id, "parent_id": parent_id, "attributes": attributes or {},}, ) def span_get_baggage(self, span_id: int, key: str) -> str: @@ -236,7 +226,6 @@ def otel_trace_start_span( span_kind: SpanKind, parent_id: int, links: List[Link], - http_headers: List[Tuple[str, str]], attributes: dict = None, ) -> StartSpanResponse: resp = self._session.post( @@ -247,7 +236,6 @@ def otel_trace_start_span( "span_kind": span_kind.value, "parent_id": parent_id, "links": links, - "http_headers": http_headers, "attributes": attributes or {}, }, ).json() @@ -369,8 +357,8 @@ def remove_all_baggage(self): def set_error(self, typestr: str = "", message: str = "", stack: str = ""): self._client.span_set_error(self.span_id, typestr, message, stack) - def add_link(self, parent_id: int, attributes: dict = None, http_headers: List[Tuple[str, str]] = None): - self._client.span_add_link(self.span_id, parent_id, attributes, http_headers) + def add_link(self, parent_id: int, attributes: dict = None): + self._client.span_add_link(self.span_id, parent_id, attributes) def finish(self): self._client.finish_span(self.span_id) @@ -464,7 +452,6 @@ def otel_start_span( parent_id: int = 0, links: Optional[List[Link]] = None, attributes: dict = None, - http_headers: Optional[List[Tuple[str, str]]] = None, ) -> Generator[_TestOtelSpan, None, None]: resp = self._client.otel_trace_start_span( name=name, @@ -473,7 +460,6 @@ def otel_start_span( parent_id=parent_id, links=links if links is not None else [], attributes=attributes, - http_headers=http_headers if http_headers is not None else [], ) span = _TestOtelSpan(self._client, resp["span_id"], resp["trace_id"]) yield span From 69058fab0c7bc26c577e6589f0825d3676278ef5 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Wed, 20 Nov 2024 18:41:06 +0100 Subject: [PATCH 11/18] Fix run.sh on groups (#3510) --- run.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run.sh b/run.sh index a9b62e4137..7a30efc1a0 100755 --- a/run.sh +++ b/run.sh @@ -119,9 +119,7 @@ function lookup_scenario_group() { python+=(utils/scripts/get-scenarios-from-group.py) - echo "${python[*]}" 1>&2 - - cat < scenario_groups.yml | "${python[@]}" "${group}" + PYTHONPATH=. "${python[@]}" "${group}" } function upcase() { From 869f1f76047cd3a98d0075bae7d6dd4abd627457 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 20 Nov 2024 12:50:41 -0500 Subject: [PATCH 12/18] parametric: ensure trace/otel/end_span is up to spec (#3489) --- manifests/cpp.yml | 3 +- manifests/golang.yml | 2 +- manifests/nodejs.yml | 2 +- .../test_otel_api_interoperability.py | 14 +--- tests/parametric/test_otel_span_methods.py | 49 ++++++------- tests/parametric/test_otel_tracer.py | 8 +-- tests/parametric/test_parametric_endpoints.py | 70 ++++++++++++++----- .../parametric/Endpoints/ApmTestApiOtel.cs | 8 +-- .../build/docker/nodejs/parametric/server.js | 13 ++-- .../parametric/apm_test_client/server.py | 2 +- utils/build/docker/ruby/parametric/server.rb | 4 +- utils/parametric/_library_client.py | 12 ++-- 12 files changed, 100 insertions(+), 87 deletions(-) diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 98b0dd59b9..31e5862218 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -181,12 +181,13 @@ tests/: Test_Parametric_DDTrace_Current_Span: missing_feature (current_span endpoint is not implemented) Test_Parametric_DDTrace_Extract_Headers: missing_feature (extract_headers endpoint is not implemented) # cpp tracer does not support the OpenTelemetry API, otel parametric endpoints are not implemented + Test_Parametric_OtelSpan_End: missing_feature (otel api is not supported) Test_Parametric_OtelSpan_Events: missing_feature (otel api is not supported) Test_Parametric_OtelSpan_Is_Recording: missing_feature (otel api is not supported) Test_Parametric_OtelSpan_Set_Attribute: missing_feature (otel api is not supported) Test_Parametric_OtelSpan_Set_Name: missing_feature (otel api is not supported) Test_Parametric_OtelSpan_Set_Status: missing_feature (otel api is not supported) - Test_Parametric_OtelSpan_Start_Finish: missing_feature (otel api is not supported) + Test_Parametric_OtelSpan_Start: missing_feature (otel api is not supported) Test_Parametric_Otel_Baggage: missing_feature (otel api is not supported) Test_Parametric_Otel_Current_Span: missing_feature (otel api is not supported) Test_Parametric_Otel_Trace_Flush: missing_feature (otel api is not supported) diff --git a/manifests/golang.yml b/manifests/golang.yml index a29e2626e9..7c3701b1c9 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -512,7 +512,7 @@ tests/: Test_Parametric_DDTrace_Crash: missing_feature (crash endpoint is not implemented) Test_Parametric_DDTrace_Current_Span: missing_feature (spans are stored in a local context, there is no global current span in the go tracer) Test_Parametric_OtelSpan_Set_Name: bug (APMAPI-778) # The set_name endpoint should set the resouce name (not the span name) - Test_Parametric_OtelSpan_Start_Finish: bug (APMAPI-778) # String attributes are incorrectly stored/serialized in a list + Test_Parametric_OtelSpan_Start: bug (APMAPI-778) # String attributes are incorrectly stored/serialized in a list Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported) Test_Parametric_Otel_Current_Span: missing_feature (otel current span endpoint is not defined) test_span_links.py: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 69241aeade..62192d351c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -661,7 +661,7 @@ tests/: Test_Parametric_DDTrace_Crash: missing_feature (crash endpoint is not implemented) Test_Parametric_DDTrace_Current_Span: missing_feature (otel current_span endpoint is not supported) Test_Parametric_OtelSpan_Set_Name: bug (APMAPI-778) # set_name endpoint should set the resource name on a span (not the operation name) - Test_Parametric_OtelSpan_Start_Finish: bug (APMAPI-778) # The expected span.kind tag is not set + Test_Parametric_OtelSpan_Start: bug (APMAPI-778) # The expected span.kind tag is not set Test_Parametric_Otel_Baggage: missing_feature (baggage is not supported) Test_Parametric_Otel_Current_Span: missing_feature (otel baggage endpoints are not implemented) test_partial_flushing.py: diff --git a/tests/parametric/test_otel_api_interoperability.py b/tests/parametric/test_otel_api_interoperability.py index 8211e89f1a..70d57f30c2 100644 --- a/tests/parametric/test_otel_api_interoperability.py +++ b/tests/parametric/test_otel_api_interoperability.py @@ -62,7 +62,6 @@ def test_otel_start_after_datadog_span(self, test_agent, test_library): # FIXME: The trace_id is encoded in hex while span_id is an int. Make this API consistent assert current_dd_span.trace_id == otel_context.get("trace_id") assert "{:016x}".format(int(current_dd_span.span_id)) == otel_context.get("span_id") - otel_span.end_span() dd_span.finish() traces = test_agent.wait_for_num_traces(1, sort_by_start=False) @@ -112,8 +111,6 @@ def test_datadog_start_after_otel_span(self, test_agent, test_library): otel_current_span = test_library.otel_current_span() assert otel_current_span.span_id == otel_span.span_id - otel_span.end_span() - traces = test_agent.wait_for_num_traces(1, sort_by_start=False) trace = find_trace(traces, otel_span.trace_id) assert len(trace) == 2 @@ -235,7 +232,6 @@ def test_span_links_add(self, test_agent, test_library): current_span.add_link( parent_id=dd_span.span_id, attributes=TEST_ATTRIBUTES, ) - otel_span.end_span() traces = test_agent.wait_for_num_traces(2, sort_by_start=False) trace = find_trace(traces, otel_span.trace_id) @@ -254,9 +250,7 @@ def test_concurrent_traces_in_order(self, test_agent, test_library): with test_library.start_span(name="dd_child", parent_id=otel_root.span_id) as dd_child: with test_library.start_span(name="dd_root", parent_id=0) as dd_root: with test_library.otel_start_span(name="otel_child", parent_id=dd_root.span_id) as otel_child: - otel_child.end_span() - dd_root.finish() - otel_root.end_span() + pass traces = test_agent.wait_for_num_traces(2, sort_by_start=False) @@ -294,7 +288,6 @@ def test_concurrent_traces_nested_otel_root(self, test_agent, test_library): name="otel_child", parent_id=otel_root.span_id, span_kind=SpanKind.INTERNAL ) as otel_child: with test_library.start_span(name="dd_child", parent_id=dd_root.span_id) as dd_child: - otel_child.end_span() current_span = test_library.current_span() assert current_span.span_id == dd_child.span_id @@ -305,7 +298,6 @@ def test_concurrent_traces_nested_otel_root(self, test_agent, test_library): current_span = test_library.current_span() assert current_span.span_id == otel_root.span_id - otel_root.end_span() traces = test_agent.wait_for_num_traces(2, sort_by_start=False) @@ -343,7 +335,6 @@ def test_concurrent_traces_nested_dd_root(self, test_agent, test_library): name="otel_child", parent_id=otel_root.span_id, span_kind=SpanKind.INTERNAL ) as otel_child: with test_library.start_span(name="dd_child", parent_id=dd_root.span_id) as dd_child: - otel_child.end_span() current_span = test_library.current_span() assert current_span.span_id == dd_child.span_id @@ -354,7 +345,6 @@ def test_concurrent_traces_nested_dd_root(self, test_agent, test_library): current_span = test_library.current_span() assert current_span.span_id == otel_root.span_id - otel_root.end_span() traces = test_agent.wait_for_num_traces(2, sort_by_start=False) @@ -498,8 +488,6 @@ def test_set_attribute_from_datadog(self, test_agent, test_library): dd_span.set_meta("nested_str_array", [["a", "b"], ["c", "d"]]) dd_span.set_metric("int_array", [1, 2, 3]) - otel_span.end_span() - traces = test_agent.wait_for_num_traces(1, sort_by_start=False) trace = find_trace(traces, otel_span.span_id) assert len(trace) == 1 diff --git a/tests/parametric/test_otel_span_methods.py b/tests/parametric/test_otel_span_methods.py index 561982c661..63b3ded69b 100644 --- a/tests/parametric/test_otel_span_methods.py +++ b/tests/parametric/test_otel_span_methods.py @@ -64,7 +64,6 @@ def test_otel_set_service_name(self, test_agent, test_library): with test_library: with test_library.otel_start_span("parent_span", span_kind=SpanKind.INTERNAL) as parent: parent.set_attributes({"service.name": "new_service"}) - parent.end_span() traces = test_agent.wait_for_num_traces(num=1) trace = find_trace(traces, parent.trace_id) @@ -90,7 +89,6 @@ def test_otel_set_attribute_remapping_httpresponsestatuscode(self, test_agent, t with test_library: with test_library.otel_start_span("operation") as span: span.set_attributes({"http.response.status_code": 200}) - span.end_span() traces = test_agent.wait_for_num_traces(num=1) trace = find_trace(traces, span.trace_id) @@ -117,7 +115,6 @@ def test_otel_set_attribute_remapping_httpstatuscode(self, test_agent, test_libr with test_library: with test_library.otel_start_span("operation") as span: span.set_attributes({"http.status_code": 200}) - span.end_span() traces = test_agent.wait_for_num_traces(num=1) trace = find_trace(traces, span.trace_id) @@ -155,7 +152,7 @@ def test_otel_set_attributes_different_types_legacy(self, test_agent, test_libra span.set_attributes({"array_val_bool": [True, False]}) span.set_attributes({"array_val_double": [10.1, 20.2]}) span.set_attributes({"d_str_val": "bye", "d_bool_val": False, "d_int_val": 2, "d_double_val": 3.14}) - span.end_span() + traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) assert len(trace) == 1 @@ -236,7 +233,7 @@ def test_otel_set_attributes_different_types_with_array_encoding(self, test_agen span.set_attributes({"array_val_bool": [True, False]}) span.set_attributes({"array_val_double": [10.1, 20.2]}) span.set_attributes({"d_str_val": "bye", "d_bool_val": False, "d_int_val": 2, "d_double_val": 3.14}) - span.end_span() + traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) assert len(trace) == 1 @@ -282,10 +279,9 @@ def test_otel_span_is_recording(self, test_agent, test_library): """ with test_library: # start parent - with test_library.otel_start_span(name="parent") as parent: + with test_library.otel_start_span(name="parent", end_on_exit=True) as parent: assert parent.is_recording() - parent.end_span() - assert not parent.is_recording() + assert not parent.is_recording() @missing_feature(context.library <= "java@1.23.0", reason="Implemented in 1.24.0") @missing_feature(context.library == "nodejs", reason="New operation name mapping not yet implemented") @@ -330,7 +326,7 @@ def test_otel_span_end(self, test_agent, test_library): - still possible to start child spans from parent context """ with test_library: - with test_library.otel_start_span(name="parent", span_kind=SpanKind.PRODUCER) as parent: + with test_library.otel_start_span(name="parent", span_kind=SpanKind.PRODUCER, end_on_exit=False) as parent: parent.end_span() # setting attributes after finish has no effect parent.set_name("new_name") @@ -338,7 +334,7 @@ def test_otel_span_end(self, test_agent, test_library): with test_library.otel_start_span( name="child", span_kind=SpanKind.CONSUMER, parent_id=parent.span_id ) as child: - child.end_span() + pass traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, parent.trace_id) @@ -375,7 +371,7 @@ def test_otel_set_span_status_error(self, test_agent, test_library): with test_library.otel_start_span(name="error_span", span_kind=SpanKind.INTERNAL) as s: s.set_status(StatusCode.ERROR, "error_desc") s.set_status(StatusCode.UNSET, "unset_desc") - s.end_span() + traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, s.trace_id) s = find_span(trace, s.span_id) @@ -403,7 +399,6 @@ def test_otel_set_span_status_ok(self, test_agent, test_library): with test_library.otel_start_span(name="ok_span", span_kind=SpanKind.INTERNAL) as span: span.set_status(StatusCode.OK, "ok_desc") span.set_status(StatusCode.ERROR, "error_desc") - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -420,9 +415,9 @@ def test_otel_get_span_context(self, test_agent, test_library): (https://opentelemetry.io/docs/reference/specification/trace/api/#get-context) """ with test_library: - with test_library.otel_start_span(name="op1") as parent: + with test_library.otel_start_span(name="op1", end_on_exit=False) as parent: parent.end_span() - with test_library.otel_start_span(name="op2", parent_id=parent.span_id) as span: + with test_library.otel_start_span(name="op2", parent_id=parent.span_id, end_on_exit=False) as span: span.end_span() context = span.span_context() assert context.get("trace_id") == parent.span_context().get("trace_id") @@ -462,7 +457,6 @@ def test_otel_set_attributes_separately(self, test_agent, test_library): with test_library.otel_start_span(name="operation", span_kind=SpanKind.CLIENT) as span: span.set_attributes({"messaging.system": "Kafka"}) span.set_attributes({"messaging.operation": "Receive"}) - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -484,14 +478,14 @@ def test_otel_span_started_with_link_from_another_span(self, test_agent, test_li given two valid span (or SpanContext) objects" as specified in the RFC. """ with test_library: - with test_library.otel_start_span("root") as parent: + with test_library.otel_start_span("root", end_on_exit=False) as parent: parent.end_span() with test_library.otel_start_span( "child", parent_id=parent.span_id, links=[Link(parent_id=parent.span_id, attributes={"foo": "bar", "array": ["a", "b", "c"]})], ) as child: - child.end_span() + pass traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, parent.trace_id) @@ -534,7 +528,7 @@ def test_otel_span_link_attribute_handling(self, test_agent, test_library): ) ], ) as s2: - s2.end_span() + pass traces = test_agent.wait_for_num_traces(2) trace = find_trace(traces, s2.trace_id) @@ -566,10 +560,11 @@ def test_otel_span_started_with_link_from_other_spans(self, test_agent, test_lib """Test adding a span link from a span to another span. """ with test_library: - with test_library.otel_start_span("root") as parent: + with test_library.otel_start_span("root", end_on_exit=False) as parent: parent.end_span() with test_library.otel_start_span("first", parent_id=parent.span_id) as first: - first.end_span() + pass + with test_library.otel_start_span( "second", parent_id=parent.span_id, @@ -578,7 +573,7 @@ def test_otel_span_started_with_link_from_other_spans(self, test_agent, test_lib Link(parent_id=first.span_id, attributes={"bools": [True, False], "nested": [1, 2]}), ], ) as second: - second.end_span() + pass traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, parent.trace_id) @@ -670,7 +665,7 @@ def test_otel_span_reserved_attributes_overrides(self, test_agent, test_library) span.set_attributes({"service.name": "new.service.name"}) span.set_attributes({"span.type": "new.span.type"}) span.set_attributes({"analytics.event": "true"}) - span.end_span() + traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) assert len(trace) == 1 @@ -792,7 +787,6 @@ def test_otel_add_event_meta_serialization(self, test_agent, test_library): timestamp=1, attributes={"int_val": 1, "string_val": "2", "int_array": [3, 4], "string_array": ["5", "6"]}, ) - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -835,7 +829,6 @@ def test_otel_record_exception_does_not_set_error(self, test_agent, test_library with test_library: with test_library.otel_start_span("operation") as span: span.record_exception(message="woof", attributes={"exception.stacktrace": "stacktrace string"}) - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -861,7 +854,6 @@ def test_otel_record_exception_meta_serialization(self, test_agent, test_library ) span.add_event(name="non_exception_event", attributes={"exception.stacktrace": "non-error"}) span.record_exception(message="woof3", attributes={"exception.message": "message override"}) - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -910,7 +902,6 @@ def test_otel_record_exception_attributes_serialization(self, test_agent, test_l ) span.add_event(name="non_exception_event", attributes={"exception.stacktrace": "non-error"}) span.record_exception(message="woof3", attributes={"exception.message": "message override"}) - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -956,7 +947,6 @@ def test_otel_record_exception_sets_all_error_tracking_tags(self, test_agent, te span.record_exception( message="woof1", attributes={"string_val": "value", "exception.stacktrace": "stacktrace1"} ) - span.end_span() traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) @@ -971,7 +961,8 @@ def test_otel_record_exception_sets_all_error_tracking_tags(self, test_agent, te def run_operation_name_test(expected_operation_name: str, span_kind: int, attributes: dict, test_library, test_agent): with test_library: with test_library.otel_start_span("otel_span_name", span_kind=span_kind, attributes=attributes) as span: - span.end_span() + pass + traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) assert len(trace) == 1 @@ -987,7 +978,7 @@ def run_otel_span_reserved_attributes_overrides_analytics_event( with test_library: with test_library.otel_start_span("operation", span_kind=SpanKind.SERVER) as span: span.set_attributes({"analytics.event": analytics_event_value}) - span.end_span() + traces = test_agent.wait_for_num_traces(1) trace = find_trace(traces, span.trace_id) assert len(trace) == 1 diff --git a/tests/parametric/test_otel_tracer.py b/tests/parametric/test_otel_tracer.py index 7ca9030d33..c035a46887 100644 --- a/tests/parametric/test_otel_tracer.py +++ b/tests/parametric/test_otel_tracer.py @@ -26,13 +26,10 @@ def test_otel_simple_trace(self, test_agent, test_library): parent1.set_attributes({"parent_k1": "parent_v1"}) with test_library.otel_start_span(name="child1", parent_id=parent1.span_id) as child1: assert parent1.span_context()["trace_id"] == child1.span_context()["trace_id"] - child1.end_span() - parent1.end_span() + with test_library.otel_start_span("root_two") as parent2: with test_library.otel_start_span(name="child2", parent_id=parent2.span_id) as child2: assert parent2.span_context()["trace_id"] == child2.span_context()["trace_id"] - child2.end_span() - parent2.end_span() traces = test_agent.wait_for_num_traces(2) trace_one = find_trace(traces, parent1.trace_id) @@ -63,7 +60,8 @@ def test_otel_force_flush(self, test_agent, test_library): """ with test_library: with test_library.otel_start_span(name="test_span") as span: - span.end_span() + pass + # force flush with 5 second time out flushed = test_library.otel_flush(5) assert flushed, "ForceFlush error" diff --git a/tests/parametric/test_parametric_endpoints.py b/tests/parametric/test_parametric_endpoints.py index 3903d39f46..1013d157ce 100644 --- a/tests/parametric/test_parametric_endpoints.py +++ b/tests/parametric/test_parametric_endpoints.py @@ -8,6 +8,7 @@ """ import json import pytest +import time from utils.parametric.spec.trace import find_trace from utils.parametric.spec.trace import find_span @@ -350,7 +351,7 @@ def test_current_span_from_otel(self, test_agent, test_library): with test_library.otel_start_span("span_test_current_span_from_otel") as s1: current_span = test_library.current_span() assert current_span.span_id == s1.span_id - s1.end_span() + current_span = test_library.current_span() assert int(current_span.span_id) == 0 assert int(current_span.trace_id) == 0 @@ -477,10 +478,10 @@ def test_remove_all_baggage(self, test_agent, test_library): @scenarios.parametric @features.parametric_endpoint_parity -class Test_Parametric_OtelSpan_Start_Finish: - def test_span_start_and_finish(self, test_agent, test_library): +class Test_Parametric_OtelSpan_Start: + def test_span_start(self, test_agent, test_library): """ - Validates that the /trace/otel/start_span creates a new span and that the /trace/otel/end_span finishes a span and sends it to the agent. + Validates that the /trace/otel/start_span creates a new span. Supported Parameters: - name: str @@ -495,10 +496,10 @@ def test_span_start_and_finish(self, test_agent, test_library): """ with test_library: with test_library.otel_start_span("otel_start_span_parent") as s1: - s1.end_span() + pass with test_library.otel_start_span("otel_start_span_linked") as s2: - s2.end_span() + pass with test_library.otel_start_span( "otel_start_span_child", @@ -508,7 +509,7 @@ def test_span_start_and_finish(self, test_agent, test_library): [Link(parent_id=s2.span_id, attributes={"link.key": "value"})], {"attr_key": "value"}, ) as s3: - s3.end_span() + pass traces = test_agent.wait_for_num_traces(2) first_trace = find_trace(traces, s1.trace_id) @@ -526,6 +527,49 @@ def test_span_start_and_finish(self, test_agent, test_library): assert links[0]["attributes"]["link.key"] == "value" +@scenarios.parametric +@features.parametric_endpoint_parity +class Test_Parametric_OtelSpan_End: + def test_span_end(self, test_agent, test_library): + """ + Validates that the /trace/otel/end_span finishes a span and sends it to the agent + + Supported Parameters: + - timestamp (μs): Optional[int] + Supported Return Values: + """ + sleep = 0.2 + t1 = time.time() + with test_library: + with test_library.otel_start_span("otel_end_span", end_on_exit=True) as s1: + time.sleep(sleep) + total_time = time.time() - t1 + + traces = test_agent.wait_for_num_traces(1) + span = find_only_span(traces) + assert sleep <= span["duration"] / 1e9 <= total_time, span["start"] + + def test_span_end_with_timestamp(self, test_agent, test_library): + """ + Validates that the /trace/otel/end_span finishes a span and sends it to the agent with the expected duration + + Supported Parameters: + - timestamp (μs): Optional[int] + Supported Return Values: + """ + start = 5_000_000 # microseconds + end = 10_000_000 # microseconds + with test_library: + with test_library.otel_start_span("otel_end_span_with_timestamp", timestamp=start, end_on_exit=False) as s1: + pass + s1.end_span(end) + + traces = test_agent.wait_for_num_traces(1) + span = find_only_span(traces) + assert span["start"] == start * 1000 + assert span["duration"] == (end - start) * 1000 + + @scenarios.parametric @features.parametric_endpoint_parity class Test_Parametric_OtelSpan_Set_Attribute: @@ -541,7 +585,6 @@ def test_otel_set_attribute(self, test_agent, test_library): with test_library: with test_library.otel_start_span("otel_set_attribute") as s1: s1.set_attribute("key", "value") - s1.end_span() traces = test_agent.wait_for_num_traces(1) span = find_only_span(traces) @@ -564,7 +607,6 @@ def test_otel_set_status(self, test_agent, test_library): with test_library: with test_library.otel_start_span("otel_set_status") as s1: s1.set_status(StatusCode.ERROR, "error message") - s1.end_span() traces = test_agent.wait_for_num_traces(1) span = find_only_span(traces) @@ -586,7 +628,6 @@ def test_otelspan_set_name(self, test_agent, test_library): with test_library: with test_library.otel_start_span("otel_set_name") as s1: s1.set_name("new_name") - s1.end_span() traces = test_agent.wait_for_num_traces(1) span = find_only_span(traces) @@ -610,7 +651,6 @@ def test_add_event(self, test_agent, test_library): with test_library: with test_library.otel_start_span("otel_add_event") as s1: s1.add_event("some_event", 1730393556000000, {"key": "value"}) - s1.end_span() traces = test_agent.wait_for_num_traces(1) span = find_only_span(traces) @@ -636,7 +676,6 @@ def test_record_exception(self, test_agent, test_library): with test_library: with test_library.otel_start_span("otel_record_exception") as s1: s1.record_exception("MyException Parametric tests rock", {"error.key": "value"}) - s1.end_span() traces = test_agent.wait_for_num_traces(1) span = find_only_span(traces) @@ -661,7 +700,6 @@ def test_is_recording(self, test_agent, test_library): """ with test_library.otel_start_span("otel_is_recording") as s1: assert s1.is_recording() - s1.end_span() @scenarios.parametric @@ -680,7 +718,6 @@ def test_set_baggage(self, test_agent, test_library): with test_library.otel_start_span("otel_set_baggage") as s1: value = test_library.otel_set_baggage(s1.span_id, "foo", "bar") assert value == "bar" - s1.end_span() @scenarios.parametric @@ -706,11 +743,9 @@ def test_otel_current_span(self, test_agent, test_library): with test_library.otel_start_span("span_test_current_spans_s2", parent_id=s1.span_id) as s2: current_span = test_library.otel_current_span() assert current_span.span_id == s2.span_id - s2.end_span() current_span = test_library.otel_current_span() assert current_span.span_id == s1.span_id - s1.end_span() current_span = test_library.otel_current_span() assert int(current_span.span_id) == 0 @@ -732,5 +767,6 @@ def test_flush(self, test_agent, test_library): # Here we are avoiding using the __exit__() operation on the contextmanager # and instead manually finishing and flushing the span. with test_library.otel_start_span("test_otel_flush") as s1: - s1.end_span() + pass + assert test_library.otel_flush(timeout_sec=5) diff --git a/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs b/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs index 4ffaf6ab90..e57963a09f 100644 --- a/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs +++ b/utils/build/docker/dotnet/parametric/Endpoints/ApmTestApiOtel.cs @@ -56,9 +56,9 @@ private static async Task OtelStartSpan(HttpRequest request) } DateTimeOffset startTime = default; - if (requestBodyObject.TryGetValue("timestamp", out var timestamp)) + if (requestBodyObject.TryGetValue("timestamp", out var timestamp) && Convert.ToInt64(timestamp) is long timestampInMicroseconds && timestampInMicroseconds > 0) { - startTime = new DateTime(1970, 1, 1) + TimeSpan.FromMicroseconds(Convert.ToInt64(timestamp)); + startTime = new DateTime(1970, 1, 1) + TimeSpan.FromMicroseconds(timestampInMicroseconds); } var parentContext = localParentContext ?? remoteParentContext ?? default; @@ -158,9 +158,9 @@ private static async Task OtelEndSpan(HttpRequest request) var activity = FindActivity(requestBodyObject["id"]); - if (!string.IsNullOrEmpty(requestBodyObject["timestamp"].ToString())) + if (requestBodyObject.TryGetValue("timestamp", out var timestamp) && timestamp is not null) { - DateTimeOffset convertedTimestamp = new DateTime(1970, 1, 1) + TimeSpan.FromMicroseconds(Convert.ToInt64(requestBodyObject["timestamp"])); + DateTimeOffset convertedTimestamp = new DateTime(1970, 1, 1) + TimeSpan.FromMicroseconds(Convert.ToInt64(timestamp)); activity.SetEndTime(convertedTimestamp.UtcDateTime); } diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index 4af103152a..3e4f7bb08a 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -21,10 +21,13 @@ const app = express(); app.use(express.json()); -function nanoLongToHrTime ({ high = 0, low = 0 } = {}) { +function microLongToHrTime (timestamp) { + if (timestamp === null) { + return [0, 0] + } return [ - high * 1e3 + Math.floor(low / 1e6), - (low % 1e6) * 1e3, + Math.floor(timestamp / 1000000), + (timestamp % 1000000) * 1000, ] } @@ -168,7 +171,7 @@ app.post('/trace/otel/start_span', (req, res) => { kind: request.kind, attributes: request.attributes, links, - startTime: nanoLongToHrTime(request.timestamp) + startTime: microLongToHrTime(request.timestamp) }, parentContext) const ctx = span._ddSpan.context() const span_id = ctx._spanId.toString(10) @@ -190,7 +193,7 @@ app.post('/trace/otel/end_span', (req, res) => { const { id, timestamp } = req.body; const span_id = `${id}` const span = otelSpans[span_id] - span.end(nanoLongToHrTime(timestamp)) + span.end(microLongToHrTime(timestamp)) res.json({}); }); diff --git a/utils/build/docker/python/parametric/apm_test_client/server.py b/utils/build/docker/python/parametric/apm_test_client/server.py index a305689d0d..e0726ae326 100644 --- a/utils/build/docker/python/parametric/apm_test_client/server.py +++ b/utils/build/docker/python/parametric/apm_test_client/server.py @@ -501,7 +501,7 @@ def otel_record_exception(args: OtelRecordExceptionArgs) -> OtelRecordExceptionR class OtelEndSpanArgs(BaseModel): id: int - timestamp: int + timestamp: Optional[int] class OtelEndSpanReturn(BaseModel): diff --git a/utils/build/docker/ruby/parametric/server.rb b/utils/build/docker/ruby/parametric/server.rb index eb75b15566..e8fd1b0cad 100644 --- a/utils/build/docker/ruby/parametric/server.rb +++ b/utils/build/docker/ruby/parametric/server.rb @@ -477,9 +477,7 @@ def extract_http_headers(headers) # OTel system tests provide times in microseconds, but Ruby OTel # measures time in seconds (Float). def otel_correct_time(microseconds) - if microseconds.nil? || microseconds == 0 - microseconds - else + unless microseconds.nil? || microseconds == 0 microseconds / 1_000_000.0 end end diff --git a/utils/parametric/_library_client.py b/utils/parametric/_library_client.py index b8d9df4f8b..e195461baf 100644 --- a/utils/parametric/_library_client.py +++ b/utils/parametric/_library_client.py @@ -243,7 +243,7 @@ def otel_trace_start_span( # and others with bignum trace_ids and uint64 span_ids (ex: python). We should standardize this. return StartSpanResponse(span_id=resp["span_id"], trace_id=resp["trace_id"]) - def otel_end_span(self, span_id: int, timestamp: int) -> None: + def otel_end_span(self, span_id: int, timestamp: Optional[int]) -> None: self._session.post(self._url("/trace/otel/end_span"), json={"id": span_id, "timestamp": timestamp}) def otel_set_attributes(self, span_id: int, attributes) -> None: @@ -390,7 +390,7 @@ def add_event(self, name: str, timestamp: Optional[int] = None, attributes: Opti def record_exception(self, message: str, attributes: Optional[dict] = None): self._client.otel_record_exception(self.span_id, message, attributes) - def end_span(self, timestamp: int = 0): + def end_span(self, timestamp: Optional[int] = None): self._client.otel_end_span(self.span_id, timestamp) def is_recording(self) -> bool: @@ -452,6 +452,7 @@ def otel_start_span( parent_id: int = 0, links: Optional[List[Link]] = None, attributes: dict = None, + end_on_exit: bool = True, ) -> Generator[_TestOtelSpan, None, None]: resp = self._client.otel_trace_start_span( name=name, @@ -463,11 +464,8 @@ def otel_start_span( ) span = _TestOtelSpan(self._client, resp["span_id"], resp["trace_id"]) yield span - - return { - "span_id": resp["span_id"], - "trace_id": resp["trace_id"], - } + if end_on_exit: + span.end_span() def flush(self) -> bool: return self._client.trace_flush() From db3d3bb299f94dd86f9db4defce3217c434bf923 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 20 Nov 2024 14:48:27 -0500 Subject: [PATCH 13/18] parametric: re-enable config consistency tests for ruby (#3381) Co-authored-by: Charles de Beauchesne --- tests/parametric/test_config_consistency.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/parametric/test_config_consistency.py b/tests/parametric/test_config_consistency.py index 3ac49ea122..912e37f4d6 100644 --- a/tests/parametric/test_config_consistency.py +++ b/tests/parametric/test_config_consistency.py @@ -89,7 +89,7 @@ def test_default_config(self, library_env, test_agent, test_library): # Assert that iff a span has service name set by DD_SERVICE, it also gets the version specified in DD_VERSION @parametrize("library_env", [{"DD_SERVICE": "version_test", "DD_VERSION": "5.2.0"}]) - @missing_feature(library="ruby") + @missing_feature(context.library < "ruby@2.7.1-dev") def test_specific_version(self, library_env, test_agent, test_library): with test_library: with test_library.start_span(name="s1") as s1: @@ -132,7 +132,6 @@ class Test_Config_TraceAgentURL: which would be unnecessarily complex. """ - @missing_feature(library="ruby") @parametrize( "library_env", [ @@ -148,7 +147,7 @@ def test_dd_trace_agent_unix_url_nonexistent(self, library_env, test_agent, test resp = t.get_tracer_config() url = urlparse(resp["dd_trace_agent_url"]) - assert url.scheme == "unix" + assert "unix" in url.scheme assert url.path == "/var/run/datadog/apm.socket" # The DD_TRACE_AGENT_URL is validated using the tracer configuration. This approach avoids the need to modify the setup file to create additional containers at the specified URL, which would be unnecessarily complex. From dd6a98741c240db26a3e6e146cdf814e934ed482 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:17:06 -0500 Subject: [PATCH 14/18] [docs]: update references in READMEs (#3486) --- README.md | 2 +- docs/edit/README.md | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index fe1ba0e39b..cef80a3e64 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Refer to the [edit docs](docs/edit/README.md). **[Complete documentation](https://github.com/DataDog/system-tests/blob/main/docs)** -System-tests supports various scenarios for running tests; read more about the different kinds of tests that this repo covers in [scenarios/README.md](scenarios/README.md). +System-tests supports various scenarios for running tests; read more about the different kinds of tests that this repo covers in [scenarios/README.md](docs/scenarios/README.md). Understand the test architecture at the [architectural overview](https://github.com/DataDog/system-tests/blob/main/docs/architecture/overview.md). diff --git a/docs/edit/README.md b/docs/edit/README.md index 097f62985f..8495c096f1 100644 --- a/docs/edit/README.md +++ b/docs/edit/README.md @@ -13,15 +13,15 @@ To make changes, you must be able to run tests locally. Instructions for running You'll commonly need to run unmerged changes to your library against system tests (e.g. to ensure the feature is up to spec). Instructions for testing against unmerged changes can be found in [enable-test.md](./enable-test.md). ## Index -1. [lifecycle.md](./lifecycle.md): Understand how system tests work -2. [add-new-test.md](./add-new-test.md): Add a new test -3. [scenarios.md](./scenarios.md): Add a new scenario -4. [format.md](./format.md): Use the linter -5. [features.md](./features.md): Mark tests for the feature parity dashboard -6. [enable-test.md](./enable-test.md): Enable a test -7. [skip-tests.md](./skip-tests.md): Disable tests -8. [manifest.md](./manifest.md): How tests are marked as enabled or disabled for libraries -9. [features.md](./features.md): Mark tests for the feature parity dashboard -10. [format.md](./format.md): Use the linter -11. [troubleshooting.md](./troubleshooting.md) Tips for debugging -12. [iast-validations.md](./iast-validations.md): Mark tests with vulnerabilities +1. [add-new-test.md](./add-new-test.md): Add a new test +2. [scenarios.md](./scenarios.md): Add a new scenario +3. [format.md](./format.md): Use the linter +4. [features.md](./features.md): Mark tests for the feature parity dashboard +5. [enable-test.md](./enable-test.md): Enable a test +6. [skip-tests.md](./skip-tests.md): Disable tests +7. [manifest.md](./manifest.md): How tests are marked as enabled or disabled for libraries +8. [troubleshooting.md](./troubleshooting.md) Tips for debugging +9. [iast-validations.md](./iast-validations.md): Mark tests with vulnerabilities +10. [CI-and-scenarios.md](./CI-and-scenarios.md): Understand how scenarios run in CI +11. [update-docker-images.md](./update-docker-images.md): Modify test app docker images +12. [remote-config.md](./remote-config.md): Write remote config tests From e2e6abdd939342e40d77b8f82c0e34587cc32c26 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 20 Nov 2024 16:08:50 -0500 Subject: [PATCH 15/18] [php] parametric: update dd and otel flush endpoints (#3514) Co-authored-by: Bob Weinand --- manifests/php.yml | 1 - utils/build/docker/php/parametric/server.php | 17 ++++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifests/php.yml b/manifests/php.yml index 7e3b9670c3..0a47e38cff 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -332,7 +332,6 @@ tests/: Test_Parametric_DDSpan_Start: bug (APMAPI-778) # Does not support creating a child span from a finished span Test_Parametric_DDTrace_Baggage: missing_feature (baggage is not supported) Test_Parametric_DDTrace_Current_Span: bug (APMAPI-778) # current span endpoint should return span and trace id of zero if no span is "active" - Test_Parametric_DDTrace_Flush: missing_feature (flush endpoint is not implemented, the lack of this feature introduce flakiness in all tests) Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported) Test_Parametric_Otel_Current_Span: bug (APMAPI-778) # otel current span endpoint should return a span and trace id of zero if no span is "active" test_partial_flushing.py: diff --git a/utils/build/docker/php/parametric/server.php b/utils/build/docker/php/parametric/server.php index 068f184f4c..bf78e46e78 100644 --- a/utils/build/docker/php/parametric/server.php +++ b/utils/build/docker/php/parametric/server.php @@ -239,8 +239,11 @@ function remappedSpanKind($spanKind) { return jsonResponse([]); })); $router->addRoute('POST', '/trace/span/flush', new ClosureRequestHandler(function () use (&$spans) { - \DDTrace\flush(); - dd_trace_internal_fn("synchronous_flush"); + dd_trace_synchronous_flush(1000); # flush spans with a timeout of 1s + return jsonResponse([]); +})); +$router->addRoute('POST', '/trace/stats/flush', new ClosureRequestHandler(function () use (&$spans) { + # NOP: php doesn't expose an API to flush trace stats return jsonResponse([]); })); $router->addRoute('GET', '/trace/span/current', new ClosureRequestHandler(function () use (&$spans, &$activeSpan) { @@ -398,11 +401,11 @@ function remappedSpanKind($spanKind) { return jsonResponse([]); })); $router->addRoute('POST', '/trace/otel/flush', new ClosureRequestHandler(function (Request $req) { - \DDTrace\flush(); - dd_trace_internal_fn("synchronous_flush"); - return jsonResponse([ - 'success' => true - ]); + $timeout = (arg($req, 'seconds') ?: 0.1) * 1000; # convert timeout to ms + dd_trace_synchronous_flush($timeout); + return jsonResponse([ + 'success' => true + ]); })); $router->addRoute('POST', '/trace/otel/is_recording', new ClosureRequestHandler(function (Request $req) use (&$otelSpans) { $spanId = arg($req, 'span_id'); From abffe1d7af095a11729e555e64e930c1156a702c Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:07:51 +0100 Subject: [PATCH 16/18] OnBoardin: Fix git sources when gitlab tag (#3518) --- utils/virtual_machine/virtual_machine_provider.py | 6 +++--- utils/virtual_machine/virtual_machine_provisioner.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/virtual_machine/virtual_machine_provider.py b/utils/virtual_machine/virtual_machine_provider.py index 1ad990917e..88efb60d82 100644 --- a/utils/virtual_machine/virtual_machine_provider.py +++ b/utils/virtual_machine/virtual_machine_provider.py @@ -89,11 +89,11 @@ def install_provision(self, vm, server, server_connection): logger_name="tested_components", output_callback=output_callback, ) - # Before install weblog, if we set the env variable: CI_COMMIT_BRANCH, we need to checkout this branch + # Before install weblog, if we set the env variable: GITLAB_CI, we need to checkout the CI_COMMIT_BRANCH branch # (we are going to copy weblog sources from git instead from local machine) # We commit the branch reference of the CI_COMMIT_BRANCH env variable only if the gitlab project is system-tests - # Proabably we need to change this in the future, and translate this logic to the pipelines - ci_commit_branch = os.getenv("CI_COMMIT_BRANCH") + # Proabably we need to change this in the future, and translate this logic to the pipelines or another class + ci_commit_branch = os.getenv("GITLAB_CI") if ci_commit_branch: ci_commit_branch = ( os.getenv("CI_COMMIT_BRANCH") if os.getenv("CI_PROJECT_NAME", "") == "system-tests" else "main" diff --git a/utils/virtual_machine/virtual_machine_provisioner.py b/utils/virtual_machine/virtual_machine_provisioner.py index 9b4bf963af..bfe53f779e 100644 --- a/utils/virtual_machine/virtual_machine_provisioner.py +++ b/utils/virtual_machine/virtual_machine_provisioner.py @@ -215,7 +215,7 @@ def _get_weblog_provision( weblog = weblog_raw_data["weblog"] assert weblog["name"] == weblog_name, f"Weblog name {weblog_name} does not match the provision file name" installations = weblog["install"] - ci_commit_branch = os.getenv("CI_COMMIT_BRANCH") + ci_commit_branch = os.getenv("GITLAB_CI") installation = self._get_installation( env, library_name, From 1df7344117b8e27f1706b0a562606f625f7ee095 Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:43:48 +0100 Subject: [PATCH 17/18] Onboarding: gitlab check timeouts (#3492) --- .gitlab-ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a805e7c60..07d1fc4313 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,6 +47,7 @@ onboarding_nodejs: stage: nodejs_tracer allow_failure: true needs: [] + timeout: 90 minutes rules: - if: $CI_PIPELINE_SOURCE == "schedule" && ($ONLY_TEST_LIBRARY == "" || $ONLY_TEST_LIBRARY == "nodejs") when: always @@ -79,13 +80,14 @@ onboarding_nodejs: DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} + - timeout 4800s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} onboarding_java: extends: .base_job_onboarding_system_tests stage: java_tracer allow_failure: true needs: [] + timeout: 90 minutes rules: - if: $CI_PIPELINE_SOURCE == "schedule" && ($ONLY_TEST_LIBRARY == "" || $ONLY_TEST_LIBRARY == "java") when: always @@ -118,12 +120,13 @@ onboarding_java: DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} + - timeout 4800s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} onboarding_python: extends: .base_job_onboarding_system_tests stage: python_tracer allow_failure: true needs: [] + timeout: 90 minutes rules: - if: $CI_PIPELINE_SOURCE == "schedule" && ($ONLY_TEST_LIBRARY == "" || $ONLY_TEST_LIBRARY == "python") when: always @@ -156,13 +159,14 @@ onboarding_python: DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} + - timeout 4800s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} onboarding_dotnet: extends: .base_job_onboarding_system_tests stage: dotnet_tracer allow_failure: true needs: [] + timeout: 90 minutes rules: - if: $CI_PIPELINE_SOURCE == "schedule" && ($ONLY_TEST_LIBRARY == "" || $ONLY_TEST_LIBRARY == "dotnet") when: always @@ -192,13 +196,14 @@ onboarding_dotnet: DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} + - timeout 4800s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} onboarding_ruby: extends: .base_job_onboarding_system_tests stage: ruby_tracer allow_failure: true needs: [] + timeout: 90 minutes rules: - if: $CI_PIPELINE_SOURCE == "schedule" && ($ONLY_TEST_LIBRARY == "" || $ONLY_TEST_LIBRARY == "ruby") when: always @@ -224,13 +229,14 @@ onboarding_ruby: DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} + - timeout 4800s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} onboarding_php: extends: .base_job_onboarding_system_tests stage: php_tracer allow_failure: true needs: [] + timeout: 90 minutes rules: - if: $CI_PIPELINE_SOURCE == "schedule" && ($ONLY_TEST_LIBRARY == "" || $ONLY_TEST_LIBRARY == "php") when: always @@ -259,7 +265,7 @@ onboarding_php: DEFAULT_VMS: ["True", "False"] script: - ./build.sh -i runner - - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} + - timeout 4800s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws --report-run-url ${CI_PIPELINE_URL} --report-environment ${ONBOARDING_FILTER_ENV} --vm-default-vms ${DEFAULT_VMS} onboarding_stats_results: From 6c4ddd1707a152d42f33cc9356f31f0ac20aae78 Mon Sep 17 00:00:00 2001 From: Roberto Montero <108007532+robertomonteromiguel@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:15:18 +0100 Subject: [PATCH 18/18] OnBoarding fix nodejs installation (#3502) --- utils/_context/_scenarios/auto_injection.py | 2 +- .../auto-inject_init_vm_config.yml | 21 +++++++++++++------ .../nodejs/provision_test-app-nodejs.yml | 4 +--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/utils/_context/_scenarios/auto_injection.py b/utils/_context/_scenarios/auto_injection.py index 87f79295b3..e3c74ac0f5 100644 --- a/utils/_context/_scenarios/auto_injection.py +++ b/utils/_context/_scenarios/auto_injection.py @@ -388,7 +388,7 @@ def __init__( include_ubuntu_22_arm64=True, include_ubuntu_23_04_amd64=True, include_ubuntu_23_04_arm64=True, - include_ubuntu_23_10_amd64=True, + include_ubuntu_23_10_amd64=False, include_ubuntu_23_10_arm64=True, include_ubuntu_24_amd64=True, include_ubuntu_24_arm64=True, diff --git a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_init_vm_config.yml b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_init_vm_config.yml index dee4860c03..facdd6c239 100644 --- a/utils/build/virtual_machine/provisions/auto-inject/auto-inject_init_vm_config.yml +++ b/utils/build/virtual_machine/provisions/auto-inject/auto-inject_init_vm_config.yml @@ -25,17 +25,26 @@ echo "After stop updates. System service apt-daily.service status" sudo systemctl list-units --all apt-daily.service || true + #There are some old machines that need to change the repositories because there are not available anymore lsb_release=/etc/lsb-release + must_update_repositories="false" if [ -e "$lsb_release" ]; then if grep -q 'Ubuntu 21.04' "$lsb_release"; then - echo "Configuring ubuntu 21.04 repositories" - sudo sed -i -r 's/ports.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list - sudo sed -i -r 's/ubuntu-ports/ubuntu/g' /etc/apt/sources.list - sudo sed -i -r 's/us-east-1.ec2.//g' /etc/apt/sources.list - echo 'apt_preserve_sources_list: true' | sudo tee -a /etc/cloud/cloud.cfg - sudo apt-get update + must_update_repositories="true" + elif grep -q 'Ubuntu 23.10' "$lsb_release"; then + #Why this works for arm machine but not for amd64? + must_update_repositories="true" fi fi + if [ "$must_update_repositories" == "true" ]; then + echo "Configuring archive ubuntu repositories" + sudo sed -i -r 's/ports.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list + sudo sed -i -r 's/ubuntu-ports/ubuntu/g' /etc/apt/sources.list + sudo sed -i -r 's/us-east-1.ec2.//g' /etc/apt/sources.list + echo 'apt_preserve_sources_list: true' | sudo tee -a /etc/cloud/cloud.cfg + sudo apt-get update + fi + sudo apt-get -y update #Install some basic tools (microvm doesn't have them by default) diff --git a/utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs.yml b/utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs.yml index b619a2e3a4..3e58935fb7 100644 --- a/utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs.yml +++ b/utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs.yml @@ -5,7 +5,7 @@ lang_variant: install: - os_type: linux os_distro: deb - remote-command: sudo apt-get install -y nodejs + remote-command: sudo apt-get update && sudo apt-get install -y nodejs - os_type: linux os_distro: rpm @@ -47,9 +47,7 @@ weblog: #centos 7: node 16 #rhel_7_amd64: I can't isntall in a standard way #amazon_linux2: Using the amazon-linux-extras - Error: Package: 1:nodejs-16.20.2-1.el7.x86_64 (epel) - #ubuntu 23.04 arm. nodejs package not found excluded_os_branches: [ubuntu22_amd64, ubuntu22_arm64, ubuntu21, ubuntu20_arm64, ubuntu20_amd64, centos_7_amd64, rhel_7_amd64, amazon_linux2] - excluded_os_names: [Ubuntu_23_04_arm64] install: - os_type: linux