Skip to content

Merge branch 'main' into mtoff/query-string-consistency #5

Merge branch 'main' into mtoff/query-string-consistency

Merge branch 'main' into mtoff/query-string-consistency #5

Workflow file for this run

name: End-to-end tests
on:
workflow_call:
inputs:
library:
description: "Library to test"
required: true
type: string
scenarios:
description: "JSON array of scenarios to run"
default: "[]"
required: false
type: string
weblogs:
description: "JSON array of weblogs to run"
default: "[]"
required: false
type: string
binaries_artifact:
description: "Artifact name containing the binaries to test"
default: ''
required: false
type: string
ci_environment:
description: "Which CI environment is running the tests, used for FPD"
default: 'custom'
required: false
type: string
build_python_base_images:
description: "Shall we build python base images for tests on python tracer"
default: false
required: false
type: boolean
build_buddies_images:
description: "Shall we build buddies images"
default: false
required: false
type: boolean
build_proxy_image:
description: "Shall we build proxy image"
default: false
required: false
type: boolean
push_to_feature_parity_dashbaord:
description: "Shall we push results to Feature Parity Dashbaord"
default: false
required: false
type: boolean
env:
REGISTRY: ghcr.io
jobs:
end-to-end:
runs-on: ubuntu-latest
strategy:
matrix:
weblog: ${{ fromJson(inputs.weblogs) }}
fail-fast: false
env:
SYSTEM_TESTS_REPORT_ENVIRONMENT: ${{ inputs.ci_environment }}
SYSTEM_TESTS_REPORT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
repository: 'DataDog/system-tests'
- name: Install runner
uses: ./.github/actions/install_runner
- name: Get binaries artifact
if : ${{ inputs.binaries_artifact != '' }}
uses: actions/download-artifact@v4
with:
name: ${{ inputs.binaries_artifact }}
path: binaries/
- name: Build python's weblog base images
if: inputs.library == 'python' && inputs.build_python_base_images
run: |
./utils/build/build_python_base_images.sh
- name: Build buddies weblog images
if: inputs.build_buddies_images
run: ./utils/build/build_tracer_buddies.sh
- name: Build proxy image
if: inputs.build_proxy_image
run: ./build.sh -i proxy
- name: Pull images
uses: ./.github/actions/pull_images
with:
library: ${{ inputs.library }}
weblog: ${{ matrix.weblog }}
scenarios: ${{ inputs.scenarios }}
- name: Log in to the Container registry
if: ${{ inputs.library == 'ruby' }}
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
- name: Build agent
run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh -i agent
- name: Build weblog
id: build
run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog }}
- name: Run APPSEC_STANDALONE scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE"')
run: ./run.sh APPSEC_STANDALONE
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEFAULT scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEFAULT"')
run: ./run.sh DEFAULT
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run CROSSED_TRACING_LIBRARIES scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"CROSSED_TRACING_LIBRARIES"')
run: ./run.sh CROSSED_TRACING_LIBRARIES
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
- name: Run PROFILING scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"PROFILING"')
run: ./run.sh PROFILING
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TRACE_PROPAGATION_STYLE_W3C scenario
if: always() && steps.build.outcome == 'success' && inputs.library != 'python' && contains(inputs.scenarios, '"TRACE_PROPAGATION_STYLE_W3C"')
run: ./run.sh TRACE_PROPAGATION_STYLE_W3C
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run INTEGRATIONS scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"INTEGRATIONS"')
run: ./run.sh INTEGRATIONS
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
- name: Run APM_TRACING_E2E_OTEL scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_E2E_OTEL"')
run: ./run.sh APM_TRACING_E2E_OTEL
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
DD_APPLICATION_KEY: ${{ secrets.DD_APPLICATION_KEY }}
DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }}
- name: Run LIBRARY_CONF_CUSTOM_HEADER_TAGS scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"LIBRARY_CONF_CUSTOM_HEADER_TAGS"')
run: ./run.sh LIBRARY_CONF_CUSTOM_HEADER_TAGS
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID"')
run: ./run.sh LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TRACING_CONFIG_NONDEFAULT scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TRACING_CONFIG_NONDEFAULT"')
run: ./run.sh TRACING_CONFIG_NONDEFAULT
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TRACING_CONFIG_NONDEFAULT_2 scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TRACING_CONFIG_NONDEFAULT_2"')
run: ./run.sh TRACING_CONFIG_NONDEFAULT_2
env:

Check failure on line 164 in .github/workflows/run-end-to-end.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/run-end-to-end.yml

Invalid workflow file

You have an error in your yaml syntax on line 164
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TRACING_CONFIG_NONDEFAULT_3 scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TRACING_CONFIG_NONDEFAULT_3"')
run: ./run.sh TRACING_CONFIG_NONDEFAULT_3
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES"')
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING"')
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD"')
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE"')
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE"')
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE"')
run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_MISSING_RULES scenario
# C++ 1.2.0 freeze when the rules file is missing
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_MISSING_RULES"') && inputs.library != 'cpp'
run: ./run.sh APPSEC_MISSING_RULES
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_CUSTOM_RULES scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CUSTOM_RULES"')
run: ./run.sh APPSEC_CUSTOM_RULES
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_CORRUPTED_RULES scenario
# C++ 1.2.0 freeze when the rules file is missing
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CORRUPTED_RULES"') && inputs.library != 'cpp'
run: ./run.sh APPSEC_CORRUPTED_RULES
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_RULES_MONITORING_WITH_ERRORS scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_RULES_MONITORING_WITH_ERRORS"')
run: ./run.sh APPSEC_RULES_MONITORING_WITH_ERRORS
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_BLOCKING scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_BLOCKING"')
run: ./run.sh APPSEC_BLOCKING
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run EVERYTHING_DISABLED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"EVERYTHING_DISABLED"')
run: ./run.sh EVERYTHING_DISABLED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_LOW_WAF_TIMEOUT scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_LOW_WAF_TIMEOUT"')
run: ./run.sh APPSEC_LOW_WAF_TIMEOUT
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_CUSTOM_OBFUSCATION scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CUSTOM_OBFUSCATION"')
run: ./run.sh APPSEC_CUSTOM_OBFUSCATION
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_RATE_LIMITER scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_RATE_LIMITER"')
run: ./run.sh APPSEC_RATE_LIMITER
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_BLOCKING_FULL_DENYLIST scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_BLOCKING_FULL_DENYLIST"')
run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_REQUEST_BLOCKING scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_REQUEST_BLOCKING"')
run: ./run.sh APPSEC_REQUEST_BLOCKING
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_RUNTIME_ACTIVATION scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_RUNTIME_ACTIVATION"')
run: ./run.sh APPSEC_RUNTIME_ACTIVATION
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_WAF_TELEMETRY scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_WAF_TELEMETRY"')
run: ./run.sh APPSEC_WAF_TELEMETRY
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_API_SECURITY scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_API_SECURITY"')
run: ./run.sh APPSEC_API_SECURITY
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_API_SECURITY_RC scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_API_SECURITY_RC"')
run: ./run.sh APPSEC_API_SECURITY_RC
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_API_SECURITY_NO_RESPONSE_BODY scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_API_SECURITY_NO_RESPONSE_BODY"')
run: ./run.sh APPSEC_API_SECURITY_NO_RESPONSE_BODY
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_API_SECURITY_WITH_SAMPLING scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_API_SECURITY_WITH_SAMPLING"')
run: |
./run.sh APPSEC_API_SECURITY_WITH_SAMPLING
cat ./logs_appsec_api_security_with_sampling/tests.log 2>/dev/null | grep "API SECURITY" || true
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_AUTO_EVENTS_EXTENDED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_AUTO_EVENTS_EXTENDED"')
run: ./run.sh APPSEC_AUTO_EVENTS_EXTENDED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_AUTO_EVENTS_RC scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_AUTO_EVENTS_RC"')
run: ./run.sh APPSEC_AUTO_EVENTS_RC
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run APPSEC_RASP scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_RASP"')
run: ./run.sh APPSEC_RASP
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run SAMPLING scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"SAMPLING"')
run: ./run.sh SAMPLING
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TELEMETRY_APP_STARTED_PRODUCTS_DISABLED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TELEMETRY_APP_STARTED_PRODUCTS_DISABLED"')
run: ./run.sh TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TELEMETRY_LOG_GENERATION_DISABLED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TELEMETRY_LOG_GENERATION_DISABLED"')
run: ./run.sh TELEMETRY_LOG_GENERATION_DISABLED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TELEMETRY_METRIC_GENERATION_DISABLED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TELEMETRY_METRIC_GENERATION_DISABLED"')
run: ./run.sh TELEMETRY_METRIC_GENERATION_DISABLED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TELEMETRY_METRIC_GENERATION_ENABLED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TELEMETRY_METRIC_GENERATION_ENABLED"')
run: ./run.sh TELEMETRY_METRIC_GENERATION_ENABLED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED"')
run: ./run.sh TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_PROBES_STATUS scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_PROBES_STATUS"')
run: ./run.sh DEBUGGER_PROBES_STATUS
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_METHOD_PROBES_SNAPSHOT scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_METHOD_PROBES_SNAPSHOT"')
run: ./run.sh DEBUGGER_METHOD_PROBES_SNAPSHOT
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_LINE_PROBES_SNAPSHOT scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_LINE_PROBES_SNAPSHOT"')
run: ./run.sh DEBUGGER_LINE_PROBES_SNAPSHOT
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_MIX_LOG_PROBE scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_MIX_LOG_PROBE"')
run: ./run.sh DEBUGGER_MIX_LOG_PROBE
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_PII_REDACTION scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_PII_REDACTION"')
run: ./run.sh DEBUGGER_PII_REDACTION
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_EXPRESSION_LANGUAGE scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_EXPRESSION_LANGUAGE"')
run: ./run.sh DEBUGGER_EXPRESSION_LANGUAGE
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run DEBUGGER_EXCEPTION_REPLAY scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"DEBUGGER_EXCEPTION_REPLAY"')
run: ./run.sh DEBUGGER_EXCEPTION_REPLAY
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
- name: Run all scenarios in replay mode
run: utils/scripts/replay_scenarios.sh
env:
DD_API_KEY: ${{ secrets.DD_API_KEY }}
DD_APPLICATION_KEY: ${{ secrets.DD_APPLICATION_KEY }}
DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }}
- name: Compress logs
id: compress_logs
if: always() && steps.build.outcome == 'success'
run: tar -czvf artifact.tar.gz $(ls | grep logs)
- name: Upload artifact
if: always() && steps.compress_logs.outcome == 'success'
uses: actions/upload-artifact@v4
with:
# log name convention to respect : logs_$SCENARIO-FAMILY_$LIBRARY_$WEBLOG_$CI-ENVIRONMENT
name: logs_endtoend_${{ inputs.library }}_${{ matrix.weblog }}_${{ inputs.ci_environment }}
path: artifact.tar.gz
- name: Push results to Feature Parity Dashboard
if: ${{ inputs.push_to_feature_parity_dashbaord && false }} # disabled for now
run: |
for folder in logs*/ ; do
curl -X POST ${FP_IMPORT_URL} \
--fail
--header "Content-Type: application/json" \
--header "FP_API_KEY: ${FP_API_KEY}" \
--data "@./${folder}feature_parity.json" \
--include
done
env:
FP_API_KEY: ${{ secrets.FP_API_KEY }}
FP_IMPORT_URL: ${{ secrets.FP_IMPORT_URL }}