Try fix smoke tests (#5889)

## Summary of changes - Install v1 docker-compose in the hosted runners used for Windows smoke tests - Cleanup the docker networks in the smoke tests ## Reason for change Microsoft [recently](https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/pipelines/sprint-240-update#dockercompose0-uses-docker-compose-v2-in-v1-compatibility-mode) stopped shipping the v1 of docker compose. That broke all our Windows smoke tests. Unfortunately, for various other reasons, we couldn't just convert to the v2 format, due to how we're reusing various bits and pieces. Additionally, we started getting this on our linux smoke tests: ``` Creating network "ddtrace_20240812_56_default" with the default driver could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network ``` This is basically because we're creating networks, and never cleaning them up, and for efficiency reasons we're not destroying the smoke test VMs after each run. ## Implementation details - Install Docker-compose v1 in the Windows VMs. A bit hacky, but the quickest solution right now - Clean up the docker network after running the snapshot tests ## Test coverage I did some manual tests to make sure this all works as expected. Also killed and restored the smoke tests machines which solved the docker network problem temporarily. ## Other details This is obviously still all kinda horrible. I'd love to replace all this yaml with just testcontainers. One day.... _stares wistfully into the distance_
DataDog · Aug 13, 2024 · 8318f76 · 8318f76
1 parent 496d420
commit 8318f76
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 20 deletions.
diff --git a/.azure-pipelines/steps/install-docker-compose-v1.yml b/.azure-pipelines/steps/install-docker-compose-v1.yml
@@ -0,0 +1,24 @@
+parameters:
+ - name: isLinux
+ type: boolean
+ default: false
+
+ - name: 'dockerComposePath'
+ type: string
+ default: 'C:/docker-compose/docker-compose.exe'
+
+steps:
+- ${{ if eq(parameters.isLinux, true) }}:
+ - bash: |
+ sudo mkdir -p "$(dirname "${{ parameters.dockerComposePath }}")" 
+ sudo curl -SL https://github.com/docker/compose/releases/download/1.29.2/docker-compose-linux-x86_64 -o ${{ parameters.dockerComposePath }}
+ sudo chmod 755 ${{ parameters.dockerComposePath }}
+ displayName: Download docker-compose
+- ${{ else }}:
+ - powershell: |
+ $dir= (Split-Path -parent "${{ parameters.dockerComposePath }}")
+ mkdir -f -p $dir
+ # GitHub now requires TLS1.2. In PowerShell, run the following
+ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+ Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-windows-x86_64.exe" -OutFile "${{ parameters.dockerComposePath }}"
+ displayName: Download docker-compose
diff --git a/.azure-pipelines/steps/run-snapshot-test.yml b/.azure-pipelines/steps/run-snapshot-test.yml
@@ -16,6 +16,10 @@ parameters:
  - name: 'apiKey'
  type: string
  default: ''
+
+ - name: 'dockerComposePath'
+ type: string
+ default: 'docker-compose'
 
 steps:
 - template: ./clean-docker-containers.yml
@@ -60,7 +64,7 @@ steps:
  displayName: Set env-specific variables
 
 - bash: |
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run --rm $(START_TEST_AGENT_TARGET)
  env:
  dockerTag: $(dockerTag)
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -69,35 +73,35 @@ steps:
 
 - script: |
  echo "Starting snapshot session"
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(START_ENDPOINT)"
  displayName: start snapshot session
  env:
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 
 - bash: |
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e PROFILER_IS_NOT_REQUIRED=${{ parameters.isNoop }} ${{ parameters.target }}
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e PROFILER_IS_NOT_REQUIRED=${{ parameters.isNoop }} ${{ parameters.target }}
  env:
  dockerTag: $(dockerTag)
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
- displayName: docker-compose run ${{ parameters.target }}
+ displayName: ${{ parameters.dockerComposePath }} run ${{ parameters.target }}
 
 - script: |
  echo "Dumping traces"
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_traces.json "http://localhost:8126$(TRACE_DUMP_ENDPOINT)"
  
  echo "Dumping stats"
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_stats.json "http://localhost:8126$(STATS_DUMP_ENDPOINT)"
  
  echo "Dumping all requests"
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) -o /debug_snapshots/${{ parameters.snapshotPrefix }}_requests.json "http://localhost:8126$(REQUESTS_DUMP_ENDPOINT)"
  displayName: dump snapshots
  env:
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 
 - ${{ if and(eq(parameters.isLinux, true), eq(parameters.isNoop, false)) }}:
  - bash: |
  echo "Verifying snapshot session (fail on mis-match)"
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --w '\nGetting a 400 means there is a diff in snapshots. You can diff the files with the artifacts generated. You can also run the tests locally. Follow the doc in /docs/development/CI/RunSmokeTestsLocally\n' --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
  displayName: check snapshots
  env:
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -120,19 +124,19 @@ steps:
 - ${{ else }}:
  - bash: |
  echo "Verifying snapshot session (fail on mis-match)"
- docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
+ ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
  displayName: check snapshots
  env:
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
 
-- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
+- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) logs $(TEST_AGENT_TARGET)
  displayName: dump docker-compose logs for $(TEST_AGENT_TARGET)
  env:
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
  condition: succeededOrFailed()
  continueOnError: true
 
-- script: docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
+- script: ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) down
  displayName: docker-compose stop services
  env:
  DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}
@@ -142,7 +146,7 @@ steps:
 # Run crash tests
 - ${{ if and(eq(parameters.isLinux, true), eq(parameters.isNoop, false)) }}:
  - bash: |
- LOGS=$(docker-compose -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
+ LOGS=$(${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) run -e dockerTag=$(dockerTag) -e CRASH_APP_ON_STARTUP=1 -e COMPlus_DbgEnableMiniDump=0 ${{ parameters.target }})
  echo $LOGS
 
  # check logs for evidence of crash detection in the output
@@ -164,4 +168,12 @@ steps:
  - script: |
  sudo chmod -R 644 tracer/build_data/dumps/* || true
  displayName: Make dumps uploadable to AzDo
- condition: succeededOrFailed()
+ condition: succeededOrFailed()
+
+- script: |
+ docker network prune -f
+ displayName: Clean up docker networks
+ condition: succeededOrFailed()
+ continueOnError: true
+
+
diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml
@@ -5687,6 +5687,7 @@ stages:
  variables:
  targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
  targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+ dockerComposePath: 'C:/docker-compose/docker-compose.exe'
  jobs:
  - template: steps/update-github-status-jobs.yml
  parameters:
@@ -5702,6 +5703,10 @@ stages:
  vmImage: windows-2022
 
  steps:
+ - template: steps/install-docker-compose-v1.yml
+ parameters:
+ dockerComposePath: $(dockerComposePath)
+
  - template: steps/clone-repo.yml
  parameters:
  targetShaId: $(targetShaId)
@@ -5725,7 +5730,7 @@ stages:
  displayName: create test data directories
 
  - bash: |
- docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+ $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
  --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
  --build-arg RUNTIME_IMAGE=$(runtimeImage) \
  --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5742,9 +5747,10 @@ stages:
  target: 'nuget-smoke-tests.windows'
  snapshotPrefix: "smoke_test"
  isLinux: false
+ dockerComposePath: $(dockerComposePath)
 
  - bash: |
- docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+ $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
  --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
  --build-arg RUNTIME_IMAGE=$(runtimeImage) \
  --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5761,6 +5767,7 @@ stages:
  target: 'nuget-dddotnet-smoke-tests.windows'
  snapshotPrefix: "smoke_test"
  isLinux: false
+ dockerComposePath: $(dockerComposePath)
 
  - publish: tracer/build_data
  artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5777,6 +5784,7 @@ stages:
  variables:
  targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
  targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+ dockerComposePath: 'C:/docker-compose/docker-compose.exe'
  jobs:
  - template: steps/update-github-status-jobs.yml
  parameters:
@@ -5792,6 +5800,10 @@ stages:
  vmImage: windows-2022
 
  steps:
+ - template: steps/install-docker-compose-v1.yml
+ parameters:
+ dockerComposePath: $(dockerComposePath)
+
  - template: steps/clone-repo.yml
  parameters:
  targetShaId: $(targetShaId)
@@ -5812,7 +5824,7 @@ stages:
  displayName: Create test data directories
 
  - bash: |
- docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+ $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
  --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
  --build-arg RUNTIME_IMAGE=$(runtimeImage) \
  --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5828,6 +5840,7 @@ stages:
  target: 'dotnet-tool-smoke-tests.windows'
  snapshotPrefix: "smoke_test"
  isLinux: false
+ dockerComposePath: $(dockerComposePath)
 
  - publish: tracer/build_data
  artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5844,6 +5857,7 @@ stages:
  variables:
  targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
  targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+ dockerComposePath: 'C:/docker-compose/docker-compose.exe'
  jobs:
  - template: steps/update-github-status-jobs.yml
  parameters:
@@ -5859,6 +5873,10 @@ stages:
  vmImage: windows-2022
 
  steps:
+ - template: steps/install-docker-compose-v1.yml
+ parameters:
+ dockerComposePath: $(dockerComposePath)
+
  - template: steps/clone-repo.yml
  parameters:
  targetShaId: $(targetShaId)
@@ -5879,7 +5897,7 @@ stages:
  displayName: Create test data directories
 
  - bash: |
- docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+ $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
  --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
  --build-arg RUNTIME_IMAGE=$(runtimeImage) \
  --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5895,6 +5913,7 @@ stages:
  target: 'smoke-tests.windows'
  snapshotPrefix: "smoke_test"
  isLinux: false
+ dockerComposePath: $(dockerComposePath)
 
  - publish: tracer/build_data
  artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5911,6 +5930,7 @@ stages:
  variables:
  targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
  targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+ dockerComposePath: 'C:/docker-compose/docker-compose.exe'
  jobs:
  - template: steps/update-github-status-jobs.yml
  parameters:
@@ -5927,6 +5947,10 @@ stages:
  vmImage: windows-2022
 
  steps:
+ - template: steps/install-docker-compose-v1.yml
+ parameters:
+ dockerComposePath: $(dockerComposePath)
+
  - template: steps/clone-repo.yml
  parameters:
  targetShaId: $(targetShaId)
@@ -5947,7 +5971,7 @@ stages:
  displayName: Create test data directories
 
  - bash: |
- docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+ $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
  --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
  --build-arg RUNTIME_IMAGE=$(runtimeImage) \
  --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -5963,6 +5987,7 @@ stages:
  target: 'dd-dotnet-smoke-tests.windows'
  snapshotPrefix: "smoke_test"
  isLinux: false
+ dockerComposePath: $(dockerComposePath)
 
  - publish: tracer/build_data
  artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
@@ -5979,6 +6004,7 @@ stages:
  variables:
  targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
  targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+ dockerComposePath: 'C:/docker-compose/docker-compose.exe'
  jobs:
  - template: steps/update-github-status-jobs.yml
  parameters:
@@ -5991,9 +6017,13 @@ stages:
  variables:
  smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest"
  pool:
- vmImage: windows-2022
+ name: azure-windows-scale-set
 
  steps:
+ - template: steps/install-docker-compose-v1.yml
+ parameters:
+ dockerComposePath: $(dockerComposePath)
+
  - template: steps/clone-repo.yml
  parameters:
  targetShaId: $(targetShaId)
@@ -6012,7 +6042,7 @@ stages:
  path: $(smokeTestAppDir)/artifacts
 
  - bash: |
- docker-compose -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+ $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
  --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
  --build-arg RUNTIME_IMAGE=$(runtimeImage) \
  --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
@@ -6029,6 +6059,7 @@ stages:
  target: 'tracer-home-smoke-tests.windows'
  snapshotPrefix: "smoke_test"
  isLinux: false
+ dockerComposePath: $(dockerComposePath)
 
  - publish: tracer/build_data
  artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)