stages.yml

parameters:
## Build stages
# NB: all stages _must_ depend on `env`
- name: buildStageDeps
  displayName: "Build stage dependencies"
  type: object
  default:
  - env
  - prechecks
- name: macBuildStageDeps
  displayName: "macOS stage dependencies"
  type: object
  default:
  - env
  - prechecks
- name: windowsBuildStageDeps
  displayName: "Windows stage dependencies"
  type: object
  default:
  - env
  - prechecks
- name: checkStageDeps
  displayName: "Check stage dependencies"
  type: object
  default:
  - env
  - linux_x64
- name: checksConcurrency
  displayName: "Check concurrency"
  type: number
  default: 3
- name: concurrencyPrechecks
  displayName: "Prechhecks concurrency"
  type: number
  default: 10


## Timeouts
- name: timeoutDockerPublish
  displayName: "Timout Docker publish"
  type: number
  default: 10

## Build settings
- name: cacheTestResults
  displayName: "Cache test results"
  type: boolean
  default: true

stages:
- stage: env
  displayName: Environment
  jobs:
  - template: env.yml

- stage: prechecks
  displayName: Prechecks
  dependsOn: ["env"]
  jobs:
  - job: prechecks
    displayName: Precheck
    timeoutInMinutes: 20
    pool:
      vmImage: "ubuntu-20.04"
    variables:
      AUTH_GITHUB: ""
      CI_TARGET: ""
      CHECK_DEPS: $[eq(stageDependencies.env.repo.outputs['changed.requirements'], true)]
    strategy:
      maxParallel: ${{ parameters.concurrencyPrechecks }}
      matrix:
        format:
          CI_TARGET: "format"
        protobuf:
          CI_TARGET: "check_and_fix_proto_format"
        ${{ if eq(variables.CHECK_DEPS, true) }}:
          dependencies:
            CI_TARGET: deps
            AUTH_GITHUB: $(GitHubPublicRepoOnlyAccessToken)
        publishing:
          CI_TARGET: docs
    steps:
    - template: bazel.yml
      parameters:
        ciTarget: $(CI_TARGET)
        cacheTestResults: ${{ parameters.cacheTestResults }}
        cacheVersion: $(cacheKeyBazel)
        publishEnvoy: false
        publishTestResults: false
        authGithub: $(AUTH_GITHUB)
        stepsPost:

        # Format fixes
        - task: PublishBuildArtifacts@1
          inputs:
            pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff"
            artifactName: format
          timeoutInMinutes: 10
          # not all have fixes so improve condition/handling
          condition: and(failed(), eq(variables['CI_TARGET'], 'format'))
        # Protobuf fixes
        - task: PublishBuildArtifacts@1
          inputs:
            pathtoPublish: "$(Build.StagingDirectory)/fix_proto_format.diff"
            artifactName: format
          timeoutInMinutes: 10
          condition: and(failed(), eq(variables['CI_TARGET'], 'protobuf'))

        # Publish docs
        - script: |
            ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
          displayName: "Upload Docs to GCS"
          env:
            ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
            GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
            GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
          condition: eq(variables['CI_TARGET'], 'docs')

        # Docker regression tests
        - script: |
            DOCKER_CI_FIX_DIFF=$(Build.StagingDirectory)/fix_docker.diff DOCKER_CI_FIX=1 ci/test_docker_ci.sh
          workingDirectory: $(Build.SourcesDirectory)
          displayName: Docker build regression test
          condition: eq(variables['CI_TARGET'], 'docs')
        - task: PublishBuildArtifacts@1
          inputs:
            pathtoPublish: "$(Build.StagingDirectory)/fix_docker.diff"
            artifactName: "docker_ci"
          timeoutInMinutes: 10
          condition: and(failed(), eq(variables['CI_TARGET'], 'docs'))

  - job: prechecked
    displayName: Prechecked
    dependsOn: ["prechecks"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        eq(variables['Build.Reason'], 'PullRequest'),
        in(dependencies.prechecks.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "Prechecked"

- stage: sync
  condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
  dependsOn: []
  jobs:
  - job: filter_example
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - checkout: self
      fetchDepth: 0
      fetchTags: true
    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="
        sshPublicKey: "$(FilterExamplePublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(FilterExamplePrivateKey)"

    - bash: ci/filter_example_mirror.sh
      displayName: "Sync envoy-filter-example"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)

  - job: data_plane_api
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - checkout: self
      fetchDepth: 0
      fetchTags: true
    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="
        sshPublicKey: "$(DataPlaneApiPublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(DataPlaneApiPrivateKey)"

    - bash: ci/api_mirror.sh
      displayName: "Sync data-plane-api"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)

  - job: go_control_plane
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - checkout: self
      fetchDepth: 0
      fetchTags: true
    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="
        sshPublicKey: "$(GoControlPlanePublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(GoControlPlanePrivateKey)"

    - bash: |
        cp -a ~/.ssh $(Build.StagingDirectory)/
        ci/run_envoy_docker.sh 'ci/go_mirror.sh'
      displayName: "Sync go-control-plane"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        AZP_BRANCH: $(Build.SourceBranch)

- stage: linux_x64
  displayName: Linux x64
  dependsOn: ${{ parameters.buildStageDeps }}
  # For post-submit builds, continue even if precheck fails
  condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
  jobs:
  - job: release
    displayName: Build and test
    condition: and(not(canceled()), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    timeoutInMinutes: 120
    pool: envoy-x64-large
    steps:
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.release
        bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
        cacheTestResults: ${{ parameters.cacheTestResults }}
        cacheVersion: $(cacheKeyBazel)

  - job: released
    displayName: Complete
    dependsOn: ["release"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: and(eq(variables['Build.Reason'], 'PullRequest'), in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "linux_x64 released"

- stage: linux_arm64
  displayName: Linux arm64
  dependsOn: ${{ parameters.buildStageDeps }}
  # For post-submit builds, continue even if precheck fails
  condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
  jobs:
  - job: release
    displayName: Build and test
    condition: and(not(canceled()), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    timeoutInMinutes: 180
    pool: "envoy-arm-large"
    steps:
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.release
        rbe: false
        artifactSuffix: ".arm64"
        bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
        cacheTestResults: ${{ parameters.cacheTestResults }}

  - job: released
    displayName: Complete
    dependsOn: ["release"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: and(eq(variables['Build.Reason'], 'PullRequest'), in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "linux_arm64 released"

- stage: check
  displayName: Checks (Linux x64)
  dependsOn: ${{ parameters.checkStageDeps }}
  jobs:
  - job: bazel
    displayName: "Linux x64"
    dependsOn: []
    # Skip checks if only mobile/ or docs/ have changed.
    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    variables:
      REPO_FETCH_DEPTH: 1
      REPO_FETCH_TAGS: false
      PUBLISH_TEST_RESULTS: true
      PUBLISH_ENVOY: true
    strategy:
      maxParallel: ${{ parameters.checksConcurrency }}
      matrix:
        # These are ordered by most time-consuming first.
        compile_time_options:
          CI_TARGET: "bazel.compile_time_options"
          ENVOY_FILTER_EXAMPLE: true
        tsan:
          CI_TARGET: "bazel.tsan"
        asan:
          CI_TARGET: "bazel.asan"
          ENVOY_FILTER_EXAMPLE: true
        # Disabled due to https://github.com/envoyproxy/envoy/pull/18218
        # api_compat:
        #  CI_TARGET: "bazel.api_compat"
        gcc:
          CI_TARGET: "bazel.gcc"
        msan:
          CI_TARGET: "bazel.msan"
          ENVOY_FILTER_EXAMPLE: true
        clang_tidy:
          CI_TARGET: "bazel.clang_tidy"
          REPO_FETCH_DEPTH: 0
          REPO_FETCH_TAGS: true
          PUBLISH_TEST_RESULTS: false
          PUBLISH_ENVOY: false
        api:
          CI_TARGET: "bazel.api"
    timeoutInMinutes: 180
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - template: bazel.yml
      parameters:
        ciTarget: $(CI_TARGET)
        envoyBuildFilterExample: $(ENVOY_FILTER_EXAMPLE)
        cacheTestResults: ${{ parameters.cacheTestResults }}
        repoFetchDepth: $(REPO_FETCH_DEPTH)
        repoFetchTags: $(REPO_FETCH_TAGS)
        publishTestResults: $(PUBLISH_TEST_RESULTS)
        publishEnvoy: $(PUBLISH_ENVOY)
        stepsPost:

        # TODO(phlax): consolidate "fix" paths/jobs
        - task: PublishBuildArtifacts@1
          inputs:
            pathtoPublish: "$(Build.StagingDirectory)/tmp/lint-fixes"
            artifactName: "$(CI_TARGET).fixes"
          timeoutInMinutes: 10
          condition: and(failed(), eq(variables['CI_TARGET'], 'bazel.clang_tidy'))

  - job: coverage
    displayName: "Linux x64"
    dependsOn: []
    # Skip checks if only mobile/ or docs/ have changed.
    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 240
    pool: "envoy-x64-large"
    strategy:
      maxParallel: 2
      matrix:
        coverage:
          CI_TARGET: "coverage"
        fuzz_coverage:
          CI_TARGET: "fuzz_coverage"
    steps:
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.$(CI_TARGET)
        rbe: false
        # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
        bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"
        cacheTestResults: ${{ parameters.cacheTestResults }}

    - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)'
      displayName: "Upload $(CI_TARGET) Report to GCS"
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
      condition: always()

  - job: complete
    displayName: "Checks complete"
    dependsOn: ["bazel", "coverage"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) check passes if all of
    # the preceding checks either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        eq(variables['Build.Reason'], 'PullRequest'),
        in(dependencies.bazel.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.coverage.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "checks complete"

- stage: publish
  displayName: Publish
  dependsOn: ["env", "linux_x64", "linux_arm64"]
  jobs:
  - job: docker
    displayName: "Docker (Linux multi arch)"
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - bash: |
        set -e

        # This is necessary as the azp version of Docker (~21.0) can hang when adding tarballs
        # in a build (https://github.com/moby/buildkit/issues/3801)

        # Update instructions lifted from https://docs.docker.com/engine/install/ubuntu/

        export DEBIAN_FRONTEND=noninteractive

        # This is almost certainly not necessary but the canonical way
        sudo apt-get -qq remove -y docker docker-engine docker.io containerd runc

        sudo install -m 0755 -d /etc/apt/keyrings
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
        sudo chmod a+r /etc/apt/keyrings/docker.gpg

        echo \
            "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
            "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" \
                | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

        sudo apt-get -qq update
        sudo apt-get -qq install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

        echo "***************************************************************"
        docker version
        docker info
      displayName: Upgrade Docker to latest

    - template: cached.yml
      parameters:
        name: publish_docker
        # VERSION.txt is included to refresh Docker images for release
        key: "ci/Dockerfile-envoy | VERSION.txt"
        version: "$(cacheKeyDockerBuild)"
        path: ""
    - bash: |
        echo "disk space at beginning of build:"
        df -h
      displayName: "Check disk space at beginning"
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/bin/*"
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/bin/*"
        targetPath: $(Build.StagingDirectory)
    - bash: |
        set -e

        mkdir -p linux/amd64 linux/arm64

        # x64
        cp -a $(Build.StagingDirectory)/bazel.release/bin/release.tar.zst linux/amd64/release.tar.zst
        cp -a $(Build.StagingDirectory)/bazel.release/bin/schema_validator_tool linux/amd64/schema_validator_tool

        # arm64
        cp -a $(Build.StagingDirectory)/bazel.release.arm64/bin/release.tar.zst linux/arm64/release.tar.zst
        cp -a $(Build.StagingDirectory)/bazel.release.arm64/bin/schema_validator_tool linux/arm64/schema_validator_tool

        # Debug what files appear to have been downloaded
        find linux -type f -name "*" | xargs ls -l

        ci/docker_ci.sh
      timeoutInMinutes: ${{ parameters.timeoutDockerPublish }}
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        AZP_SHA1: $(Build.SourceVersion)
        DOCKERHUB_USERNAME: $(DockerUsername)
        DOCKERHUB_PASSWORD: $(DockerPassword)
    - bash: |
        echo "disk space at end of build:"
        df -h
      displayName: "Check disk space at end"
      condition: always()
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/build_images"
        artifactName: docker
      timeoutInMinutes: 10
      condition: always()
    - script: sudo .azure-pipelines/docker/save_cache.sh /mnt/docker_cache
      displayName: "Cache/save (publish_docker)"

  - job: package_x64
    displayName: Linux debs (x64)
    dependsOn: []
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/bin/*"
        targetPath: $(Build.StagingDirectory)
    - template: bazel.yml
      parameters:
        ciTarget: bazel.distribution
  - job: package_arm64
    displayName: Linux debs (arm64)
    dependsOn: []
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool: "envoy-arm-large"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/bin/*"
        targetPath: $(Build.StagingDirectory)
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.distribution
        rbe: false
        artifactSuffix: ".arm64"
        bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"

  - job: success
    dependsOn: ["docker", "package_x64", "package_arm64"]
    displayName: Success (linux artefacts)
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) check passes if all of
    # the preceding checks either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.package_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.package_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "building publishable assets complete"

  - job: docs
    displayName: Publish docs to netlify
    dependsOn: ["success"]
    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: Cache@2
      inputs:
        key: 'docs | $(cacheKeyBazelFiles)'
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true

    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)"
        AZP_BRANCH: $(Build.SourceBranch)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Generate docs"

    - script: |
        bazel run //tools/distribution:update_dockerhub_repository
      displayName: "Publish Dockerhub description and README"
      env:
        DOCKERHUB_USERNAME: $(DockerUsername)
        DOCKERHUB_PASSWORD: $(DockerPassword)
      condition: and(eq(variables['isMain'], 'true'), eq(variables['PostSubmit'], true))

    - script: |
        ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
      displayName: "Upload Docs to GCS"
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
      condition: eq(variables['isMain'], 'true')

    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="
        sshPublicKey: "$(DocsPublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(DocsPrivateKey)"

    - script: docs/publish.sh
      displayName: "Publish to GitHub"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        NETLIFY_TRIGGER_URL: $(NetlifyTriggerURL)

  - job: github
    displayName: Publish release tag
    dependsOn: ["docs"]
    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true), eq(variables['isStableBranch'], 'true'), eq(stageDependencies.env.repo.outputs['state.isDev'], 'false'))
    pool:
      vmImage: "ubuntu-20.04"
    steps:

    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/bin/*"
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/bin/*"
        targetPath: $(Build.StagingDirectory)

    - task: DownloadSecureFile@1
      name: MaintainerGPGKey
      displayName: 'Download maintainer GPG key'
      inputs:
        # NB: This is the path to the key and **must** be set on a per-pipeline basis
        secureFile: '$(maintainerGPGKey)'

    - script: ./ci/run_envoy_docker.sh './ci/do_ci.sh publish'
      displayName: "Publish release to Github"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)

    # TODO(phlax): combine this with publish step
    - bash: |
        set -e

        VERSION="$(cat VERSION.txt)"

        mkdir -p linux/amd64 linux/arm64 publish

        # linux/amd64
        tar xf $(Build.StagingDirectory)/bazel.release/release.tar.zst -C ./linux/amd64
        cp -a linux/amd64/envoy "publish/envoy-${VERSION}-linux-x86_64"
        cp -a linux/amd64/envoy-contrib "publish/envoy-contrib-${VERSION}-linux-x86_64"

        # linux/arm64
        tar xf $(Build.StagingDirectory)/bazel.release.arm64/release.tar.zst -C ./linux/arm64
        cp -a linux/arm64/envoy "publish/envoy-${VERSION}-linux-aarch_64"
        cp -a linux/arm64/envoy-contrib "publish/envoy-contrib-${VERSION}-linux-aarch_64"

        echo "$MAINTAINER_GPG_KEY_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --import "$(MaintainerGPGKey.secureFilePath)"

        ci/publish_github_assets.sh "v${VERSION}" "${PWD}/publish"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)
        MAINTAINER_GPG_KEY_PASSPHRASE: $(MaintainerGPGKeyPassphrase)

- stage: verify
  displayName: Verify
  dependsOn: ["env", "publish"]
  jobs:
  - job: examples
    displayName: Examples (Docker/x64)
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - bash: .azure-pipelines/cleanup.sh
      displayName: "Removing tools from agent"
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "docker"
        itemPattern: "docker/envoy*.tar"
        targetPath: $(Build.StagingDirectory)
    - bash: ./ci/do_ci.sh verify_examples
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        NO_BUILD_SETUP: 1
        # ENVOY_EXAMPLES_DEBUG: 1

  - job: packages_x64
    displayName: Debs (x64)
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.distribution"
        itemPattern: "bazel.distribution/packages.x64.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        ENVOY_DOCKER_IN_DOCKER: 1
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Verify packages"
  - job: packages_arm64
    displayName: Debs (arm64)
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool: "envoy-arm-large"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.distribution.arm64"
        itemPattern: "bazel.distribution.arm64/packages.arm64.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        ENVOY_DOCKER_IN_DOCKER: 1
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Verify packages"

  - job: verified
    displayName: Verification complete
    dependsOn: ["examples", "packages_x64", "packages_arm64"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) check passes if all of
    # the preceding checks either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        eq(variables['Build.Reason'], 'PullRequest'),
        in(dependencies.examples.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.packages_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.packages_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "checks complete"

- stage: macos
  displayName: macOS
  dependsOn: ${{ parameters.macBuildStageDeps }}
  jobs:
  - job: test
    displayName: Build and test
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 180
    pool:
      vmImage: "macos-11"
    steps:
    - script: ./ci/mac_ci_setup.sh
      displayName: "Install dependencies"

    - script: ./ci/mac_ci_steps.sh
      displayName: "Run Mac CI"
      env:
        BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2"
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

    - task: PublishTestResults@2
      inputs:
        testResultsFiles: "**/bazel-testlogs/**/test.xml"
        testRunTitle: "macOS"
      timeoutInMinutes: 10
      condition: always()

  - job: tested
    displayName: Complete
    dependsOn: ["test"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: and(eq(variables['Build.Reason'], 'PullRequest'), in(dependencies.test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "macos tested"

- stage: windows
  displayName: Windows
  dependsOn: ${{ parameters.windowsBuildStageDeps }}
  jobs:
  - job: release
    displayName: Build and test
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 180
    pool:
      vmImage: "windows-2019"
    steps:
    - task: Cache@2
      inputs:
        key: '"windows.release" | $(cacheKeyBazelFiles)'
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true
    - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh
      displayName: "Run Windows msvc-cl CI"
      env:
        CI_TARGET: "windows"
        ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)"
        SLACK_TOKEN: $(SLACK_TOKEN)
        REPO_URI: $(Build.Repository.Uri)
        BUILD_URI: $(Build.BuildUri)
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2"
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

    - task: PublishTestResults@2
      inputs:
        testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml"
        testRunTitle: "windows"
        searchFolder: $(Build.StagingDirectory)/tmp
      timeoutInMinutes: 10
      condition: always()
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/envoy"
        artifactName: windows.release
      timeoutInMinutes: 10
      condition: always()

  - job: docker
    displayName: Build Docker image
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    strategy:
      matrix:
        windows2019:
          imageName: 'windows-2019'
          windowsBuildType: "windows"
          windowsImageBase: "mcr.microsoft.com/windows/servercore"
          windowsImageTag: "ltsc2019"
        windows2022:
          imageName: 'windows-2022'
          windowsBuildType: "windows-ltsc2022"
          windowsImageBase: "mcr.microsoft.com/windows/nanoserver"
          windowsImageTag: "ltsc2022"
    dependsOn: ["release"]
    timeoutInMinutes: 120
    pool:
      vmImage: $(imageName)
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "windows.release"
        itemPattern: "windows.release/envoy_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - bash: |
        set -e
        # Convert to Unix-style path so tar doesn't think drive letter is a hostname
        STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')"
        mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64
        ci/docker_ci.sh
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        AZP_SHA1: $(Build.SourceVersion)
        DOCKERHUB_USERNAME: $(DockerUsername)
        DOCKERHUB_PASSWORD: $(DockerPassword)
        WINDOWS_BUILD_TYPE: $(windowsBuildType)
        WINDOWS_IMAGE_BASE: $(windowsImageBase)
        WINDOWS_IMAGE_TAG: $(windowsImageTag)
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/build_images"
        artifactName: docker_windows
      timeoutInMinutes: 10
      condition: always()

  - job: released
    displayName: Complete
    dependsOn: ["release", "docker"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        eq(variables['Build.Reason'], 'PullRequest'),
        in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "windows released"