Update win-ci-pipeline.yml: enable xnnpack tests (#16244)

1. Enable xnnpack test 2. Change TSA database name from onnxruntime_master to onnxruntime_main. This is a leftover of renaming the "master" branch to "main" 3. Add two static analysis jobs for WinML and DML 4. Rename the machine pool "aiinfra-dml-winbuild" to "onnxruntime-Win2019-GPU-dml-A10", so that the internal and public ADO instances use the same machine pool name. 5. Move Windows GPU CI build pipeline from "onnxruntime-Win2022-GPU-T4" to "onnxruntime-Win2022-GPU-A10" machine pool, because we do not have enough T4 GPUs.
microsoft · Jun 15, 2023 · dbc7a19 · dbc7a19
1 parent 9be1332
commit dbc7a19
Show file tree

Hide file tree

Showing 14 changed files with 130 additions and 71 deletions.
diff --git a/.gdn/.gdntsa b/.gdn/.gdntsa
@@ -1,3 +1,3 @@
 {
-    "codebaseName": "onnxruntime_master"
+    "codebaseName": "onnxruntime_main"
 }
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -3,8 +3,8 @@ name: Lint
 on:
   push:
     branches:
-      - master
       - main
+      - rel-*
   pull_request:
 
 jobs:

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -2,10 +2,14 @@ name: Linux_CI
 on:
   push:
     branches:
-      - master
       - main
+      - rel-*
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   Onnxruntime-TVM:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/sca.yml b/.github/workflows/sca.yml
@@ -3,18 +3,19 @@ on:
   push:
     branches:
       - main
+      - rel-*
   pull_request:
 
 concurrency:
-  group: ${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 env:
   AZCOPY_AUTO_LOGIN_TYPE: MSI
   AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4
 
 jobs:
-  Onnxruntime-SCA:
+  Onnxruntime-SCA-training-CUDA:
     runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
     steps:
       - uses: actions/checkout@v3
@@ -33,6 +34,11 @@ jobs:
         run: azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" cuda_sdk
 
 
+      - name: Delete build folder
+        run: |
+          if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
+          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
+
       # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
       - name: Build code
         env:
@@ -49,3 +55,79 @@ jobs:
         with:
           sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
           category: VS_SCA
+
+  # No python
+  Onnxruntime-SCA-win32-WINML-x64:
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: false
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.11.x'
+          architecture: 'x64'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Delete build folder
+        run: |
+          if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
+          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
+
+      # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
+      - name: Build code
+        env:
+           CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake'
+        run:  python tools\ci_build\build.py --enable_training --build_java --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib
+
+      - name: Generate sarif
+        working-directory: D:\b
+        run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output
+
+      - name: Upload SARIF to GitHub
+        uses: github/codeql-action/upload-sarif@v2
+        continue-on-error: true
+        with:
+          sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
+          category: VS_SCA_WIN32_WINML_X64
+
+  # No java, No python
+  Onnxruntime-SCA-win32-WINML-x86:
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: false
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.11.x'
+          architecture: 'x86'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Delete build folder
+        run: |
+          if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
+          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x86 -install_prefix D:\b\Debug\installed -build_config Debug
+
+      # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
+      - name: Build code
+        env:
+           CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake'
+        run:  python tools\ci_build\build.py --enable_training --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib
+
+      - name: Generate sarif
+        working-directory: D:\b
+        run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output
+
+      - name: Upload SARIF to GitHub
+        uses: github/codeql-action/upload-sarif@v2
+        continue-on-error: true
+        with:
+          sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
+          category: VS_SCA_WIN32_WINML_X86
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -2,10 +2,14 @@ name: Windows_CI
 on:
   push:
     branches:
-      - master
       - main
+      - rel-*
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   Onnxruntime-TVM:
     runs-on: windows-latest

diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
@@ -794,7 +794,7 @@ stages:
 
 - template: nuget/templates/dml-vs-2022.yml
   parameters:
-    AgentPool : 'aiinfra-dml-winbuild'
+    AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
     IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
     ArtifactName: 'drop-nuget-dml'
     StageName: 'Windows_CI_GPU_DML_Dev'
@@ -817,7 +817,7 @@ stages:
 
 - template: nuget/templates/dml-vs-2022.yml
   parameters:
-    AgentPool : 'aiinfra-dml-winbuild'
+    AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
     IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
     ArtifactName: 'drop-win-dml-x86-zip'
     StageName: 'Windows_CI_GPU_DML_Dev_x86'
@@ -840,7 +840,7 @@ stages:
 
 - template: nuget/templates/dml-vs-2022.yml
   parameters:
-    AgentPool : 'aiinfra-dml-winbuild'
+    AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
     IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
     ArtifactName: 'drop-win-dml-arm64-zip'
     StageName: 'Windows_CI_GPU_DML_Dev_arm64'

diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml
@@ -48,7 +48,7 @@ stages:
       ALLOW_RELEASED_ONNX_OPSET_ONLY: ${{ parameters.AllowReleasedOpsetOnly }}
       ${{ if eq(parameters.EnableLto, true) }}:
         build_py_lto_flag: --enable_lto
-      ${{ if or(eq(parameters.AgentPool, 'aiinfra-dml-winbuild'), eq(parameters.AgentPool, 'onnxruntime-Win-CPU-2019')) }}:
+      ${{ if or(eq(parameters.AgentPool, 'onnxruntime-Win2019-GPU-dml-A10'), eq(parameters.AgentPool, 'onnxruntime-Win-CPU-2019')) }}:
         vsVersion: '2019'
 
     steps:
@@ -72,7 +72,13 @@ stages:
 
       - task: NodeTool@0
         inputs:
-          versionSpec: '16.x'
+          versionSpec: '18.x'
+
+      - task: onebranch.pipeline.tsaoptions@1
+        displayName: 'OneBranch TSAOptions'
+        inputs:
+          tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
+          appendSourceBranchName: false
 
       - task: UsePythonVersion@0
         inputs:
@@ -272,9 +278,6 @@ stages:
         - template: ../../templates/compliance.yml
           parameters :
             msbuildPlatform: ${{ parameters.sln_platform }}
-            # parameter couldn't be compared with string, so I have to use boolean.
-            ${{ if eq(variables.vsVersion, '2019') }}:
-              vs2022: false
 
       - template: ../../templates/component-governance-component-detection-steps.yml
         parameters :

diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml
@@ -4,11 +4,6 @@ parameters:
   type: string
   default: x64
 
-- name: vs2022
-  displayName: Visual Studio Version
-  type: string
-  default: true
-
 steps:
 - task: CredScan@2
   displayName: 'Run CredScan'
@@ -23,45 +18,10 @@ steps:
     arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.dll  --recurse --verbose'
   continueOnError: true
 
-- task: DeleteFiles@1
-  displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
-  inputs:
-    SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
-    Contents: |
-     **/*.obj
-     **/*.pdb
-     **/*.dll
-
-#Manually set msBuildCommandline so that we can also set CAExcludePath
-- task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2
-  displayName: 'Run the PREfast SDL Native Rules for MSBuild'
-  inputs:
-    userProvideBuildInfo: msBuildInfo
-    msBuildArchitecture: x64
-    ${{ if eq(parameters.vs2022, false)}}:
-      msBuildVersion: 16.0
-      msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
-    ${{ else }}:
-      msBuildVersion: 17.0
-      msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
-  continueOnError: true
-
-- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
-  displayName: 'Create Security Analysis Report'
-  inputs:
-    BinSkim: true
-    BinSkimBreakOn: WarningAbove
-    CredScan: true
-    SDLNativeRules: true
-
-- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
-  displayName: 'Publish Security Analysis Logs'
-  continueOnError: true
-
-- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1
+- task: TSAUpload@2
+  displayName: 'TSA upload'
   condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
-  displayName: 'TSA Upload'
   inputs:
-    tsaVersion: TsaV2
-    codeBaseName: 'onnxruntime_master'
+    GdnPublishTsaOnboard: false
+    GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
   continueOnError: true
diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
@@ -64,6 +64,7 @@ jobs:
     setVcvars: true
     ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
     DocUpdateNeeded: false  # Set to true during document generation if there are diffs
+    NVIDIA_TF32_OVERRIDE: '0'
     skipComponentGovernanceDetection: true
     DEPS_CACHE_DIR: $(Agent.TempDirectory)/deps_ccache
     ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache

diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -303,31 +303,31 @@ stages:
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
           PYTHON_VERSION: '3.8'
           EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
           ENV_SETUP_SCRIPT: setup_env.bat
           EP_NAME: directml
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
           PYTHON_VERSION: '3.9'
           EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
           ENV_SETUP_SCRIPT: setup_env.bat
           EP_NAME: directml
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
           PYTHON_VERSION: '3.10'
           EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
           ENV_SETUP_SCRIPT: setup_env.bat
           EP_NAME: directml
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
           PYTHON_VERSION: '3.11'
           EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
           ENV_SETUP_SCRIPT: setup_env.bat

diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
@@ -28,7 +28,7 @@ jobs:
     clean: all
   pool: ${{ parameters.MACHINE_POOL }}
   variables:
-    ${{if or(eq(parameters.MACHINE_POOL, 'onnxruntime-gpu-winbuild-T4'), eq(parameters.MACHINE_POOL, 'aiinfra-dml-winbuild'))}}:
+    ${{if or(eq(parameters.MACHINE_POOL, 'onnxruntime-gpu-winbuild-T4'), eq(parameters.MACHINE_POOL, 'onnxruntime-Win2019-GPU-dml-A10'))}}:
       VSGenerator: 'Visual Studio 16 2019'
     ${{else}}:
       VSGenerator: 'Visual Studio 17 2022'

diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@@ -81,7 +81,7 @@ stages:
         msbuildPlatform: x64
         isX86: false
         job_name_suffix: x64_release
-        RunOnnxRuntimeTests: false
+        RunOnnxRuntimeTests: true
         RunStaticCodeAnalysis: false
         isTraining: false
         ORT_EP_NAME: XNNPACK
@@ -211,4 +211,4 @@ stages:
         ORT_EP_NAME: CPU
         GenerateDocumentation: false
         WITH_CACHE: true
-        MachinePool: 'onnxruntime-Win-CPU-2022'
+        MachinePool: 'onnxruntime-Win-CPU-2022'
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,8 +3,8 @@ name: Lint @@
     on:
       push:
         branches:
-          - master
           - main
+          - rel-*
       pull_request:
     jobs:
@@ Expand Down @@