Update Python-Cuda-Publishing-Pipeline (#23253)

### Description 1. Currently Python-Cuda-Publishing-Pipeline only publishes Linux wheels, not Windows wheels. It is because recently we refactored the upstream pipeline("Python-CUDA-Packaging-Pipeline") to use 1ES PT. This PR fixed the issue 2. tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml no longer includes component-governance-component-detection-steps.yml , because 1ES PT already inserted such a thing 3. Delete tools/ci_build/github/windows/eager/requirements.txt because it is no longer used. ### Motivation and Context The "Python-CUDA-Packaging-Pipeline" is for CUDA 12. "Python CUDA ALT Packaging Pipeline" is for CUDA 11. The two pipelines are very similar, except the CUDA versions are different. Each of them has three parts: build, test, publish. "Python-CUDA-Packaging-Pipeline" is the first part: build. "Python CUDA12 Package Test Pipeline" is the second part. "Python-Cuda-Publishing-Pipeline" is the third part that publishes the packages to an internal ADO feed.
microsoft · Jan 12, 2025 · 5bddcc3 · 5bddcc3
1 parent 31c34e2
commit 5bddcc3
Show file tree

Hide file tree

Showing 9 changed files with 102 additions and 42 deletions.
diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml
@@ -19,6 +19,39 @@ stages:
           python_wheel_suffix: '_gpu'
           timeout: 480
           docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241120.3
-          trt_version: '10.7.0.23-1.cuda11.8'
           cuda_version: '11.8'
 
+  - stage: Republish_Wheels
+    dependsOn:
+    jobs:
+    - job: Python_Publishing_GPU
+      pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
+      steps:
+      - checkout: none
+      - download: build
+        displayName: 'Download Pipeline Artifact - onnxruntime_gpu'
+        artifact: 'onnxruntime_gpu'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.10'
+        artifact: 'win_gpu_wheel_3.10'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.11'
+        artifact: 'win_gpu_wheel_3.11'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.12'
+        artifact: 'win_gpu_wheel_3.12'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.13'
+        artifact: 'win_gpu_wheel_3.13'
+        patterns: '*.whl'
+
+      - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \;
+        displayName: 'Merge files together'
+
+      - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu
+        artifact: whl
+        displayName: Republish artifacts
diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml
@@ -29,6 +29,12 @@ extends:
   template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
   parameters:
     sdl:
+      componentgovernance:
+        ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)'
+        alertWarningLevel: High
+        failOnAlert: false
+        verbosity: Normal
+        timeout: 3600
       tsa:
         enabled: true
       codeSignValidation:

diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
@@ -21,3 +21,37 @@ stages:
           docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241120.3
           cuda_version: '12.2'
 
+  - stage: Republish_Wheels
+    dependsOn:
+    jobs:
+    - job: Python_Publishing_GPU
+      pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
+      steps:
+      - checkout: none
+      - download: build
+        displayName: 'Download Pipeline Artifact - onnxruntime_gpu'
+        artifact: 'onnxruntime_gpu'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.10'
+        artifact: 'win_gpu_wheel_3.10'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.11'
+        artifact: 'win_gpu_wheel_3.11'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.12'
+        artifact: 'win_gpu_wheel_3.12'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.13'
+        artifact: 'win_gpu_wheel_3.13'
+        patterns: '*.whl'
+
+      - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \;
+        displayName: 'Merge files together'
+
+      - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu
+        artifact: whl
+        displayName: Republish artifacts
diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
@@ -32,6 +32,12 @@ extends:
   template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
   parameters:
     sdl:
+      componentgovernance:
+        ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)'
+        alertWarningLevel: High
+        failOnAlert: false
+        verbosity: Normal
+        timeout: 3600
       tsa:
         enabled: true
       codeSignValidation:

diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml
@@ -1,7 +1,7 @@
 resources:
   pipelines:
   - pipeline: build
-    source: 'Python-CUDA-Packaging-Pipeline'
+    source: 'Python CUDA12 Package Test Pipeline'
     trigger:
       branches:
         include:

diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml
@@ -11,20 +11,21 @@ stages:
     steps:
     - checkout: none
     - download: build
-      displayName: 'Download Pipeline Artifact - onnxruntime_gpu'
-      artifact: 'onnxruntime_gpu'
+      displayName: 'Download Pipeline Artifact'
+      artifact: 'whl'
+
     - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.13'
       displayName: 'Use Python 3.x'
-    - script: 'pip install twine==3.4.2'
+    - script: 'pip install twine==6.0.1'
       displayName: 'Install Twine'
     - task: TwineAuthenticate@1
       displayName: 'Twine Authenticate '
       inputs:
         artifactFeed: PublicPackages/${{ parameters.artifact_feed }}
-    - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive --skip-existing *.whl'
-      workingDirectory: '$(Pipeline.Workspace)/build/onnxruntime_gpu'
+
+    - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive *.whl'
+      workingDirectory: '$(Pipeline.Workspace)/build/whl'
       displayName: 'Uploading wheels to ${{ parameters.artifact_feed }}'
-      retryCountOnTaskFailure: 3
-      env:
-        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
 
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml
@@ -39,16 +39,6 @@ stages:
       name: ${{ parameters.machine_pool }}
       os: linux
     templateContext:
-      codeSignValidation:
-        enabled: true
-        break: true
-      psscriptanalyzer:
-        enabled: true
-      sdl:
-        binskim:
-          enabled: true
-          scanOutputDirectoryOnly: true
-          targetPathPattern: '\".*.so\"'
       outputs:
       - output: pipelineArtifact
         targetPath: $(Build.ArtifactStagingDirectory)/dist
@@ -57,9 +47,6 @@ stages:
         targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.cmake_build_type }}
         artifactName: linux_gpu_wheel_${{ parameters.arch }}
     variables:
-      # The build machine pool doesn't have dotnet, so it can't run CG.
-      - name: skipComponentGovernanceDetection
-        value: true
       - name: extra_build_args
         ${{ if ne(parameters.extra_build_arg, '') }}:
           value: -x ${{ parameters.extra_build_arg }}

diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml
@@ -50,16 +50,19 @@ stages:
         name: onnxruntime-Win-CPU-2022
         os: windows
       templateContext:
-        codeSignValidation:
-          enabled: true
-          break: true
-        psscriptanalyzer:
-          enabled: true
         sdl:
+          codeSignValidation:
+            enabled: true
+            # TODO: check why pyd file was not signed
+            break: false
+            additionalTargetsGlobPattern: f|**\*.pyd
+          psscriptanalyzer:
+            enabled: true
           binskim:
+            preReleaseVersion: '4.3.1'
             enabled: true
             scanOutputDirectoryOnly: true
-            targetPathPattern: '+:file|*.dll;-:file|DirectML.dll'
+            analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)\**\*.pyd;+:file|$(BUILD.STAGINGDIRECTORY)\**\*.dll;-:file|DirectML.dll'
         outputs:
         - output: pipelineArtifact
           targetPath: $(Build.ArtifactStagingDirectory)
@@ -86,7 +89,7 @@ stages:
 
           - checkout: self
             clean: true
-            submodules: recursive
+            submodules: none
 
           - template: ../templates/telemetry-steps.yml
 
@@ -121,7 +124,7 @@ stages:
           - template: ../templates/set-nightly-build-option-variable-step.yml
 
           - task: PythonScript@0
-            displayName: 'Generate cmake config'
+            displayName: 'Build'
             inputs:
               scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
               arguments: >
@@ -131,7 +134,7 @@ stages:
                 --cmake_generator "$(VSGenerator)"
                 --enable_pybind
                 --enable_onnx_tests
-                --parallel --use_binskim_compliant_compile_flags --update --build
+                --parallel 8 --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40
                 $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }}
               workingDirectory: '$(Build.BinariesDirectory)'
 
@@ -162,9 +165,6 @@ stages:
             workingDirectory: '$(Build.ArtifactStagingDirectory)'
             displayName: 'unzip the package'
 
-          - template: ../templates/component-governance-component-detection-steps.yml
-            parameters:
-              condition: 'succeeded'
 
   - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
     dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build

diff --git a/tools/ci_build/github/windows/eager/requirements.txt b/tools/ci_build/github/windows/eager/requirements.txt