From 636a40c4b0d5a4d325a45ab3fc1db7c0eb6fad00 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 27 Mar 2024 12:17:22 +0800 Subject: [PATCH] Fix and Refactor Python Packaging Pipeline (#20085) ### Description Make Windows GPU Packaging stage in Python Packaging pipeline run on CPU machine as well ### Motivation and Context ### Test Link https://dev.azure.com/aiinfra/Lotus/_build/results?buildId=430961&view=results --- .../templates/py-packaging-stage.yml | 211 ++++++++++-------- .../azure-pipelines/templates/py-win-gpu.yml | 16 +- 2 files changed, 123 insertions(+), 104 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 2e5aee77604c1..7868ebe3cef6f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -57,9 +57,8 @@ parameters: - MinSizeRel stages: -- stage: Python_Packaging +- stage: Python_Packaging_Windows_CPU dependsOn: [] - jobs: - ${{ if eq(parameters.enable_windows_cpu, true) }}: - job: Windows_py_Wheels @@ -275,88 +274,91 @@ stages: displayName: 'Clean Agent Directories' condition: always() - - ${{ if eq(parameters.enable_windows_gpu, true) }}: - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' - PYTHON_VERSION: '3.8' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' - PYTHON_VERSION: '3.9' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' - PYTHON_VERSION: '3.10' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' - PYTHON_VERSION: '3.11' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' - PYTHON_VERSION: '3.12' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.8' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.9' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.10' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.11' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.12' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - - - ${{ if eq(parameters.enable_mac_cpu, true) }}: +- ${{ if eq(parameters.enable_windows_gpu, true) }}: + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' + PYTHON_VERSION: '3.8' + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + ENV_SETUP_SCRIPT: setup_env_gpu.bat + EP_NAME: gpu + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' + PYTHON_VERSION: '3.9' + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + ENV_SETUP_SCRIPT: setup_env_gpu.bat + EP_NAME: gpu + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' + PYTHON_VERSION: '3.10' + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + ENV_SETUP_SCRIPT: setup_env_gpu.bat + EP_NAME: gpu + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' + PYTHON_VERSION: '3.11' + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + ENV_SETUP_SCRIPT: setup_env_gpu.bat + EP_NAME: gpu + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4' + PYTHON_VERSION: '3.12' + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + ENV_SETUP_SCRIPT: setup_env_gpu.bat + EP_NAME: gpu + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: '3.8' + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: '3.9' + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: '3.10' + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: '3.11' + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: '3.12' + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + +- ${{ if eq(parameters.enable_mac_cpu, true) }}: + - stage: Python_Packaging_MacOS + dependsOn: [] + jobs: - job: MacOS_py_Wheels timeoutInMinutes: 180 workspace: @@ -426,19 +428,26 @@ stages: parameters: condition: 'succeeded' + - ${{ if eq(parameters.enable_linux_arm, true) }}: - - template: py-linux.yml - parameters: - arch: 'aarch64' - machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' - base_image: 'arm64v8/almalinux:8' - devtoolset_rootpath: /opt/rh/gcc-toolset-12/root - ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64 - prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:' - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} + - stage: Python_Packaging_Linux_ARM + dependsOn: [] + jobs: + - template: py-linux.yml + parameters: + arch: 'aarch64' + machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' + base_image: 'arm64v8/almalinux:8' + devtoolset_rootpath: /opt/rh/gcc-toolset-12/root + ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64 + prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:' + extra_build_arg: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} - ${{ if eq(parameters.enable_linux_cpu, true) }}: + - stage: Python_Packaging_Linux_CPU + dependsOn: [] + jobs: - template: py-linux.yml parameters: arch: 'x86_64' @@ -460,6 +469,9 @@ stages: cmake_build_type: ${{ parameters.cmake_build_type }} - ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: + - stage: Python_Packaging_Windows_ARM64_QNN + dependsOn: [] + jobs: - template: py-win-arm64-qnn.yml parameters: MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' @@ -468,7 +480,10 @@ stages: NUMPY_VERSION: '1.25.2' - ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: - - template: py-win-x64-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: 'qnn-v2.18.0.240101_win' + - stage: Python_Packaging_Windows_x64_QNN + dependsOn: [] + jobs: + - template: py-win-x64-qnn.yml + parameters: + MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' + QNN_SDK: 'qnn-v2.18.0.240101_win' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index 17915d107dbe6..e200fb9e93bee 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -1,4 +1,8 @@ parameters: +- name: MACHINE_POOL + type: string + default: 'onnxruntime-Win2022-GPU-T4' + - name: EP_NAME type: string @@ -44,8 +48,6 @@ stages: clean: all pool: name: onnxruntime-Win-CPU-2022 - # demands: - # - ImageVersionOverride -equals 1.0.367516 variables: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' @@ -126,7 +128,9 @@ stages: inputs: targetPath: '$(Agent.TempDirectory)\onnx\onnx-1.15.0\dist\' publishLocation: 'pipeline' - artifactName: onnx_py12_wheel + # In fact, the onnx artifact is same with different EP_NAME, but in CI, we couldn't publish artifact with same name. + # And there's not way to check if the artifact exists. + artifactName: onnx_py12_wheel_${{ parameters.EP_NAME }} - template: set-nightly-build-option-variable-step.yml @@ -207,7 +211,7 @@ stages: workspace: clean: all pool: - name: onnxruntime-Win2022-GPU-T4 + name: ${{parameters.MACHINE_POOL}} steps: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' @@ -225,7 +229,7 @@ stages: - template: flex-downloadPipelineArtifact.yml parameters: - ArtifactName: "onnxruntime_${{ parameters.EP_NAME }}" + ArtifactName: onnxruntime_${{ parameters.EP_NAME }} StepName: 'Download Pipeline Artifact - Windows GPU Build' TargetPath: '$(Build.ArtifactStagingDirectory)' SpecificArtifact: ${{ parameters.SpecificArtifact }} @@ -235,7 +239,7 @@ stages: - ${{ if eq(parameters.PYTHON_VERSION, '3.12') }}: - template: flex-downloadPipelineArtifact.yml parameters: - ArtifactName: "onnx_py12_wheel" + ArtifactName: onnx_py12_wheel_${{ parameters.EP_NAME }} StepName: 'Download Pipeline Artifact - Onnx Python12 wheel' TargetPath: '$(Agent.TempDirectory)\onnx\' SpecificArtifact: ${{ parameters.SpecificArtifact }}