Skip to content

Commit

Permalink
Refactor cuda packaging pipeline (#22542)
Browse files Browse the repository at this point in the history
### Description
<!-- Describe your changes. -->



### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
  • Loading branch information
jchen351 authored Oct 23, 2024
1 parent 0028d3f commit ffaddea
Show file tree
Hide file tree
Showing 11 changed files with 172 additions and 261 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,8 @@ stages:
# build Python packages
# Linux GPU only
- ${{ if parameters.BuildPythonPackages }}:
- template: templates/py-packaging-stage.yml
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_linux_gpu: true
enable_linux_cpu: false
enable_windows_cpu: false
enable_windows_gpu: false
enable_mac_cpu: false
enable_linux_arm: false
cuda_version: 12.2
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
trigger: none

parameters:
- name: enable_linux_cuda
type: boolean
default: true

- name: enable_windows_cuda
type: boolean
default: true

- name: cmake_build_type
type: string
default: 'Release'
values:
- Debug
- Release
- RelWithDebInfo
- MinSizeRel

stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: '11.8'
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
trigger: none

# The `resources` specify the location and version of the 1ES PT.
resources:
repositories:
- repository: 1esPipelines
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
parameters:
- name: enable_linux_gpu
- name: enable_linux_cuda
type: boolean
default: true
- name: enable_windows_gpu

- name: enable_windows_cuda
type: boolean
default: true

- name: cmake_build_type
type: string
default: 'Release'
Expand All @@ -15,28 +23,22 @@ parameters:
- Release
- RelWithDebInfo
- MinSizeRel
- name: cuda_version
type: string
default: '12.2'
values:
- 11.8
- 12.2
- name: SpecificArtifact
displayName: Use Specific Artifact
type: boolean
default: false

- name: BuildId
displayName: Specific Artifact's BuildId
type: string
default: '0'
extends:
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
# For non-production pipelines, use "Unofficial" as defined below.
# For productions pipelines, use "Official".
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
parameters:
# Update the pool with your team's 1ES hosted pool.
pool:
name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool
os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS

stages:
- template: stages/py-cuda-packaging-stage.yml
parameters:
enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: ${{ parameters.cuda_version }}
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: '12.2'
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
trigger: none

parameters:
- name: cmake_build_type
type: string
default: 'Release'
values:
- Debug
- Release
- RelWithDebInfo
- MinSizeRel

stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_windows_dml: true
cmake_build_type: ${{ parameters.cmake_build_type }}
publish_symbols: true
15 changes: 2 additions & 13 deletions tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,11 @@ parameters:
type: boolean
default: true

- name: enable_linux_gpu
displayName: 'Whether Linux GPU package is built.'
type: boolean
default: true

- name: enable_windows_cpu
displayName: 'Whether Windows CPU package is built.'
type: boolean
default: true

- name: enable_windows_gpu
displayName: 'Whether Windows GPU package is built.'
type: boolean
default: true

- name: enable_mac_cpu
displayName: 'Whether Mac CPU package is built.'
type: boolean
Expand Down Expand Up @@ -74,12 +64,10 @@ parameters:
trigger: none

stages:
- template: templates/py-packaging-stage.yml
- template: stages/py-cpu-packaging-stage.yml
parameters:
enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
enable_mac_cpu: ${{ parameters.enable_mac_cpu }}
enable_linux_arm: ${{ parameters.enable_linux_arm }}
enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }}
Expand All @@ -90,3 +78,4 @@ stages:
cmake_build_type: ${{ parameters.cmake_build_type }}
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
publish_symbols: true

Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,15 @@ jobs:
- checkout: self
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'drop-linux-gpu-x86_64'
targetPath: '$(Build.SourcesDirectory)/drop-linux-gpu-x86_64'
artifact: 'linux_gpu_wheel_x86_64'
targetPath: '$(Build.SourcesDirectory)/linux_gpu_wheel_x86_64'
${{ if ne(parameters.build_id, 'latest') }}:
buildType: 'specific'
project: '${{ parameters.project }}'
pipeline: '${{ parameters.pipeline }}'
buildVersionToDownload: 'specific'
buildId: '${{ parameters.build_id }}'
displayName: 'Download Build Artifacts - drop-linux-gpu-x86_64'
displayName: 'Download Build Artifacts - linux_gpu_wheel_x86_64'

- task: DownloadPipelineArtifact@2
inputs:
Expand All @@ -82,7 +82,7 @@ jobs:
- bash: |
set -e -x
ls $(Build.SourcesDirectory)
mv "$(Build.SourcesDirectory)/drop-linux-gpu-x86_64" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}}
mv "$(Build.SourcesDirectory)/linux_gpu_wheel_x86_64" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}}
mv "$(Build.SourcesDirectory)/onnxruntime_gpu" "$(Build.BinariesDirectory)/whl"
cp -r "$(Build.BinariesDirectory)/whl" $(Build.BinariesDirectory)/tmp
find "$(Build.BinariesDirectory)/tmp" -name '*.whl' -exec bash -c 'unzip -d "${1%.*}" "$1"' _ {} \;
Expand Down
Loading

0 comments on commit ffaddea

Please sign in to comment.