Skip to content

Commit

Permalink
Enable 1ES on Python CUDA Package Pipelines (microsoft#22560)
Browse files Browse the repository at this point in the history
### Description
These 3 following CUDA packaging pipeline shoud be enabled with 1ES
after this pull request.
•
[Python-CUDA-Packaging-Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1299&view=runs)
• [Python CUDA Alt Packaging
Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1626)
• [Python DML Packaging
Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1625)

This should also fix the issue where [Python packaging
pipeline](https://aiinfra.visualstudio.com/Lotus/_build?definitionId=841&_a=summary)
failed due to cannot find `publish_symbols`


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
  • Loading branch information
jchen351 authored and ankitm3k committed Dec 11, 2024
1 parent e809f42 commit 340704d
Show file tree
Hide file tree
Showing 6 changed files with 63 additions and 90 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
trigger: none

resources:
repositories:
- repository: 1esPipelines
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
parameters:
- name: enable_linux_cuda
type: boolean
Expand All @@ -17,11 +22,21 @@ parameters:
- Release
- RelWithDebInfo
- MinSizeRel
extends:
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
# For non-production pipelines, use "Unofficial" as defined below.
# For productions pipelines, use "Official".
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
parameters:
# Update the pool with your team's 1ES hosted pool.
pool:
name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool
os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS

stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: '11.8'
stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: '11.8'
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
trigger: none

resources:
repositories:
- repository: 1esPipelines
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
parameters:
- name: cmake_build_type
type: string
Expand All @@ -9,10 +14,19 @@ parameters:
- Release
- RelWithDebInfo
- MinSizeRel
extends:
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
# For non-production pipelines, use "Unofficial" as defined below.
# For productions pipelines, use "Official".
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
parameters:
# Update the pool with your team's 1ES hosted pool.
pool:
name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool
os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS

stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_windows_dml: true
cmake_build_type: ${{ parameters.cmake_build_type }}
publish_symbols: true
stages:
- template: stages/py-gpu-packaging-stage.yml
parameters:
enable_windows_dml: true
cmake_build_type: ${{ parameters.cmake_build_type }}
Original file line number Diff line number Diff line change
Expand Up @@ -77,5 +77,3 @@ stages:
build_py_parameters: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
publish_symbols: true

Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,6 @@ parameters:
- '3.12'
- '3.13'

- name: publish_symbols
type: boolean
default: false

stages:
- ${{ if eq(parameters.enable_windows_cuda, true) }}:
- ${{ each python_version in parameters.PythonVersions }}:
Expand Down Expand Up @@ -89,5 +85,4 @@ stages:
EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
ENV_SETUP_SCRIPT: setup_env.bat
EP_NAME: directml
publish_symbols: ${{ parameters.publish_symbols }}
cmake_build_type: ${{ parameters.cmake_build_type }}
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ stages:
mv $(Build.BinariesDirectory)/dist ./dist
pushd dist
find . -name \*.whl -exec unzip -qq -o {} \;
rm -r onnxruntime
popd
pushd ${{ parameters.cmake_build_type }}
find . -name \*.whl -exec unzip -qq -o {} \;
Expand Down
86 changes: 18 additions & 68 deletions tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,6 @@ parameters:
- RelWithDebInfo
- MinSizeRel

- name: publish_symbols
type: boolean
default: false

stages:
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
dependsOn: []
Expand All @@ -52,6 +48,22 @@ stages:
clean: all
pool:
name: onnxruntime-Win-CPU-2022
os: windows
templateContext:
codeSignValidation:
enabled: true
break: true
psscriptanalyzer:
enabled: true
sdl:
binskim:
enabled: true
scanOutputDirectoryOnly: true
targetPathPattern: '+:file|*.dll;-:file|DirectML.dll'
outputs:
- output: pipelineArtifact
targetPath: $(Build.ArtifactStagingDirectory)
artifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
variables:
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
VSGenerator: 'Visual Studio 17 2022'
Expand All @@ -73,12 +85,6 @@ stages:
addToPath: true
architecture: 'x64'

- task: onebranch.pipeline.tsaoptions@1
displayName: 'OneBranch TSAOptions'
inputs:
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
appendSourceBranchName: false

- template: ../templates/download-deps.yml

- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
Expand Down Expand Up @@ -106,13 +112,6 @@ stages:
arguments: --new_dir $(Build.BinariesDirectory)/deps
workingDirectory: $(Build.BinariesDirectory)

- task: PowerShell@2
displayName: 'Install ONNX'
inputs:
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
workingDirectory: '$(Build.BinariesDirectory)'
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}

- template: ../templates/set-nightly-build-option-variable-step.yml

- task: PythonScript@0
Expand All @@ -126,19 +125,7 @@ stages:
--cmake_generator "$(VSGenerator)"
--enable_pybind
--enable_onnx_tests
--parallel --use_binskim_compliant_compile_flags --update
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'

# building with build.py so the parallelization parameters are added to the msbuild command
- task: PythonScript@0
displayName: 'Build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config ${{ parameters.cmake_build_type }}
--build_dir $(Build.BinariesDirectory)
--parallel --build
--parallel --use_binskim_compliant_compile_flags --update --build
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'

Expand All @@ -164,48 +151,11 @@ stages:
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'

- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}

- ${{ if eq(parameters.publish_symbols, true) }}:
- task: PublishSymbols@2
displayName: 'Publish symbols'
condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
inputs:
SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
SearchPattern: |
onnxruntime_pybind11_state.pdb
onnxruntime_providers_shared.pdb
IndexSources: true
SymbolServerType: TeamServices
SymbolExpirationInDays: 3650
SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'

- script: |
7z x *.whl
workingDirectory: '$(Build.ArtifactStagingDirectory)'
displayName: 'unzip the package'
- task: CredScan@3
displayName: 'Run CredScan'
inputs:
debugMode: false
continueOnError: true

- task: BinSkim@4
displayName: 'Run BinSkim'
inputs:
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'

- task: TSAUpload@2
displayName: 'TSA upload'
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
inputs:
GdnPublishTsaOnboard: false
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'

- template: ../templates/component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
Expand Down Expand Up @@ -235,7 +185,7 @@ stages:

- template: ../templates/flex-downloadPipelineArtifact.yml
parameters:
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
ArtifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
StepName: 'Download Pipeline Artifact - Windows GPU Build'
TargetPath: '$(Build.ArtifactStagingDirectory)'

Expand Down

0 comments on commit 340704d

Please sign in to comment.