Skip to content

Commit

Permalink
Split more windows GPU workflow into 2 stages, building and testing, …
Browse files Browse the repository at this point in the history
…to make them more stable (microsoft#20080)

### Description
reactor win-ci.yml to solve the random hang issue in more GPU workflows,
move nugget-zip packages and python cuda12 packages building to CPU
machine.

---------

Co-authored-by: Yi Zhang <[email protected]>
  • Loading branch information
2 people authored and Ted Themistokleous committed May 7, 2024
1 parent e596330 commit c54566b
Show file tree
Hide file tree
Showing 6 changed files with 238 additions and 74 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ stages:
buildJava: true
java_artifact_id: onnxruntime_gpu
CudaVersion: 11.8
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}

# CUDA with Tensorrt
- template: templates/win-ci.yml
Expand Down Expand Up @@ -310,8 +312,8 @@ stages:
dependsOn:
- Linux_C_API_Packaging_GPU_x64
- Linux_C_API_Packaging_GPU_TensorRT_x64
- Windows_Packaging_gpu
- Windows_Packaging_tensorrt
- Windows_Packaging_gpu_Testing
- Windows_Packaging_tensorrt_Testing
- Download_Java_Tools
condition: succeeded()
jobs:
Expand Down Expand Up @@ -488,8 +490,8 @@ stages:

- stage: Windows_Packaging_combined_GPU
dependsOn:
- Windows_Packaging_gpu
- Windows_Packaging_tensorrt
- Windows_Packaging_gpu_Testing
- Windows_Packaging_tensorrt_Testing
condition: succeeded()
jobs:
- job:
Expand Down Expand Up @@ -582,9 +584,9 @@ stages:
- stage: NuGet_Packaging_GPU
dependsOn:
- Setup
- Windows_Packaging_gpu
- Windows_Packaging_gpu_Testing
- Windows_Packaging_CPU_x64_default
- Windows_Packaging_tensorrt
- Windows_Packaging_tensorrt_Testing
- Linux_C_API_Packaging_GPU_x64
- Linux_C_API_Packaging_GPU_TensorRT_x64
condition: succeeded()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ stages:
CudaVersion: ${{ parameters.CudaVersion }}
win_trt_home: ${{ variables.win_trt_home }}
win_cuda_home: ${{ variables.win_cuda_home }}
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
- template: stages/nuget-combine-cuda-stage.yml
parameters:
DoCompliance: ${{ parameters.DoCompliance }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ stages:
- stage: NuGet_Packaging_GPU
dependsOn:
- Set_ReleaseVersionSuffix
- Windows_Packaging_gpu
- Windows_Packaging_tensorrt
- Windows_Packaging_gpu_Testing
- Windows_Packaging_tensorrt_Testing
- Linux_C_API_Packaging_CPU_x64
- Linux_C_API_Packaging_GPU_x64
- Linux_C_API_Packaging_GPU_TensorRT_x64
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,16 @@ parameters:
- name: win_trt_home
type: string

- name: SpecificArtifact
displayName: Use Specific Artifact
type: boolean
default: false

- name: BuildId
displayName: Specific Artifact's BuildId
type: string
default: '0'

stages:
# Windows CUDA without TensorRT Packaging
- template: ../templates/win-ci.yml
Expand All @@ -40,6 +50,8 @@ stages:
buildJava: false
java_artifact_id: onnxruntime_gpu
PublishProtoc: true
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
# Windows CUDA with TensorRT Packaging
- template: ../templates/win-ci.yml
parameters:
Expand All @@ -56,12 +68,14 @@ stages:
buildJava: false
java_artifact_id: onnxruntime_gpu
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}

# Windows CUDA Combined Testing and Publishing
- stage: Windows_Packaging_combined_GPU
dependsOn:
- Windows_Packaging_gpu
- Windows_Packaging_tensorrt
- Windows_Packaging_gpu_Testing
- Windows_Packaging_tensorrt_Testing
condition: succeeded()

jobs:
Expand Down Expand Up @@ -159,4 +173,4 @@ stages:
displayName: 'Publish Pipeline Combined GPU Package Artifact'
inputs:
artifactName: 'onnxruntime-win-x64-gpu'
targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip'
targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip'
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
parameters:
- name: msbuildPlatform
type: string
- name: java_artifact_id
type: string

steps:
- task: CmdLine@2
displayName: 'Add symbols and notices to Java'
inputs:
script: |
@echo on
cd $(Build.SourcesDirectory)\java
call $(Build.SourcesDirectory)\java\gradlew.bat cmakeCheck -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo
if %errorlevel% neq 0 exit /b %errorlevel%
cd $(Build.BinariesDirectory)\RelWithDebInfo
set NATIVE_FOLDER=$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\ai\onnxruntime\native\win-x64
mkdir %NATIVE_FOLDER%
echo "Directories created"
copy .\java\build\libs\*.jar $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}
pushd $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}
set artifact_id=${{ parameters.java_artifact_id }}
jar xf onnxruntime-$(OnnxRuntimeVersion).jar META-INF\maven\com.microsoft.onnxruntime\%artifact_id%\pom.xml
move META-INF\maven\com.microsoft.onnxruntime\%artifact_id%\pom.xml onnxruntime-$(OnnxRuntimeVersion).pom
rd /s /q META-INF
popd
copy .\RelWithDebInfo\onnxruntime.pdb %NATIVE_FOLDER%
copy .\RelWithDebInfo\onnxruntime4j_jni.pdb %NATIVE_FOLDER%
copy $(Build.SourcesDirectory)\docs\Privacy.md $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\Privacy.md
copy $(Build.SourcesDirectory)\ThirdPartyNotices.txt $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\ThirdPartyNotices.txt
@echo $(OnnxRuntimeGitCommitHash) > $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\GIT_COMMIT_ID
pushd $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage
jar uf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\onnxruntime-$(OnnxRuntimeVersion).jar ai\onnxruntime\native\win-x64\onnxruntime.pdb
jar uf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\onnxruntime-$(OnnxRuntimeVersion).jar ai\onnxruntime\native\win-x64\onnxruntime4j_jni.pdb
jar uf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\onnxruntime-$(OnnxRuntimeVersion).jar Privacy.md ThirdPartyNotices.txt GIT_COMMIT_ID
popd
pushd $(Build.SourcesDirectory)\java\build\classes\java\test
if %errorlevel% neq 0 exit /b %errorlevel%
jar cvf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\testing.jar .
if %errorlevel% neq 0 exit /b %errorlevel%
popd
pushd $(Build.SourcesDirectory)\java\build\resources\test
rd /s /q ai\onnxruntime\native
jar uvf $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\testing.jar .
popd
rd /s /q $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage
dir /s /b $(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}
Loading

0 comments on commit c54566b

Please sign in to comment.