From 800493e3618bca7c5714d05d8016631cb85c6b9d Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 13:41:53 -0400 Subject: [PATCH 1/8] Enable 1ES on Python CUDA Package Pipeline --- .../stages/py-linux-gpu-stage.yml | 1 + .../stages/py-win-gpu-stage.yml | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index f9053cba56835..c23c49e0b7c3c 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -99,6 +99,7 @@ stages: mv $(Build.BinariesDirectory)/dist ./dist pushd dist find . -name \*.whl -exec unzip -qq -o {} \; + rm -r onnxruntime _mainifest popd pushd ${{ parameters.cmake_build_type }} find . -name \*.whl -exec unzip -qq -o {} \; diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 0cbcd2b74371e..1ed7ec0aa01ec 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -51,7 +51,23 @@ stages: workspace: clean: all pool: - name: onnxruntime-Win-CPU-2022 + name: ${{ parameters.machine_pool }} + os: linux + templateContext: + codeSignValidation: + enabled: true + break: true + psscriptanalyzer: + enabled: true + sdl: + binskim: + enabled: true + scanOutputDirectoryOnly: true + targetPathPattern: '\".*.so\"' + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: onnxruntime_${{ parameters.EP_NAME }} variables: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' @@ -164,11 +180,6 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime_${{ parameters.EP_NAME }} - - ${{ if eq(parameters.publish_symbols, true) }}: - task: PublishSymbols@2 displayName: 'Publish symbols' From 6e3990a0bd847b09c87c72841315f4fb297943d6 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 13:46:56 -0400 Subject: [PATCH 2/8] Enable 1ES on Python CUDA Package Pipeline --- .../github/azure-pipelines/stages/py-win-gpu-stage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 1ed7ec0aa01ec..6d08839bc645a 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -51,8 +51,8 @@ stages: workspace: clean: all pool: - name: ${{ parameters.machine_pool }} - os: linux + name: onnxruntime-Win-CPU-2022 + os: windows templateContext: codeSignValidation: enabled: true From 97df6c6b4db252a39034e81e6fc91c2299db40c9 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 13:52:20 -0400 Subject: [PATCH 3/8] Enable 1ES on Python CUDA Package Pipeline --- .../py-cuda-alt-packaging-pipeline.yml | 24 +++++++++++++------ .../py-dml-packaging-pipeline.yml | 22 ++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml index cc2977721d03b..b24b74651f577 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml @@ -17,11 +17,21 @@ parameters: - Release - RelWithDebInfo - MinSizeRel +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + # Update the pool with your team's 1ES hosted pool. + pool: + name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool + os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS -stages: - - template: stages/py-gpu-packaging-stage.yml - parameters: - enable_linux_cuda: ${{ parameters.enable_linux_cuda }} - enable_windows_cuda: ${{ parameters.enable_windows_cuda }} - cmake_build_type: ${{ parameters.cmake_build_type }} - cuda_version: '11.8' + stages: + - template: stages/py-gpu-packaging-stage.yml + parameters: + enable_linux_cuda: ${{ parameters.enable_linux_cuda }} + enable_windows_cuda: ${{ parameters.enable_windows_cuda }} + cmake_build_type: ${{ parameters.cmake_build_type }} + cuda_version: '11.8' diff --git a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml index 0c7c6abeb35da..5694ac78f49f2 100644 --- a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml @@ -9,10 +9,20 @@ parameters: - Release - RelWithDebInfo - MinSizeRel +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + # Update the pool with your team's 1ES hosted pool. + pool: + name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool + os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS -stages: - - template: stages/py-gpu-packaging-stage.yml - parameters: - enable_windows_dml: true - cmake_build_type: ${{ parameters.cmake_build_type }} - publish_symbols: true + stages: + - template: stages/py-gpu-packaging-stage.yml + parameters: + enable_windows_dml: true + cmake_build_type: ${{ parameters.cmake_build_type }} + publish_symbols: true From 6638e588d085762d15caf395662de98c2bb89ad9 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 13:56:20 -0400 Subject: [PATCH 4/8] Enable 1ES on Python CUDA Package Pipeline --- .../azure-pipelines/py-cuda-alt-packaging-pipeline.yml | 7 ++++++- .../github/azure-pipelines/py-dml-packaging-pipeline.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml index b24b74651f577..844991c475ff7 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml @@ -1,5 +1,10 @@ trigger: none - +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release parameters: - name: enable_linux_cuda type: boolean diff --git a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml index 5694ac78f49f2..663e66db8c135 100644 --- a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml @@ -1,5 +1,10 @@ trigger: none - +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release parameters: - name: cmake_build_type type: string From b324d889650f59ffd79d654715bb5dd1e0e869e6 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 14:13:29 -0400 Subject: [PATCH 5/8] Enable 1ES on Python CUDA Package Pipeline --- .../py-dml-packaging-pipeline.yml | 1 - .../stages/py-gpu-packaging-stage.yml | 5 -- .../stages/py-win-gpu-stage.yml | 69 ++----------------- 3 files changed, 4 insertions(+), 71 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml index 663e66db8c135..280b54e4b9c2d 100644 --- a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml @@ -30,4 +30,3 @@ extends: parameters: enable_windows_dml: true cmake_build_type: ${{ parameters.cmake_build_type }} - publish_symbols: true diff --git a/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml index 1ae95a296162c..0160fdd6ddd95 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml @@ -48,10 +48,6 @@ parameters: - '3.12' - '3.13' -- name: publish_symbols - type: boolean - default: false - stages: - ${{ if eq(parameters.enable_windows_cuda, true) }}: - ${{ each python_version in parameters.PythonVersions }}: @@ -89,5 +85,4 @@ stages: EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos ENV_SETUP_SCRIPT: setup_env.bat EP_NAME: directml - publish_symbols: ${{ parameters.publish_symbols }} cmake_build_type: ${{ parameters.cmake_build_type }} \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 6d08839bc645a..37f6a2b7e61e6 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -38,10 +38,6 @@ parameters: - RelWithDebInfo - MinSizeRel -- name: publish_symbols - type: boolean - default: false - stages: - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build dependsOn: [] @@ -63,11 +59,11 @@ stages: binskim: enabled: true scanOutputDirectoryOnly: true - targetPathPattern: '\".*.so\"' + targetPathPattern: '+:file|*.dll;-:file|DirectML.dll' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) - artifactName: onnxruntime_${{ parameters.EP_NAME }} + artifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }} variables: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' @@ -89,12 +85,6 @@ stages: addToPath: true architecture: 'x64' - - task: onebranch.pipeline.tsaoptions@1 - displayName: 'OneBranch TSAOptions' - inputs: - tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' - appendSourceBranchName: false - - template: ../templates/download-deps.yml - ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}: @@ -122,13 +112,6 @@ stages: arguments: --new_dir $(Build.BinariesDirectory)/deps workingDirectory: $(Build.BinariesDirectory) - - task: PowerShell@2 - displayName: 'Install ONNX' - inputs: - filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' - workingDirectory: '$(Build.BinariesDirectory)' - arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }} - - template: ../templates/set-nightly-build-option-variable-step.yml - task: PythonScript@0 @@ -142,19 +125,7 @@ stages: --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests - --parallel --use_binskim_compliant_compile_flags --update - $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} - workingDirectory: '$(Build.BinariesDirectory)' - - # building with build.py so the parallelization parameters are added to the msbuild command - - task: PythonScript@0 - displayName: 'Build' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config ${{ parameters.cmake_build_type }} - --build_dir $(Build.BinariesDirectory) - --parallel --build + --parallel --use_binskim_compliant_compile_flags --update --build $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} workingDirectory: '$(Build.BinariesDirectory)' @@ -180,43 +151,11 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - ${{ if eq(parameters.publish_symbols, true) }}: - - task: PublishSymbols@2 - displayName: 'Publish symbols' - condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-'))) - inputs: - SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}' - SearchPattern: | - onnxruntime_pybind11_state.pdb - onnxruntime_providers_shared.pdb - IndexSources: true - SymbolServerType: TeamServices - SymbolExpirationInDays: 3650 - SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)' - - script: | 7z x *.whl workingDirectory: '$(Build.ArtifactStagingDirectory)' displayName: 'unzip the package' - - task: CredScan@3 - displayName: 'Run CredScan' - inputs: - debugMode: false - continueOnError: true - - - task: BinSkim@4 - displayName: 'Run BinSkim' - inputs: - AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll' - - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - template: ../templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' @@ -248,7 +187,7 @@ stages: parameters: ArtifactName: onnxruntime_${{ parameters.EP_NAME }} StepName: 'Download Pipeline Artifact - Windows GPU Build' - TargetPath: '$(Build.ArtifactStagingDirectory)' + TargetPath: 'win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}' - task: PowerShell@2 displayName: 'Install ONNX' From 55d50a9c2a175f149c6cb8f9d47ac0c712159b06 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 14:16:21 -0400 Subject: [PATCH 6/8] Enable 1ES on Python CUDA Package Pipeline --- tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index ed992be31257a..bb9ada7d6cb4b 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -77,5 +77,3 @@ stages: build_py_parameters: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} qnn_sdk_version: ${{ parameters.qnn_sdk_version }} - publish_symbols: true - From 87ce644591889cc0cad0ad9cc11188722323f5ec Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 17:57:33 -0400 Subject: [PATCH 7/8] '$(Build.ArtifactStagingDirectory)' --- .../github/azure-pipelines/stages/py-win-gpu-stage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 37f6a2b7e61e6..88937cc2e154d 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -185,9 +185,9 @@ stages: - template: ../templates/flex-downloadPipelineArtifact.yml parameters: - ArtifactName: onnxruntime_${{ parameters.EP_NAME }} + ArtifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }} StepName: 'Download Pipeline Artifact - Windows GPU Build' - TargetPath: 'win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}' + TargetPath: '$(Build.ArtifactStagingDirectory)' - task: PowerShell@2 displayName: 'Install ONNX' From 4d9238a865781cd7a2b394d9b423edadabf3e38d Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 17:58:25 -0400 Subject: [PATCH 8/8] _mainifest --- .../github/azure-pipelines/stages/py-linux-gpu-stage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index c23c49e0b7c3c..83b863f18fbc4 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -99,7 +99,7 @@ stages: mv $(Build.BinariesDirectory)/dist ./dist pushd dist find . -name \*.whl -exec unzip -qq -o {} \; - rm -r onnxruntime _mainifest + rm -r onnxruntime popd pushd ${{ parameters.cmake_build_type }} find . -name \*.whl -exec unzip -qq -o {} \;