From b3c4dfa6904090c77f5f4cd600a917aedda56984 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 04:14:14 +0000 Subject: [PATCH 01/10] update --- .../py-cuda-alt-package-test-pipeline.yml | 34 +++++++++++++++++++ .../py-cuda-packaging-pipeline.yml | 6 ++++ .../stages/py-cuda-publishing-stage.yml | 25 +++++++++++--- .../stages/py-linux-gpu-stage.yml | 16 ++++----- .../stages/py-win-gpu-stage.yml | 22 ++++++------ .../github/windows/eager/requirements.txt | 7 ---- .../requirements/pybind/requirements.txt | 2 +- 7 files changed, 80 insertions(+), 32 deletions(-) delete mode 100644 tools/ci_build/github/windows/eager/requirements.txt diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml index cf434e4eadf0d..27cb61942ff3a 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml @@ -22,3 +22,37 @@ stages: trt_version: '10.7.0.23-1.cuda11.8' cuda_version: '11.8' + - stage: Linux_Test_CUDA_Alt_x86_64_stage + dependsOn: + jobs: + - job: Python_Publishing_GPU + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + steps: + - checkout: none + - download: build + displayName: 'Download Pipeline Artifact - onnxruntime_gpu' + artifact: 'onnxruntime_gpu' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.10' + artifact: 'win_gpu_wheel_3.10' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.11' + artifact: 'win_gpu_wheel_3.11' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.12' + artifact: 'win_gpu_wheel_3.12' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.13' + artifact: 'win_gpu_wheel_3.13' + patterns: '*.whl' + + - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; + displayName: 'Merge files together' + + - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu + artifact: whl + displayName: Republish artifacts \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml index 2e040698fad2a..a80527373e0d0 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml @@ -32,6 +32,12 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: sdl: + componentgovernance: + ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)' + alertWarningLevel: High + failOnAlert: false + verbosity: Normal + timeout: 3600 tsa: enabled: true codeSignValidation: diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml index 85bd5de5b7eb1..a5c516b805ffd 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml @@ -13,6 +13,23 @@ stages: - download: build displayName: 'Download Pipeline Artifact - onnxruntime_gpu' artifact: 'onnxruntime_gpu' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.10' + artifact: 'win_gpu_wheel_3.10' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.11' + artifact: 'win_gpu_wheel_3.11' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.12' + artifact: 'win_gpu_wheel_3.12' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.13' + artifact: 'win_gpu_wheel_3.13' + patterns: '*.whl' + - task: UsePythonVersion@0 displayName: 'Use Python 3.x' - script: 'pip install twine==3.4.2' @@ -21,10 +38,10 @@ stages: displayName: 'Twine Authenticate ' inputs: artifactFeed: PublicPackages/${{ parameters.artifact_feed }} - - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive --skip-existing *.whl' + - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; + displayName: 'Merge files together' + + - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive *.whl' workingDirectory: '$(Pipeline.Workspace)/build/onnxruntime_gpu' displayName: 'Uploading wheels to ${{ parameters.artifact_feed }}' - retryCountOnTaskFailure: 3 - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index 89f6d8b7923cf..2f442de410577 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -39,16 +39,17 @@ stages: name: ${{ parameters.machine_pool }} os: linux templateContext: - codeSignValidation: - enabled: true - break: true - psscriptanalyzer: - enabled: true sdl: + codeSignValidation: + enabled: true + break: true + psscriptanalyzer: + enabled: true binskim: + preReleaseVersion: '4.3.1' enabled: true scanOutputDirectoryOnly: true - targetPathPattern: '\".*.so\"' + analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)\**\*.so' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory)/dist @@ -57,9 +58,6 @@ stages: targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.cmake_build_type }} artifactName: linux_gpu_wheel_${{ parameters.arch }} variables: - # The build machine pool doesn't have dotnet, so it can't run CG. - - name: skipComponentGovernanceDetection - value: true - name: extra_build_args ${{ if ne(parameters.extra_build_arg, '') }}: value: -x ${{ parameters.extra_build_arg }} diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 548641d1985b8..3db56c16ab870 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -50,16 +50,19 @@ stages: name: onnxruntime-Win-CPU-2022 os: windows templateContext: - codeSignValidation: - enabled: true - break: true - psscriptanalyzer: - enabled: true sdl: + codeSignValidation: + enabled: true + # TODO: check why pyd file was not signed + break: false + additionalTargetsGlobPattern: f|**\*.pyd + psscriptanalyzer: + enabled: true binskim: + preReleaseVersion: '4.3.1' enabled: true scanOutputDirectoryOnly: true - targetPathPattern: '+:file|*.dll;-:file|DirectML.dll' + analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)\**\*.pyd;+:file|$(BUILD.STAGINGDIRECTORY)\**\*.dll;-:file|DirectML.dll' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) @@ -121,7 +124,7 @@ stages: - template: ../templates/set-nightly-build-option-variable-step.yml - task: PythonScript@0 - displayName: 'Generate cmake config' + displayName: 'Build' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' arguments: > @@ -131,7 +134,7 @@ stages: --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests - --parallel --use_binskim_compliant_compile_flags --update --build + --parallel 8 --use_binskim_compliant_compile_flags --update --build $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }} workingDirectory: '$(Build.BinariesDirectory)' @@ -162,9 +165,6 @@ stages: workingDirectory: '$(Build.ArtifactStagingDirectory)' displayName: 'unzip the package' - - template: ../templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build diff --git a/tools/ci_build/github/windows/eager/requirements.txt b/tools/ci_build/github/windows/eager/requirements.txt deleted file mode 100644 index d118280c8e6d6..0000000000000 --- a/tools/ci_build/github/windows/eager/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -setuptools -wheel -numpy==1.21.6 ; python_version < '3.9' -numpy==2.1.2 ; python_version >= '3.9' -typing_extensions -torch==2.2.0 -parameterized diff --git a/tools/ci_build/requirements/pybind/requirements.txt b/tools/ci_build/requirements/pybind/requirements.txt index 8f00a25627c21..870d219d0f892 100644 --- a/tools/ci_build/requirements/pybind/requirements.txt +++ b/tools/ci_build/requirements/pybind/requirements.txt @@ -1,7 +1,7 @@ setuptools wheel pytest -numpy>=1.19.0 +numpy sympy>=1.10 packaging cerberus From aa88d2cf3f00dfd390f7b1c15a1b273b10abfdad Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 04:16:17 +0000 Subject: [PATCH 02/10] update --- .../py-cuda-alt-packaging-pipeline.yml | 9 ++++++--- .../azure-pipelines/py-cuda-packaging-pipeline.yml | 3 --- .../azure-pipelines/stages/py-linux-gpu-stage.yml | 12 ++++-------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml index 93a38b212d934..1857cafbce668 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml @@ -29,6 +29,12 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: sdl: + componentgovernance: + ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)' + alertWarningLevel: High + failOnAlert: false + verbosity: Normal + timeout: 3600 tsa: enabled: true codeSignValidation: @@ -37,9 +43,6 @@ extends: policheck: enabled: true exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml' - pool: - name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool - os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS stages: - template: stages/py-gpu-packaging-stage.yml diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml index a80527373e0d0..821a928d91ca9 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml @@ -46,9 +46,6 @@ extends: policheck: enabled: true exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml' - pool: - name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool - os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS stages: - template: stages/py-gpu-packaging-stage.yml diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index 2f442de410577..da8fbe40716ca 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -40,16 +40,12 @@ stages: os: linux templateContext: sdl: - codeSignValidation: - enabled: true - break: true - psscriptanalyzer: - enabled: true binskim: - preReleaseVersion: '4.3.1' - enabled: true + preReleaseVersion: '4.3.1' + # The tool failed with error: No usable version of libssl was found + enabled: false scanOutputDirectoryOnly: true - analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)\**\*.so' + analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)/**/*.so' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory)/dist From 4a8db50ed3ba736f84d6e797c66536ea7a70b521 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 04:17:49 +0000 Subject: [PATCH 03/10] update --- .../github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml | 3 +++ .../github/azure-pipelines/py-cuda-packaging-pipeline.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml index 1857cafbce668..80202928789cf 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml @@ -43,6 +43,9 @@ extends: policheck: enabled: true exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml' + pool: + name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool + os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS stages: - template: stages/py-gpu-packaging-stage.yml diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml index 821a928d91ca9..a80527373e0d0 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml @@ -46,6 +46,9 @@ extends: policheck: enabled: true exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml' + pool: + name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool + os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS stages: - template: stages/py-gpu-packaging-stage.yml From 9c290262c4360827f3d8ab7a5cc4108947d42062 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 04:35:17 +0000 Subject: [PATCH 04/10] update --- .../py-cuda-alt-package-test-pipeline.yml | 58 +++++++++---------- .../py-cuda-package-test-pipeline.yml | 34 +++++++++++ 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml index 27cb61942ff3a..8f26d1ae4976a 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml @@ -25,34 +25,34 @@ stages: - stage: Linux_Test_CUDA_Alt_x86_64_stage dependsOn: jobs: - - job: Python_Publishing_GPU - pool: 'onnxruntime-Ubuntu2204-AMD-CPU' - steps: - - checkout: none - - download: build - displayName: 'Download Pipeline Artifact - onnxruntime_gpu' - artifact: 'onnxruntime_gpu' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.10' - artifact: 'win_gpu_wheel_3.10' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.11' - artifact: 'win_gpu_wheel_3.11' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.12' - artifact: 'win_gpu_wheel_3.12' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.13' - artifact: 'win_gpu_wheel_3.13' - patterns: '*.whl' + - job: Python_Publishing_GPU + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + steps: + - checkout: none + - download: build + displayName: 'Download Pipeline Artifact - onnxruntime_gpu' + artifact: 'onnxruntime_gpu' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.10' + artifact: 'win_gpu_wheel_3.10' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.11' + artifact: 'win_gpu_wheel_3.11' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.12' + artifact: 'win_gpu_wheel_3.12' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.13' + artifact: 'win_gpu_wheel_3.13' + patterns: '*.whl' - - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; - displayName: 'Merge files together' + - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; + displayName: 'Merge files together' - - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu - artifact: whl - displayName: Republish artifacts \ No newline at end of file + - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu + artifact: whl + displayName: Republish artifacts \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml index 307415b7be16f..b0a40cf31912a 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml @@ -21,3 +21,37 @@ stages: docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241120.3 cuda_version: '12.2' + - stage: Linux_Test_CUDA_Alt_x86_64_stage + dependsOn: + jobs: + - job: Python_Publishing_GPU + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + steps: + - checkout: none + - download: build + displayName: 'Download Pipeline Artifact - onnxruntime_gpu' + artifact: 'onnxruntime_gpu' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.10' + artifact: 'win_gpu_wheel_3.10' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.11' + artifact: 'win_gpu_wheel_3.11' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.12' + artifact: 'win_gpu_wheel_3.12' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.13' + artifact: 'win_gpu_wheel_3.13' + patterns: '*.whl' + + - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; + displayName: 'Merge files together' + + - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu + artifact: whl + displayName: Republish artifacts \ No newline at end of file From 8bd53e25352990895c821f1f2e455f1c9a45d3a9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 04:38:33 +0000 Subject: [PATCH 05/10] update --- .../py-cuda-publishing-pipeline.yml | 2 +- .../stages/py-cuda-publishing-stage.yml | 28 ++++--------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml index 1217163c07132..e4499705efdf5 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - pipeline: build - source: 'Python-CUDA-Packaging-Pipeline' + source: 'Python CUDA12 Package Test Pipeline' trigger: branches: include: diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml index a5c516b805ffd..1ff43667f4788 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml @@ -11,37 +11,21 @@ stages: steps: - checkout: none - download: build - displayName: 'Download Pipeline Artifact - onnxruntime_gpu' - artifact: 'onnxruntime_gpu' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.10' - artifact: 'win_gpu_wheel_3.10' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.11' - artifact: 'win_gpu_wheel_3.11' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.12' - artifact: 'win_gpu_wheel_3.12' - patterns: '*.whl' - - download: build - displayName: 'Download Pipeline Artifact - Win GPU 3.13' - artifact: 'win_gpu_wheel_3.13' - patterns: '*.whl' + displayName: 'Download Pipeline Artifact' + artifact: 'whl' - task: UsePythonVersion@0 + inputs: + versionSpec: '3.13' displayName: 'Use Python 3.x' - - script: 'pip install twine==3.4.2' + - script: 'pip install twine==6.0.1' displayName: 'Install Twine' - task: TwineAuthenticate@1 displayName: 'Twine Authenticate ' inputs: artifactFeed: PublicPackages/${{ parameters.artifact_feed }} - - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; - displayName: 'Merge files together' - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive *.whl' - workingDirectory: '$(Pipeline.Workspace)/build/onnxruntime_gpu' + workingDirectory: '$(Pipeline.Workspace)/build/whl' displayName: 'Uploading wheels to ${{ parameters.artifact_feed }}' From 75f47699656ab151c6fafb4839b7b8e0364e73ca Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 06:17:47 +0000 Subject: [PATCH 06/10] update --- .../github/azure-pipelines/stages/py-linux-gpu-stage.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index da8fbe40716ca..60b2e04e82136 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -39,13 +39,6 @@ stages: name: ${{ parameters.machine_pool }} os: linux templateContext: - sdl: - binskim: - preReleaseVersion: '4.3.1' - # The tool failed with error: No usable version of libssl was found - enabled: false - scanOutputDirectoryOnly: true - analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)/**/*.so' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory)/dist From 3e9b425ab5b7ad5ea0d4444e51a5280c7457d21e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 06:29:13 +0000 Subject: [PATCH 07/10] update --- .../github/azure-pipelines/stages/py-win-gpu-stage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 3db56c16ab870..caa733d8b9ee1 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -89,7 +89,7 @@ stages: - checkout: self clean: true - submodules: recursive + submodules: none - template: ../templates/telemetry-steps.yml @@ -134,7 +134,7 @@ stages: --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests - --parallel 8 --use_binskim_compliant_compile_flags --update --build + --parallel 8 --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40 $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }} workingDirectory: '$(Build.BinariesDirectory)' From eb180250a108e134d63bb9b05f15ef81f0675286 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 17:27:36 +0000 Subject: [PATCH 08/10] update --- .../azure-pipelines/py-cuda-alt-package-test-pipeline.yml | 2 +- .../github/azure-pipelines/py-cuda-package-test-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml index 8f26d1ae4976a..019643b004ce0 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml @@ -22,7 +22,7 @@ stages: trt_version: '10.7.0.23-1.cuda11.8' cuda_version: '11.8' - - stage: Linux_Test_CUDA_Alt_x86_64_stage + - stage: Republish_Wheels dependsOn: jobs: - job: Python_Publishing_GPU diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml index b0a40cf31912a..184620812d097 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml @@ -21,7 +21,7 @@ stages: docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241120.3 cuda_version: '12.2' - - stage: Linux_Test_CUDA_Alt_x86_64_stage + - stage: Republish_Wheels dependsOn: jobs: - job: Python_Publishing_GPU From 3b784bb01f61d20b353b0dd893f2736f3c3e62d2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 4 Jan 2025 19:51:19 +0000 Subject: [PATCH 09/10] update --- .../github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml index 019643b004ce0..5edf6e93c9284 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml @@ -19,7 +19,6 @@ stages: python_wheel_suffix: '_gpu' timeout: 480 docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241120.3 - trt_version: '10.7.0.23-1.cuda11.8' cuda_version: '11.8' - stage: Republish_Wheels From 352279ad59cf9f9726bd576bde29466faca59bad Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sun, 5 Jan 2025 03:17:10 +0000 Subject: [PATCH 10/10] revert --- tools/ci_build/requirements/pybind/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/requirements/pybind/requirements.txt b/tools/ci_build/requirements/pybind/requirements.txt index 870d219d0f892..8f00a25627c21 100644 --- a/tools/ci_build/requirements/pybind/requirements.txt +++ b/tools/ci_build/requirements/pybind/requirements.txt @@ -1,7 +1,7 @@ setuptools wheel pytest -numpy +numpy>=1.19.0 sympy>=1.10 packaging cerberus