diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml index cf434e4eadf0d..5edf6e93c9284 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml @@ -19,6 +19,39 @@ stages: python_wheel_suffix: '_gpu' timeout: 480 docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241120.3 - trt_version: '10.7.0.23-1.cuda11.8' cuda_version: '11.8' + - stage: Republish_Wheels + dependsOn: + jobs: + - job: Python_Publishing_GPU + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + steps: + - checkout: none + - download: build + displayName: 'Download Pipeline Artifact - onnxruntime_gpu' + artifact: 'onnxruntime_gpu' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.10' + artifact: 'win_gpu_wheel_3.10' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.11' + artifact: 'win_gpu_wheel_3.11' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.12' + artifact: 'win_gpu_wheel_3.12' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.13' + artifact: 'win_gpu_wheel_3.13' + patterns: '*.whl' + + - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; + displayName: 'Merge files together' + + - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu + artifact: whl + displayName: Republish artifacts \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml index 93a38b212d934..80202928789cf 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml @@ -29,6 +29,12 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: sdl: + componentgovernance: + ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)' + alertWarningLevel: High + failOnAlert: false + verbosity: Normal + timeout: 3600 tsa: enabled: true codeSignValidation: diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml index 307415b7be16f..184620812d097 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml @@ -21,3 +21,37 @@ stages: docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241120.3 cuda_version: '12.2' + - stage: Republish_Wheels + dependsOn: + jobs: + - job: Python_Publishing_GPU + pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + steps: + - checkout: none + - download: build + displayName: 'Download Pipeline Artifact - onnxruntime_gpu' + artifact: 'onnxruntime_gpu' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.10' + artifact: 'win_gpu_wheel_3.10' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.11' + artifact: 'win_gpu_wheel_3.11' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.12' + artifact: 'win_gpu_wheel_3.12' + patterns: '*.whl' + - download: build + displayName: 'Download Pipeline Artifact - Win GPU 3.13' + artifact: 'win_gpu_wheel_3.13' + patterns: '*.whl' + + - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \; + displayName: 'Merge files together' + + - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu + artifact: whl + displayName: Republish artifacts \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml index 2e040698fad2a..a80527373e0d0 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml @@ -32,6 +32,12 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: sdl: + componentgovernance: + ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)' + alertWarningLevel: High + failOnAlert: false + verbosity: Normal + timeout: 3600 tsa: enabled: true codeSignValidation: diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml index 1217163c07132..e4499705efdf5 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - pipeline: build - source: 'Python-CUDA-Packaging-Pipeline' + source: 'Python CUDA12 Package Test Pipeline' trigger: branches: include: diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml index 85bd5de5b7eb1..1ff43667f4788 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml @@ -11,20 +11,21 @@ stages: steps: - checkout: none - download: build - displayName: 'Download Pipeline Artifact - onnxruntime_gpu' - artifact: 'onnxruntime_gpu' + displayName: 'Download Pipeline Artifact' + artifact: 'whl' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.13' displayName: 'Use Python 3.x' - - script: 'pip install twine==3.4.2' + - script: 'pip install twine==6.0.1' displayName: 'Install Twine' - task: TwineAuthenticate@1 displayName: 'Twine Authenticate ' inputs: artifactFeed: PublicPackages/${{ parameters.artifact_feed }} - - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive --skip-existing *.whl' - workingDirectory: '$(Pipeline.Workspace)/build/onnxruntime_gpu' + + - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive *.whl' + workingDirectory: '$(Pipeline.Workspace)/build/whl' displayName: 'Uploading wheels to ${{ parameters.artifact_feed }}' - retryCountOnTaskFailure: 3 - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index 89f6d8b7923cf..60b2e04e82136 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -39,16 +39,6 @@ stages: name: ${{ parameters.machine_pool }} os: linux templateContext: - codeSignValidation: - enabled: true - break: true - psscriptanalyzer: - enabled: true - sdl: - binskim: - enabled: true - scanOutputDirectoryOnly: true - targetPathPattern: '\".*.so\"' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory)/dist @@ -57,9 +47,6 @@ stages: targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.cmake_build_type }} artifactName: linux_gpu_wheel_${{ parameters.arch }} variables: - # The build machine pool doesn't have dotnet, so it can't run CG. - - name: skipComponentGovernanceDetection - value: true - name: extra_build_args ${{ if ne(parameters.extra_build_arg, '') }}: value: -x ${{ parameters.extra_build_arg }} diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 548641d1985b8..caa733d8b9ee1 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -50,16 +50,19 @@ stages: name: onnxruntime-Win-CPU-2022 os: windows templateContext: - codeSignValidation: - enabled: true - break: true - psscriptanalyzer: - enabled: true sdl: + codeSignValidation: + enabled: true + # TODO: check why pyd file was not signed + break: false + additionalTargetsGlobPattern: f|**\*.pyd + psscriptanalyzer: + enabled: true binskim: + preReleaseVersion: '4.3.1' enabled: true scanOutputDirectoryOnly: true - targetPathPattern: '+:file|*.dll;-:file|DirectML.dll' + analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)\**\*.pyd;+:file|$(BUILD.STAGINGDIRECTORY)\**\*.dll;-:file|DirectML.dll' outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory) @@ -86,7 +89,7 @@ stages: - checkout: self clean: true - submodules: recursive + submodules: none - template: ../templates/telemetry-steps.yml @@ -121,7 +124,7 @@ stages: - template: ../templates/set-nightly-build-option-variable-step.yml - task: PythonScript@0 - displayName: 'Generate cmake config' + displayName: 'Build' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' arguments: > @@ -131,7 +134,7 @@ stages: --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests - --parallel --use_binskim_compliant_compile_flags --update --build + --parallel 8 --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40 $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }} workingDirectory: '$(Build.BinariesDirectory)' @@ -162,9 +165,6 @@ stages: workingDirectory: '$(Build.ArtifactStagingDirectory)' displayName: 'unzip the package' - - template: ../templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build diff --git a/tools/ci_build/github/windows/eager/requirements.txt b/tools/ci_build/github/windows/eager/requirements.txt deleted file mode 100644 index d118280c8e6d6..0000000000000 --- a/tools/ci_build/github/windows/eager/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -setuptools -wheel -numpy==1.21.6 ; python_version < '3.9' -numpy==2.1.2 ; python_version >= '3.9' -typing_extensions -torch==2.2.0 -parameterized