From e155c66b4a3df8a173ce3b16b31707aefec7b052 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 4 Jan 2024 17:44:49 -0800 Subject: [PATCH] Change all macOS python packages to use universal2 (#19013) ### Description Change all macOS python packages to use universal2, to reduce the number of packages we have. ### Motivation and Context According to [wikipedia](https://en.wikipedia.org/wiki/MacOS_Big_Sur), macOS 11 is the first macOS version that supports universal 2. And it is the min macOS version we support. So we no longer need to maintain separate binaries for different CPU archs. --- .../templates/py-packaging-stage.yml | 95 +------------------ .../linux/docker/scripts/requirements.txt | 4 +- 2 files changed, 5 insertions(+), 94 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index f2b91bbaacb89..44904f9248b10 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -356,96 +356,11 @@ stages: inputs: versionSpec: $(PythonVersion) - - template: use-xcode-version.yml - - script: | set -e -x - pushd . - mkdir -p /tmp/scripts - mkdir -p $(Build.BinariesDirectory)/installed - cp $(Build.SourcesDirectory)/cmake/deps.txt /tmp/scripts - $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -p $(Build.BinariesDirectory)/installed - popd - export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH - export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-x86_64 + export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-universal2 python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --config Release --skip_onnx_tests --build_wheel ${{ parameters.build_py_parameters }} - displayName: 'Command Line Script' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)/Release/dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - ${{ if eq(parameters.enable_mac_silicon, true) }}: - - job: MacOS_silicon_py_Wheels - timeoutInMinutes: 120 - workspace: - clean: all - pool: - vmImage: 'macOS-13' - variables: - MACOSX_DEPLOYMENT_TARGET: '11.0' - strategy: - # As of 3.9.1, Python now fully supports building and running on macOS 11.0 (Big Sur) and on Apple Silicon Macs (based on the ARM64 architecture). - # https://docs.python.org/3/whatsnew/3.9.html - matrix: - Python38: - PythonVersion: '3.8' - Python39: - PythonVersion: '3.9' - Python310: - PythonVersion: '3.10' - Python311: - PythonVersion: '3.11' - steps: - - checkout: self - clean: true - submodules: recursive - - - task: UsePythonVersion@0 - displayName: 'Use Python' - inputs: - versionSpec: $(PythonVersion) - - - script: | - set -ex - uname -m - system_profiler SPSoftwareDataType SPHardwareDataType - displayName: 'Mac machine info' - - - template: use-xcode-version.yml - - # Don't remove _PYTHON_HOST_PLATFORM, it's used to generate correct package name - # Setting _PYTHON_HOST_PLATFORM overwrites the value return by get_platform() - # Ref: https://wiki.debian.org/Python/MultiArch - - script: | - set -e -x - pushd . - mkdir -p /tmp/scripts - mkdir -p $(Build.BinariesDirectory)/installed - cp $(Build.SourcesDirectory)/cmake/deps.txt /tmp/scripts - $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -p $(Build.BinariesDirectory)/installed - popd - export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH - export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-arm64 - python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --config Release --skip_tests --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64 --build_wheel ${{ parameters.build_py_parameters }} + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build displayName: 'Command Line Script' - script: | @@ -454,13 +369,9 @@ stages: cd '$(Build.BinariesDirectory)/Release/dist' ls for file in *.whl - do - [[ "$file" == *arm64* ]] || ( echo "Mac Silicon package name is NOT correct" && exit 1) - done - for file in *.whl do delocate-listdeps "$file" - delocate-wheel --require-archs=arm64 -w fixed_wheels -v "$file" + delocate-wheel --require-archs=x86_64,arm64 -w fixed_wheels -v "$file" done displayName: 'delocate wheel' diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index a6452721a2b7d..0fc80b30c1b3a 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -3,8 +3,8 @@ numpy==1.21.6 ; python_version < '3.11' numpy==1.24.2 ; python_version >= '3.11' mypy pytest -setuptools>=68.2.2 -wheel>=0.35.1 +setuptools==69.0.3 +wheel==0.42.0 onnx==1.15.0 argparse sympy==1.12