From 38ad05bdf8163e31275288d4e9945508d34cbf25 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 20 Nov 2023 12:33:06 +0800 Subject: [PATCH] Refactor building wheels (#436) --- .github/workflows/.gitignore | 1 + .../workflows/aarch64-linux-gnu-shared.yaml | 4 +- .github/workflows/build-wheels-aarch64.yaml | 7 +- .github/workflows/build-wheels-armv7l.yaml | 1 + ...ld-wheels.yaml => build-wheels-linux.yaml} | 46 +++++-- .../workflows/build-wheels-macos-arm64.yaml | 104 ++++++++++++++++ .github/workflows/build-wheels-macos-x64.yaml | 104 ++++++++++++++++ .github/workflows/build-wheels-macos.yaml | 62 ---------- .github/workflows/build-wheels-win32.yaml | 36 +++++- .github/workflows/build-wheels-win64.yaml | 117 ++++++++++++++++++ 10 files changed, 404 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/.gitignore rename .github/workflows/{build-wheels.yaml => build-wheels-linux.yaml} (61%) create mode 100644 .github/workflows/build-wheels-macos-arm64.yaml create mode 100644 .github/workflows/build-wheels-macos-x64.yaml delete mode 100644 .github/workflows/build-wheels-macos.yaml create mode 100644 .github/workflows/build-wheels-win64.yaml diff --git a/.github/workflows/.gitignore b/.github/workflows/.gitignore new file mode 100644 index 000000000..545d9ab4f --- /dev/null +++ b/.github/workflows/.gitignore @@ -0,0 +1 @@ +!*.yaml diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index 5835345db..ddafe08a9 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -142,7 +142,7 @@ jobs: shell: bash run: | export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH - aarch64-none-linux-gnu-strip --version + aarch64-linux-gnu-strip --version SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) @@ -157,7 +157,7 @@ jobs: ls -lh $dst/bin/ echo "strip" - aarch64-none-linux-gnu-strip $dst/bin/* + aarch64-linux-gnu-strip $dst/bin/* tree $dst diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index c75a32243..d88947c9e 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -17,12 +17,13 @@ concurrency: jobs: build_wheels_aarch64: - name: Build wheels for aarch64 on ${{ matrix.os }} + name: ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] steps: - uses: actions/checkout@v4 @@ -35,9 +36,9 @@ jobs: # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ # for a list of versions - name: Build wheels - uses: pypa/cibuildwheel@v2.11.4 + uses: pypa/cibuildwheel@v2.15.0 env: - CIBW_BEFORE_BUILD: "pip install -U cmake numpy" + CIBW_BUILD: "${{ matrix.python-version}}-* " CIBW_SKIP: "cp27-* cp35-* cp36-* *-win32 pp* *-musllinux* *-manylinux_i686" CIBW_BUILD_VERBOSITY: 3 CIBW_ARCHS_LINUX: aarch64 diff --git a/.github/workflows/build-wheels-armv7l.yaml b/.github/workflows/build-wheels-armv7l.yaml index 910d04db9..07a72b969 100644 --- a/.github/workflows/build-wheels-armv7l.yaml +++ b/.github/workflows/build-wheels-armv7l.yaml @@ -45,6 +45,7 @@ jobs: echo PYTHON_VERSION=$PYTHON_VERSION >> $GITHUB_ENV # https://github.com/mshr-h/onnx-dockerfile-for-raspberry-pi/blob/main/3.10-bullseye-build/Dockerfile.arm32v7 + # https://hub.docker.com/r/balenalib/raspberrypi3-python - name: Run docker uses: addnab/docker-run-action@v3 with: diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels-linux.yaml similarity index 61% rename from .github/workflows/build-wheels.yaml rename to .github/workflows/build-wheels-linux.yaml index 0384c3271..c1c51666b 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -1,4 +1,4 @@ -name: build-wheels +name: build-wheels-linux on: push: @@ -16,13 +16,14 @@ concurrency: cancel-in-progress: true jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} + build_wheels_linux: + name: ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] steps: - uses: actions/checkout@v4 @@ -30,9 +31,9 @@ jobs: # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ # for a list of versions - name: Build wheels - uses: pypa/cibuildwheel@v2.11.4 + uses: pypa/cibuildwheel@v2.15.0 env: - CIBW_BEFORE_BUILD: "pip install -U cmake numpy" + CIBW_BUILD: "${{ matrix.python-version}}-* " CIBW_SKIP: "cp27-* cp35-* cp36-* *-win32 pp* *-musllinux* *-manylinux_i686" CIBW_BUILD_VERBOSITY: 3 CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH='/project/build/bdist.linux-x86_64/wheel/sherpa_onnx/lib' @@ -65,6 +66,35 @@ jobs: with: path: ./wheelhouse/*.whl + - name: Publish to huggingface + if: matrix.python-version == 'cp38' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../wheelhouse/*.whl . + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + - name: Publish wheels to PyPI env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} @@ -76,14 +106,14 @@ jobs: twine upload ./wheelhouse/*.whl - name: Build sdist - if: ${{ matrix.os == 'ubuntu-latest' }} + if: matrix.python-version == 'cp38' shell: bash run: | python3 setup.py sdist ls -l dist/* - name: Publish sdist to PyPI - if: ${{ matrix.os == 'ubuntu-latest' }} + if: matrix.python-version == 'cp38' env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml new file mode 100644 index 000000000..a58a26b0d --- /dev/null +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -0,0 +1,104 @@ +name: build-wheels-macos-arm64 + +on: + push: + branches: + - wheel + tags: + - '*' + workflow_dispatch: + +env: + SHERPA_ONNX_IS_IN_GITHUB_ACTIONS: 1 + +concurrency: + group: build-wheels-macos-arm64-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + + steps: + - uses: actions/checkout@v4 + + # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ + # for a list of versions + - name: Build wheels + if: matrix.python-version == 'cp37' + uses: pypa/cibuildwheel@v2.11.4 + env: + CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='arm64'" + CIBW_ARCHS: "arm64" + CIBW_BUILD_VERBOSITY: 3 + + # Don't repair macOS wheels + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + + - name: Build wheels + if: matrix.python-version != 'cp37' + uses: pypa/cibuildwheel@v2.15.0 + env: + CIBW_BUILD: "${{ matrix.python-version}}-* " + CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='arm64'" + CIBW_ARCHS: "arm64" + CIBW_BUILD_VERBOSITY: 3 + + # Don't repair macOS wheels + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + + - name: Display wheels + shell: bash + run: | + ls -lh ./wheelhouse/ + + ls -lh ./wheelhouse/*.whl + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + - name: Publish to huggingface + if: matrix.python-version == 'cp38' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../wheelhouse/*.whl . + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + + - name: Publish wheels to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python3 -m pip install --upgrade pip + python3 -m pip install wheel twine setuptools + + twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml new file mode 100644 index 000000000..a15d84cee --- /dev/null +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -0,0 +1,104 @@ +name: build-wheels-macos-x64 + +on: + push: + branches: + - wheel + tags: + - '*' + workflow_dispatch: + +env: + SHERPA_ONNX_IS_IN_GITHUB_ACTIONS: 1 + +concurrency: + group: build-wheels-macos-x64-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + + steps: + - uses: actions/checkout@v4 + + # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ + # for a list of versions + - name: Build wheels + if: matrix.python-version == 'cp37' + uses: pypa/cibuildwheel@v2.11.4 + env: + CIBW_BUILD: "${{ matrix.python-version}}-* " + CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='x86_64'" + CIBW_ARCHS: "x86_64" + CIBW_BUILD_VERBOSITY: 3 + + # Don't repair macOS wheels + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + - name: Build wheels + if: matrix.python-version != 'cp37' + uses: pypa/cibuildwheel@v2.15.0 + env: + CIBW_BUILD: "${{ matrix.python-version}}-* " + CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='x86_64'" + CIBW_ARCHS: "x86_64" + CIBW_BUILD_VERBOSITY: 3 + + # Don't repair macOS wheels + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + + - name: Display wheels + shell: bash + run: | + ls -lh ./wheelhouse/ + + ls -lh ./wheelhouse/*.whl + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + - name: Publish to huggingface + if: matrix.python-version == 'cp38' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../wheelhouse/*.whl . + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + + - name: Publish wheels to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python3 -m pip install --upgrade pip + python3 -m pip install wheel twine setuptools + + twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-macos.yaml b/.github/workflows/build-wheels-macos.yaml deleted file mode 100644 index 56d3d2e5f..000000000 --- a/.github/workflows/build-wheels-macos.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: build-wheels-macos - -on: - push: - branches: - - wheel - tags: - - '*' - workflow_dispatch: - -env: - SHERPA_ONNX_IS_IN_GITHUB_ACTIONS: 1 - -concurrency: - group: build-wheels-macos-${{ github.ref }} - cancel-in-progress: true - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest] - - steps: - - uses: actions/checkout@v4 - - # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ - # for a list of versions - - name: Build wheels - uses: pypa/cibuildwheel@v2.11.4 - env: - CIBW_BEFORE_BUILD: "pip install -U cmake numpy" - CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES='arm64;x86_64'" - CIBW_ARCHS: "universal2" - CIBW_BUILD_VERBOSITY: 3 - - # Don't repair macOS wheels - CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" - - - name: Display wheels - shell: bash - run: | - ls -lh ./wheelhouse/ - - ls -lh ./wheelhouse/*.whl - - - uses: actions/upload-artifact@v3 - with: - path: ./wheelhouse/*.whl - - - name: Publish wheels to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python3 -m pip install --upgrade pip - python3 -m pip install wheel twine setuptools - - twine upload ./wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-win32.yaml b/.github/workflows/build-wheels-win32.yaml index f17b771de..9452ea372 100644 --- a/.github/workflows/build-wheels-win32.yaml +++ b/.github/workflows/build-wheels-win32.yaml @@ -17,12 +17,13 @@ concurrency: jobs: build_wheels_win32: - name: Build wheels on ${{ matrix.os }} + name: ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [windows-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] steps: - uses: actions/checkout@v4 @@ -30,10 +31,10 @@ jobs: # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ # for a list of versions - name: Build wheels - uses: pypa/cibuildwheel@v2.11.4 + uses: pypa/cibuildwheel@v2.15.0 env: - CIBW_BEFORE_BUILD: "pip install -U cmake numpy" CIBW_ENVIRONMENT: SHERPA_ONNX_CMAKE_ARGS="-A Win32" + CIBW_BUILD: "${{ matrix.python-version}}-* " CIBW_SKIP: "*-win_amd64" CIBW_BUILD_VERBOSITY: 3 @@ -48,6 +49,35 @@ jobs: with: path: ./wheelhouse/*.whl + - name: Publish to huggingface + if: matrix.python-version == 'cp38' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../wheelhouse/*.whl . + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + - name: Publish wheels to PyPI env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} diff --git a/.github/workflows/build-wheels-win64.yaml b/.github/workflows/build-wheels-win64.yaml new file mode 100644 index 000000000..fe711953d --- /dev/null +++ b/.github/workflows/build-wheels-win64.yaml @@ -0,0 +1,117 @@ +name: build-wheels-win64 + +on: + push: + branches: + - wheel + tags: + - '*' + workflow_dispatch: + +env: + SHERPA_ONNX_IS_IN_GITHUB_ACTIONS: 1 + +concurrency: + group: build-wheels-win64-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_wheels_win64: + name: ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest] + python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"] + + steps: + - uses: actions/checkout@v4 + + # see https://cibuildwheel.readthedocs.io/en/stable/changelog/ + # for a list of versions + - name: Build wheels + uses: pypa/cibuildwheel@v2.15.0 + env: + CIBW_BUILD: "${{ matrix.python-version}}-* " + CIBW_SKIP: "cp27-* cp35-* *-win32 pp* *-musllinux*" + CIBW_BUILD_VERBOSITY: 3 + + - name: Display wheels + shell: bash + run: | + ls -lh ./wheelhouse/ + + ls -lh ./wheelhouse/*.whl + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + - name: Publish to huggingface + if: matrix.python-version == 'cp38' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../wheelhouse/*.whl . + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + + - name: Publish to huggingface + if: matrix.python-version == 'cp38' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v2 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface + cd huggingface + git fetch + git pull + git merge -m "merge remote" --ff origin main + + cp -v ../wheelhouse/*.whl . + + git status + git add . + git commit -m "add more wheels" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main + + - name: Publish wheels to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python3 -m pip install --upgrade pip + python3 -m pip install wheel twine setuptools + + twine upload ./wheelhouse/*.whl