From 2cb3c7af8520361f92849bd14e796a132e24bf81 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 24 Apr 2024 10:38:33 +0800 Subject: [PATCH] Use Ninja --- .github/workflows/run-python-test-macos.yaml | 4 +- .github/workflows/run-python-test.yaml | 94 ++++++++++++++++++- .github/workflows/test-build-wheel.yaml | 46 ++++++++- .github/workflows/test-pip-install.yaml | 47 +++++++++- .../test-python-offline-websocket-server.yaml | 4 +- .../test-python-online-websocket-server.yaml | 4 +- cmake/cmake_extension.py | 17 +++- 7 files changed, 196 insertions(+), 20 deletions(-) diff --git a/.github/workflows/run-python-test-macos.yaml b/.github/workflows/run-python-test-macos.yaml index 9ac3d5ba2..af973ae31 100644 --- a/.github/workflows/run-python-test-macos.yaml +++ b/.github/workflows/run-python-test-macos.yaml @@ -51,9 +51,9 @@ jobs: python-version: "3.9" - os: macos-13 python-version: "3.10" - - - os: macos-14 + - os: macos-13 python-version: "3.11" + - os: macos-14 python-version: "3.12" diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml index 29df8408a..c7433e0e9 100644 --- a/.github/workflows/run-python-test.yaml +++ b/.github/workflows/run-python-test.yaml @@ -61,7 +61,7 @@ jobs: shell: bash run: | uname -a - head -n 100 /etc/*version + find "/etc" -maxdepth 1 -type f -name "*version" -exec head -n 100 {} \; - name: ccache uses: hendrikmuhs/ccache-action@v1.2 @@ -77,20 +77,108 @@ jobs: shell: bash run: | python3 -m pip install --upgrade pip numpy pypinyin sentencepiece>=0.1.96 soundfile + python3 -m pip install wheel twine setuptools - - name: Install sherpa-onnx + - name: Install ninja + shell: bash + run: | + sudo apt-get install ninja-build + + - name: Display ninja version + shell: bash + run: | + ninja --version + ninja --help || true + which ninja + + - name: Display site packages dir + shell: bash + run: | + python3 -c 'import site; print(site.getsitepackages())' + p=$(python3 -c 'import site; print(site.getsitepackages())') + echo "p: $p" + + - name: Install patchelf + shell: bash + run: | + sudo apt-get update -q + sudo apt-get install -q -y patchelf + patchelf --help + + - name: Build sherpa-onnx shell: bash run: | export CMAKE_CXX_COMPILER_LAUNCHER=ccache export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" cmake --version + export SHERPA_ONNX_CMAKE_ARGS="-G Ninja -DCMAKE_BUILD_TYPE=Release" + export SHERPA_ONNX_MAKE_ARGS="-j 6" + + python3 setup.py bdist_wheel + + - name: Patch wheels + shell: bash + run: | + mkdir ./dist2 + sudo ./scripts/wheel/patch_wheel.py --in-dir ./dist --out-dir ./dist2 + + - name: Install sherpa-onnx + shell: bash + run: | + ls -lh dist2 + + python3 -m pip install ./dist2/*.whl + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.python-version }}-whl + path: ./dist + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.python-version }}-whl-patched + path: ./dist2 + + - name: Show dependencies + shell: bash + run: | + cd dist + mkdir t + cd t + unzip ../*.whl + readelf -d _sherpa_onnx*.so + + echo "----" + + readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx + + - name: Show dependencies (patched) + shell: bash + run: | + cd dist2 + mkdir t + cd t + unzip ../*.whl + readelf -d _sherpa_onnx*.so + + echo "----" - python3 -m pip install --verbose . + readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx - name: Test sherpa-onnx shell: bash run: | export OS=${{ matrix.os }} + + p=$(python3 -c 'import site; print(site.getsitepackages()[0])') + echo "p: $p" + p=$p/sherpa_onnx/lib + echo "p: $p" + ls -lh $p + + export LD_LIBRARY_PATH=$p:$LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + .github/scripts/test-python.sh .github/scripts/test-speaker-recognition-python.sh diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml index 0100e10b1..2c6070693 100644 --- a/.github/workflows/test-build-wheel.yaml +++ b/.github/workflows/test-build-wheel.yaml @@ -35,8 +35,50 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # See https://github.com/actions/runner-images + include: + - os: ubuntu-20.04 + python-version: "3.7" + - os: ubuntu-20.04 + python-version: "3.8" + - os: ubuntu-20.04 + python-version: "3.9" + - os: ubuntu-22.04 + python-version: "3.10" + - os: ubuntu-22.04 + python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" + + - os: macos-11 + python-version: "3.7" + + - os: macos-12 + python-version: "3.8" + + - os: macos-13 + python-version: "3.9" + - os: macos-13 + python-version: "3.10" + - os: macos-13 + python-version: "3.11" + + - os: macos-14 + python-version: "3.12" + + - os: windows-2019 + python-version: "3.7" + - os: windows-2019 + python-version: "3.8" + - os: windows-2019 + python-version: "3.9" + + - os: windows-2022 + python-version: "3.10" + - os: windows-2022 + python-version: "3.11" + - os: windows-2022 + python-version: "3.12" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml index ca1fdbd8e..998469cef 100644 --- a/.github/workflows/test-pip-install.yaml +++ b/.github/workflows/test-pip-install.yaml @@ -28,11 +28,50 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - exclude: - - os: macos-latest + # See https://github.com/actions/runner-images + include: + - os: ubuntu-20.04 python-version: "3.7" + - os: ubuntu-20.04 + python-version: "3.8" + - os: ubuntu-20.04 + python-version: "3.9" + - os: ubuntu-22.04 + python-version: "3.10" + - os: ubuntu-22.04 + python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" + + - os: macos-11 + python-version: "3.7" + + - os: macos-12 + python-version: "3.8" + + - os: macos-13 + python-version: "3.9" + - os: macos-13 + python-version: "3.10" + - os: macos-13 + python-version: "3.11" + + - os: macos-14 + python-version: "3.12" + + - os: windows-2019 + python-version: "3.7" + - os: windows-2019 + python-version: "3.8" + - os: windows-2019 + python-version: "3.9" + + - os: windows-2022 + python-version: "3.10" + - os: windows-2022 + python-version: "3.11" + - os: windows-2022 + python-version: "3.12" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-python-offline-websocket-server.yaml b/.github/workflows/test-python-offline-websocket-server.yaml index 349e0b61d..18183f482 100644 --- a/.github/workflows/test-python-offline-websocket-server.yaml +++ b/.github/workflows/test-python-offline-websocket-server.yaml @@ -35,8 +35,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-20.04, ubuntu-22.04, windows-latest, macos-latest, macos-14] + python-version: ["3.10"] model_type: ["transducer", "paraformer", "nemo_ctc", "whisper", "tdnn"] steps: diff --git a/.github/workflows/test-python-online-websocket-server.yaml b/.github/workflows/test-python-online-websocket-server.yaml index 1f4096c0f..888af36ae 100644 --- a/.github/workflows/test-python-online-websocket-server.yaml +++ b/.github/workflows/test-python-online-websocket-server.yaml @@ -35,8 +35,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-20.04, ubuntu-22.04, windows-latest, macos-latest, macos-14] + python-version: ["3.10"] model_type: ["transducer", "paraformer", "zipformer2-ctc"] steps: diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 163276135..b207a4171 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -180,13 +180,20 @@ def build_extension(self, ext: setuptools.extension.Extension): print('Setting make_args to "-j4"') make_args = "-j4" - build_cmd = f""" - cd {self.build_temp} + if "-G Ninja" in cmake_args: + build_cmd = f""" + cd {self.build_temp} + cmake {cmake_args} {sherpa_onnx_dir} + ninja {make_args} install + """ + else: + build_cmd = f""" + cd {self.build_temp} - cmake {cmake_args} {sherpa_onnx_dir} + cmake {cmake_args} {sherpa_onnx_dir} - make {make_args} install/strip - """ + make {make_args} install/strip + """ print(f"build command is:\n{build_cmd}") ret = os.system(build_cmd)