From c7691650d7e737bef32dd2fdd009519a3dcd2274 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 24 Apr 2024 13:01:06 +0800 Subject: [PATCH] Fix CI tests (#804) --- .github/workflows/run-python-test-macos.yaml | 105 ++++++++++++++ .github/workflows/run-python-test.yaml | 128 ++++++++++++++---- .github/workflows/test-build-wheel.yaml | 46 ++++++- .github/workflows/test-pip-install.yaml | 47 ++++++- .../test-python-offline-websocket-server.yaml | 4 +- .../test-python-online-websocket-server.yaml | 4 +- cmake/cmake_extension.py | 17 ++- 7 files changed, 311 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/run-python-test-macos.yaml diff --git a/.github/workflows/run-python-test-macos.yaml b/.github/workflows/run-python-test-macos.yaml new file mode 100644 index 000000000..af973ae31 --- /dev/null +++ b/.github/workflows/run-python-test-macos.yaml @@ -0,0 +1,105 @@ +name: run-python-test-macos + +on: + push: + branches: + - master + paths: + - '.github/workflows/run-python-test-macos.yaml' + - '.github/scripts/test-python.sh' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'python-api-examples/**' + pull_request: + branches: + - master + paths: + - '.github/workflows/run-python-test-macos.yaml' + - '.github/scripts/test-python.sh' + - 'CMakeLists.txt' + - 'cmake/**' + - 'sherpa-onnx/csrc/*' + - 'python-api-examples/**' + workflow_dispatch: + +concurrency: + group: run-python-test-macos-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + run-python-test: + name: ${{ matrix.os }} ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # See https://github.com/actions/runner-images + # macos-14 is for arm64 + # macos-14-large is for x64 + include: + - os: macos-11 + python-version: "3.7" + + - os: macos-12 + python-version: "3.8" + + - os: macos-13 + python-version: "3.9" + - os: macos-13 + python-version: "3.10" + - os: macos-13 + python-version: "3.11" + + - os: macos-14 + python-version: "3.12" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Display OS version + shell: bash + run: | + uname -a + sw_vers + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }}-python-${{ matrix.python-version }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip numpy pypinyin sentencepiece>=0.1.96 soundfile + + - name: Install sherpa-onnx + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --version + + python3 -m pip install --verbose . + + - name: Test sherpa-onnx + shell: bash + run: | + export OS=${{ matrix.os }} + .github/scripts/test-python.sh + .github/scripts/test-speaker-recognition-python.sh + + - uses: actions/upload-artifact@v4 + with: + name: tts-generated-test-files-${{ matrix.os }}-${{ matrix.python-version }} + path: tts diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml index ec6a10916..c7433e0e9 100644 --- a/.github/workflows/run-python-test.yaml +++ b/.github/workflows/run-python-test.yaml @@ -30,9 +30,6 @@ concurrency: permissions: contents: read -env: - SCCACHE_GHA_ENABLED: "true" - jobs: run-python-test: name: ${{ matrix.os }} ${{ matrix.python-version }} @@ -40,33 +37,37 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.7", "3.8", "3.9", "3.10","3.11", "3.12"] - exclude: - - os: macos-latest + include: + - os: ubuntu-20.04 + python-version: "3.7" + - os: ubuntu-20.04 + python-version: "3.8" + - os: ubuntu-20.04 python-version: "3.9" - - os: macos-latest + + - os: ubuntu-22.04 python-version: "3.10" - # - os: windows-latest - # python-version: "3.12" + - os: ubuntu-22.04 + python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Display OS version + shell: bash + run: | + uname -a + find "/etc" -maxdepth 1 -type f -name "*version" -exec head -n 100 {} \; + - name: ccache - if: matrix.os != 'windows-latest' uses: hendrikmuhs/ccache-action@v1.2 with: key: ${{ matrix.os }}-python-${{ matrix.python-version }} - - name: Run sccache-cache - if: matrix.os == 'windows-latest' - uses: mozilla-actions/sccache-action@v0.0.4 - with: - version: "v0.7.4" - - name: Setup Python uses: actions/setup-python@v5 with: @@ -76,31 +77,108 @@ jobs: shell: bash run: | python3 -m pip install --upgrade pip numpy pypinyin sentencepiece>=0.1.96 soundfile + python3 -m pip install wheel twine setuptools - - name: Install sherpa-onnx - if: matrix.os != 'windows-latest' + - name: Install ninja + shell: bash + run: | + sudo apt-get install ninja-build + + - name: Display ninja version + shell: bash + run: | + ninja --version + ninja --help || true + which ninja + + - name: Display site packages dir + shell: bash + run: | + python3 -c 'import site; print(site.getsitepackages())' + p=$(python3 -c 'import site; print(site.getsitepackages())') + echo "p: $p" + + - name: Install patchelf + shell: bash + run: | + sudo apt-get update -q + sudo apt-get install -q -y patchelf + patchelf --help + + - name: Build sherpa-onnx shell: bash run: | export CMAKE_CXX_COMPILER_LAUNCHER=ccache export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" cmake --version + export SHERPA_ONNX_CMAKE_ARGS="-G Ninja -DCMAKE_BUILD_TYPE=Release" + export SHERPA_ONNX_MAKE_ARGS="-j 6" - python3 -m pip install --verbose . + python3 setup.py bdist_wheel + + - name: Patch wheels + shell: bash + run: | + mkdir ./dist2 + sudo ./scripts/wheel/patch_wheel.py --in-dir ./dist --out-dir ./dist2 - name: Install sherpa-onnx - if: matrix.os == 'windows-latest' shell: bash run: | - cmake --version - export SHERPA_ONNX_CMAKE_ARGS="-DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache" - echo "SHERPA_ONNX_CMAKE_ARGS: ${SHERPA_ONNX_CMAKE_ARGS}" + ls -lh dist2 + + python3 -m pip install ./dist2/*.whl + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.python-version }}-whl + path: ./dist - python3 -m pip install --verbose . + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.python-version }}-whl-patched + path: ./dist2 + + - name: Show dependencies + shell: bash + run: | + cd dist + mkdir t + cd t + unzip ../*.whl + readelf -d _sherpa_onnx*.so + + echo "----" + + readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx + + - name: Show dependencies (patched) + shell: bash + run: | + cd dist2 + mkdir t + cd t + unzip ../*.whl + readelf -d _sherpa_onnx*.so + + echo "----" + + readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx - name: Test sherpa-onnx shell: bash run: | export OS=${{ matrix.os }} + + p=$(python3 -c 'import site; print(site.getsitepackages()[0])') + echo "p: $p" + p=$p/sherpa_onnx/lib + echo "p: $p" + ls -lh $p + + export LD_LIBRARY_PATH=$p:$LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + .github/scripts/test-python.sh .github/scripts/test-speaker-recognition-python.sh diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml index 0100e10b1..2c6070693 100644 --- a/.github/workflows/test-build-wheel.yaml +++ b/.github/workflows/test-build-wheel.yaml @@ -35,8 +35,50 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # See https://github.com/actions/runner-images + include: + - os: ubuntu-20.04 + python-version: "3.7" + - os: ubuntu-20.04 + python-version: "3.8" + - os: ubuntu-20.04 + python-version: "3.9" + - os: ubuntu-22.04 + python-version: "3.10" + - os: ubuntu-22.04 + python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" + + - os: macos-11 + python-version: "3.7" + + - os: macos-12 + python-version: "3.8" + + - os: macos-13 + python-version: "3.9" + - os: macos-13 + python-version: "3.10" + - os: macos-13 + python-version: "3.11" + + - os: macos-14 + python-version: "3.12" + + - os: windows-2019 + python-version: "3.7" + - os: windows-2019 + python-version: "3.8" + - os: windows-2019 + python-version: "3.9" + + - os: windows-2022 + python-version: "3.10" + - os: windows-2022 + python-version: "3.11" + - os: windows-2022 + python-version: "3.12" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml index ca1fdbd8e..998469cef 100644 --- a/.github/workflows/test-pip-install.yaml +++ b/.github/workflows/test-pip-install.yaml @@ -28,11 +28,50 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - exclude: - - os: macos-latest + # See https://github.com/actions/runner-images + include: + - os: ubuntu-20.04 python-version: "3.7" + - os: ubuntu-20.04 + python-version: "3.8" + - os: ubuntu-20.04 + python-version: "3.9" + - os: ubuntu-22.04 + python-version: "3.10" + - os: ubuntu-22.04 + python-version: "3.11" + - os: ubuntu-22.04 + python-version: "3.12" + + - os: macos-11 + python-version: "3.7" + + - os: macos-12 + python-version: "3.8" + + - os: macos-13 + python-version: "3.9" + - os: macos-13 + python-version: "3.10" + - os: macos-13 + python-version: "3.11" + + - os: macos-14 + python-version: "3.12" + + - os: windows-2019 + python-version: "3.7" + - os: windows-2019 + python-version: "3.8" + - os: windows-2019 + python-version: "3.9" + + - os: windows-2022 + python-version: "3.10" + - os: windows-2022 + python-version: "3.11" + - os: windows-2022 + python-version: "3.12" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-python-offline-websocket-server.yaml b/.github/workflows/test-python-offline-websocket-server.yaml index 349e0b61d..18183f482 100644 --- a/.github/workflows/test-python-offline-websocket-server.yaml +++ b/.github/workflows/test-python-offline-websocket-server.yaml @@ -35,8 +35,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-20.04, ubuntu-22.04, windows-latest, macos-latest, macos-14] + python-version: ["3.10"] model_type: ["transducer", "paraformer", "nemo_ctc", "whisper", "tdnn"] steps: diff --git a/.github/workflows/test-python-online-websocket-server.yaml b/.github/workflows/test-python-online-websocket-server.yaml index 1f4096c0f..888af36ae 100644 --- a/.github/workflows/test-python-online-websocket-server.yaml +++ b/.github/workflows/test-python-online-websocket-server.yaml @@ -35,8 +35,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-20.04, ubuntu-22.04, windows-latest, macos-latest, macos-14] + python-version: ["3.10"] model_type: ["transducer", "paraformer", "zipformer2-ctc"] steps: diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 163276135..b207a4171 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -180,13 +180,20 @@ def build_extension(self, ext: setuptools.extension.Extension): print('Setting make_args to "-j4"') make_args = "-j4" - build_cmd = f""" - cd {self.build_temp} + if "-G Ninja" in cmake_args: + build_cmd = f""" + cd {self.build_temp} + cmake {cmake_args} {sherpa_onnx_dir} + ninja {make_args} install + """ + else: + build_cmd = f""" + cd {self.build_temp} - cmake {cmake_args} {sherpa_onnx_dir} + cmake {cmake_args} {sherpa_onnx_dir} - make {make_args} install/strip - """ + make {make_args} install/strip + """ print(f"build command is:\n{build_cmd}") ret = os.system(build_cmd)