Skip to content

Commit

Permalink
Use Ninja
Browse files Browse the repository at this point in the history
  • Loading branch information
csukuangfj committed Apr 24, 2024
1 parent d12a4c6 commit 2cb3c7a
Show file tree
Hide file tree
Showing 7 changed files with 196 additions and 20 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/run-python-test-macos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ jobs:
python-version: "3.9"
- os: macos-13
python-version: "3.10"

- os: macos-14
- os: macos-13
python-version: "3.11"

- os: macos-14
python-version: "3.12"

Expand Down
94 changes: 91 additions & 3 deletions .github/workflows/run-python-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
shell: bash
run: |
uname -a
head -n 100 /etc/*version
find "/etc" -maxdepth 1 -type f -name "*version" -exec head -n 100 {} \;
- name: ccache
uses: hendrikmuhs/[email protected]
Expand All @@ -77,20 +77,108 @@ jobs:
shell: bash
run: |
python3 -m pip install --upgrade pip numpy pypinyin sentencepiece>=0.1.96 soundfile
python3 -m pip install wheel twine setuptools
- name: Install sherpa-onnx
- name: Install ninja
shell: bash
run: |
sudo apt-get install ninja-build
- name: Display ninja version
shell: bash
run: |
ninja --version
ninja --help || true
which ninja
- name: Display site packages dir
shell: bash
run: |
python3 -c 'import site; print(site.getsitepackages())'
p=$(python3 -c 'import site; print(site.getsitepackages())')
echo "p: $p"
- name: Install patchelf
shell: bash
run: |
sudo apt-get update -q
sudo apt-get install -q -y patchelf
patchelf --help
- name: Build sherpa-onnx
shell: bash
run: |
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
cmake --version
export SHERPA_ONNX_CMAKE_ARGS="-G Ninja -DCMAKE_BUILD_TYPE=Release"
export SHERPA_ONNX_MAKE_ARGS="-j 6"
python3 setup.py bdist_wheel
- name: Patch wheels
shell: bash
run: |
mkdir ./dist2
sudo ./scripts/wheel/patch_wheel.py --in-dir ./dist --out-dir ./dist2
- name: Install sherpa-onnx
shell: bash
run: |
ls -lh dist2
python3 -m pip install ./dist2/*.whl
- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}-${{ matrix.python-version }}-whl
path: ./dist

- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}-${{ matrix.python-version }}-whl-patched
path: ./dist2

- name: Show dependencies
shell: bash
run: |
cd dist
mkdir t
cd t
unzip ../*.whl
readelf -d _sherpa_onnx*.so
echo "----"
readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx
- name: Show dependencies (patched)
shell: bash
run: |
cd dist2
mkdir t
cd t
unzip ../*.whl
readelf -d _sherpa_onnx*.so
echo "----"
python3 -m pip install --verbose .
readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx
- name: Test sherpa-onnx
shell: bash
run: |
export OS=${{ matrix.os }}
p=$(python3 -c 'import site; print(site.getsitepackages()[0])')
echo "p: $p"
p=$p/sherpa_onnx/lib
echo "p: $p"
ls -lh $p
export LD_LIBRARY_PATH=$p:$LD_LIBRARY_PATH
echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
.github/scripts/test-python.sh
.github/scripts/test-speaker-recognition-python.sh
Expand Down
46 changes: 44 additions & 2 deletions .github/workflows/test-build-wheel.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,50 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
# See https://github.com/actions/runner-images
include:
- os: ubuntu-20.04
python-version: "3.7"
- os: ubuntu-20.04
python-version: "3.8"
- os: ubuntu-20.04
python-version: "3.9"
- os: ubuntu-22.04
python-version: "3.10"
- os: ubuntu-22.04
python-version: "3.11"
- os: ubuntu-22.04
python-version: "3.12"

- os: macos-11
python-version: "3.7"

- os: macos-12
python-version: "3.8"

- os: macos-13
python-version: "3.9"
- os: macos-13
python-version: "3.10"
- os: macos-13
python-version: "3.11"

- os: macos-14
python-version: "3.12"

- os: windows-2019
python-version: "3.7"
- os: windows-2019
python-version: "3.8"
- os: windows-2019
python-version: "3.9"

- os: windows-2022
python-version: "3.10"
- os: windows-2022
python-version: "3.11"
- os: windows-2022
python-version: "3.12"

steps:
- uses: actions/checkout@v4
Expand Down
47 changes: 43 additions & 4 deletions .github/workflows/test-pip-install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,50 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
exclude:
- os: macos-latest
# See https://github.com/actions/runner-images
include:
- os: ubuntu-20.04
python-version: "3.7"
- os: ubuntu-20.04
python-version: "3.8"
- os: ubuntu-20.04
python-version: "3.9"
- os: ubuntu-22.04
python-version: "3.10"
- os: ubuntu-22.04
python-version: "3.11"
- os: ubuntu-22.04
python-version: "3.12"

- os: macos-11
python-version: "3.7"

- os: macos-12
python-version: "3.8"

- os: macos-13
python-version: "3.9"
- os: macos-13
python-version: "3.10"
- os: macos-13
python-version: "3.11"

- os: macos-14
python-version: "3.12"

- os: windows-2019
python-version: "3.7"
- os: windows-2019
python-version: "3.8"
- os: windows-2019
python-version: "3.9"

- os: windows-2022
python-version: "3.10"
- os: windows-2022
python-version: "3.11"
- os: windows-2022
python-version: "3.12"

steps:
- uses: actions/checkout@v4
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test-python-offline-websocket-server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
os: [ubuntu-20.04, ubuntu-22.04, windows-latest, macos-latest, macos-14]
python-version: ["3.10"]
model_type: ["transducer", "paraformer", "nemo_ctc", "whisper", "tdnn"]

steps:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test-python-online-websocket-server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
os: [ubuntu-20.04, ubuntu-22.04, windows-latest, macos-latest, macos-14]
python-version: ["3.10"]
model_type: ["transducer", "paraformer", "zipformer2-ctc"]

steps:
Expand Down
17 changes: 12 additions & 5 deletions cmake/cmake_extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,13 +180,20 @@ def build_extension(self, ext: setuptools.extension.Extension):
print('Setting make_args to "-j4"')
make_args = "-j4"

build_cmd = f"""
cd {self.build_temp}
if "-G Ninja" in cmake_args:
build_cmd = f"""
cd {self.build_temp}
cmake {cmake_args} {sherpa_onnx_dir}
ninja {make_args} install
"""
else:
build_cmd = f"""
cd {self.build_temp}
cmake {cmake_args} {sherpa_onnx_dir}
cmake {cmake_args} {sherpa_onnx_dir}
make {make_args} install/strip
"""
make {make_args} install/strip
"""
print(f"build command is:\n{build_cmd}")

ret = os.system(build_cmd)
Expand Down

0 comments on commit 2cb3c7a

Please sign in to comment.