merged latest code

k2-fsa · Mar 11, 2024 · 7b7ee17 · 7b7ee17
1 parent 18c9322
commit 7b7ee17
Show file tree

Hide file tree

Showing 38 changed files with 801 additions and 127 deletions.
diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml
@@ -17,13 +17,14 @@ concurrency:
 
 jobs:
   build_wheels_aarch64:
-    name: ${{ matrix.python-version }}
+    name: ${{ matrix.manylinux }} ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
         python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"]
+        manylinux: [manylinux2014, manylinux_2_28]
 
     steps:
       - uses: actions/checkout@v4
@@ -51,7 +52,7 @@ jobs:
           CIBW_SKIP: "cp27-* cp35-* cp36-* *-win32 pp* *-musllinux* *-manylinux_i686"
           CIBW_BUILD_VERBOSITY: 3
           CIBW_ARCHS_LINUX: aarch64
-          CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64
+          CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/${{ matrix.manylinux }}_aarch64
           # From onnxruntime >= 1.17.0, it drops support for CentOS 7.0 and it supports only manylinux_2_28.
           # manylinux_2_24 is no longer supported
 
@@ -63,7 +64,7 @@ jobs:
           ls -lh ./wheelhouse/*.whl
 
       - name: Publish to huggingface
-        if: matrix.python-version == 'cp38'
+        if: matrix.python-version == 'cp38' && matrix.manylinux == 'manylinux_2_28'
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3
@@ -93,7 +94,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: wheel-${{ matrix.python-version }}
+          name: wheel-${{ matrix.python-version }}-${{ matrix.manylinux }}
           path: ./wheelhouse/*.whl
 
       - name: Publish wheels to PyPI

diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml
@@ -17,13 +17,15 @@ concurrency:
 
 jobs:
   build_wheels_linux:
-    name: ${{ matrix.python-version }}
+    name: ${{ matrix.manylinux }} ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
         python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"]
+        manylinux: [manylinux2014, manylinux_2_28]
+
 
     steps:
       - uses: actions/checkout@v4
@@ -46,9 +48,7 @@ jobs:
           CIBW_BUILD: "${{ matrix.python-version}}-* "
           CIBW_SKIP: "cp27-* cp35-* cp36-* *-win32 pp* *-musllinux* *-manylinux_i686"
           CIBW_BUILD_VERBOSITY: 3
-          CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64
-          # From onnxruntime >= 1.17.0, it drops support for CentOS 7.0 and it supports only manylinux_2_28.
-          # manylinux_2_24 is no longer supported
+          CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.manylinux }}_x86_64
 
       - name: Display wheels
         shell: bash
@@ -76,7 +76,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: wheel-${{ matrix.python-version }}
+          name: wheel-${{ matrix.python-version }}-${{ matrix.manylinux }}
           path: ./wheelhouse/*.whl
 
       - name: Publish to huggingface
@@ -119,14 +119,14 @@ jobs:
           twine upload ./wheelhouse/*.whl
 
       - name: Build sdist
-        if: matrix.python-version == 'cp38'
+        if: matrix.python-version == 'cp38' && matrix.manylinux == 'manylinux_2_28'
         shell: bash
         run: |
           python3 setup.py sdist
           ls -l dist/*
 
       - name: Publish sdist to PyPI
-        if: matrix.python-version == 'cp38'
+        if: matrix.python-version == 'cp38' && matrix.manylinux == 'manylinux_2_28'
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml
@@ -227,7 +227,7 @@ jobs:
           tar cjvf ${dst}.tar.bz2 $dst
 
       - name: Release pre-compiled binaries and libs for linux x64
-        if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+        if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-20.04' && matrix.gcc_version == '7'
         uses: svenstaro/upload-release-action@v2
         with:
           file_glob: true

diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml
@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        lib_type: [static, shared]
+        lib_type: [shared] #, static]
 
     steps:
       - uses: actions/checkout@v4
@@ -55,45 +55,35 @@ jobs:
         uses: actions/cache@v4
         with:
           path: qemu-install
-          key: qemu-riscv-install-20240225
-
-      - name: install-qemu-build-deps
-        if: steps.cache-qemu.outputs.cache-hit != 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get install autoconf automake autotools-dev ninja-build
-
-      - name: checkout-qemu
-        if: steps.cache-qemu.outputs.cache-hit != 'true'
-        uses: actions/checkout@v3
-        with:
-          repository: qemu/qemu
-          path: qemu
+          key: qemu-riscv-xuantie-install-20240306
 
       - name: qemu
         if: steps.cache-qemu.outputs.cache-hit != 'true'
         run: |
-          cd qemu
-          ./configure --prefix=$GITHUB_WORKSPACE/qemu-install --target-list=riscv64-linux-user --disable-system
-          make -j2
-          make install
-          ls -lh $GITHUB_WORKSPACE/qemu-install
-          ls -lh $GITHUB_WORKSPACE/qemu-install/bin
+          # https://pypi.org/project/xuantie-qemu/#files
+          wget -q https://files.pythonhosted.org/packages/21/f4/733f29c435987e8bb264a6504c7a4ea4c04d0d431b38a818ab63eef082b9/xuantie_qemu-20230825-py3-none-manylinux1_x86_64.whl
+          unzip xuantie_qemu-20230825-py3-none-manylinux1_x86_64.whl
+          mkdir -p qemu-install/bin
+
+          cp -v ./qemu/qemu-riscv64 ./qemu-install/bin
 
       - name: cache-toolchain
         id: cache-toolchain
         uses: actions/cache@v4
         with:
           path: toolchain
-          key: riscv64-glibc-ubuntu-20.04-gcc-nightly-2023.10.17-nightly
+          key: Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.6.1-20220906.tar.gz
 
       - name: Download toolchain
         if: steps.cache-toolchain.outputs.cache-hit != 'true'
         shell: bash
         run: |
+          wget -q https://occ-oss-prod.oss-cn-hangzhou.aliyuncs.com/resource//1663142514282/Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.6.1-20220906.tar.gz
+
           mkdir $GITHUB_WORKSPACE/toolchain
-          wget -q https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2023.10.17/riscv64-glibc-ubuntu-20.04-gcc-nightly-2023.10.17-nightly.tar.gz
-          tar xvf ./riscv64-glibc-ubuntu-20.04-gcc-nightly-2023.10.17-nightly.tar.gz --strip-components 1 -C $GITHUB_WORKSPACE/toolchain
+
+          tar xvf ./Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.6.1-20220906.tar.gz --strip-components 1 -C $GITHUB_WORKSPACE/toolchain
+          ls -lh $GITHUB_WORKSPACE/toolchain/bin
 
       - name: Display toolchain info
         shell: bash
@@ -139,6 +129,7 @@ jobs:
           export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH
           export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH
           export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/sysroot
+          export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/toolchain/sysroot/lib
 
           ls -lh ./build-riscv64-linux-gnu/bin
 
@@ -154,6 +145,44 @@ jobs:
           qemu-riscv64 ./build-riscv64-linux-gnu/bin/sherpa-onnx-offline-tts --help
           readelf -d ./build-riscv64-linux-gnu/bin/sherpa-onnx-offline-tts
 
+      - name: Test streaming speech recognition
+        shell: bash
+        run: |
+          export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH
+          export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH
+          export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/sysroot
+          export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/toolchain/sysroot/lib
+
+          wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2
+          tar xvf sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2
+          rm sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2
+
+          qemu-riscv64 ./build-riscv64-linux-gnu/bin/sherpa-onnx \
+            --tokens=./sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23/tokens.txt \
+            --encoder=./sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23/encoder-epoch-99-avg-1.onnx \
+            --decoder=./sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23/decoder-epoch-99-avg-1.onnx \
+            --joiner=./sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23/joiner-epoch-99-avg-1.onnx \
+            ./sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23/test_wavs/0.wav
+
+      - name: Test offline tts
+        shell: bash
+        run: |
+          export PATH=$GITHUB_WORKSPACE/toolchain/bin:$PATH
+          export PATH=$GITHUB_WORKSPACE/qemu-install/bin:$PATH
+          export QEMU_LD_PREFIX=$GITHUB_WORKSPACE/toolchain/sysroot
+          export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/toolchain/sysroot/lib
+
+          wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-lessac-medium.tar.bz2
+          tar xf vits-piper-en_US-lessac-medium.tar.bz2
+          rm vits-piper-en_US-lessac-medium.tar.bz2
+
+          qemu-riscv64 ./build-riscv64-linux-gnu/bin/sherpa-onnx-offline-tts \
+            --vits-model=./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx \
+            --vits-data-dir=./vits-piper-en_US-lessac-medium/espeak-ng-data \
+            --vits-tokens=./vits-piper-en_US-lessac-medium/tokens.txt \
+            --output-filename=./liliana-piper-en_US-lessac-medium.wav \
+            'liliana, the most beautiful and lovely assistant of our team!'
+
       - name: Copy files
         shell: bash
         run: |
@@ -190,6 +219,12 @@ jobs:
           name: sherpa-onnx-linux-riscv64-shared
           path: sherpa-onnx-*linux-riscv64-shared.tar.bz2
 
+      - uses: actions/upload-artifact@v4
+        if: matrix.lib_type == 'shared'
+        with:
+          name: wave
+          path: ./*.wav
+
       - uses: actions/upload-artifact@v4
         if: matrix.lib_type == 'static'
         with:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,6 +27,7 @@ option(SHERPA_ONNX_ENABLE_WASM_KWS "Whether to enable WASM for KWS" OFF)
 option(SHERPA_ONNX_ENABLE_WASM_NODEJS "Whether to enable WASM for NodeJS" OFF)
 option(SHERPA_ONNX_ENABLE_BINARY "Whether to build binaries" ON)
 option(SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY "True to link libstdc++ statically. Used only when BUILD_SHARED_LIBS is OFF on Linux" ON)
+option(SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE "True to use pre-installed onnxruntime if available" ON)
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
@@ -111,6 +112,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_WASM ${SHERPA_ONNX_ENABLE_WASM}")
 message(STATUS "SHERPA_ONNX_ENABLE_WASM_TTS ${SHERPA_ONNX_ENABLE_WASM_TTS}")
 message(STATUS "SHERPA_ONNX_ENABLE_WASM_ASR ${SHERPA_ONNX_ENABLE_WASM_ASR}")
 message(STATUS "SHERPA_ONNX_ENABLE_WASM_NODEJS ${SHERPA_ONNX_ENABLE_WASM_NODEJS}")
+message(STATUS "SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE ${SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE}")
 
 if(SHERPA_ONNX_ENABLE_WASM_TTS)
   if(NOT SHERPA_ONNX_ENABLE_WASM)
@@ -149,6 +151,7 @@ include(CheckIncludeFileCXX)
 if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME STREQUAL Android)
   check_include_file_cxx(alsa/asoundlib.h SHERPA_ONNX_HAS_ALSA)
   if(SHERPA_ONNX_HAS_ALSA)
+    message(STATUS "With Alsa")
     add_definitions(-DSHERPA_ONNX_ENABLE_ALSA=1)
   else()
     message(WARNING "\

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,12 @@
+include LICENSE
+include README.md
+include CMakeLists.txt
+recursive-include c-api-examples *.*
+recursive-include sherpa-onnx *.*
+recursive-include cmake *.*
+prune */__pycache__
+prune android
+prune sherpa-onnx/java-api
+prune ios-swift
+prune ios-swiftui
+
diff --git a/android/SherpaOnnx2Pass/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt b/android/SherpaOnnx2Pass/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt
@@ -75,6 +75,9 @@ data class OfflineParaformerModelConfig(
 data class OfflineWhisperModelConfig(
     var encoder: String = "",
     var decoder: String = "",
+    var language: String = "en", // Used with multilingual model
+    var task: String = "transcribe", // transcribe or translate
+    var tailPaddings: Int = 1000, // Padding added at the end of the samples
 )
 
 data class OfflineModelConfig(

diff --git a/build-ios.sh b/build-ios.sh
@@ -5,7 +5,7 @@ set -e
 dir=build-ios
 mkdir -p $dir
 cd $dir
-onnxruntime_version=1.16.3
+onnxruntime_version=1.17.1
 onnxruntime_dir=ios-onnxruntime/$onnxruntime_version
 
 if [ ! -f $onnxruntime_dir/onnxruntime.xcframework/ios-arm64/onnxruntime.a ]; then

diff --git a/build-riscv64-linux-gnu.sh b/build-riscv64-linux-gnu.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -ex
 
 if ! command -v riscv64-unknown-linux-gnu-g++  &> /dev/null; then
   echo "Please install the toolchain first."
@@ -42,8 +43,8 @@ export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH
 export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs
 
 if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then
-  # By default, use static link
-  BUILD_SHARED_LIBS=OFF
+  # By default, use shared libraries
+  BUILD_SHARED_LIBS=ON
 fi
 
 cmake \

diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt
@@ -6,3 +6,9 @@ target_link_libraries(decode-file-c-api sherpa-onnx-c-api cargs)
 
 add_executable(offline-tts-c-api offline-tts-c-api.c)
 target_link_libraries(offline-tts-c-api sherpa-onnx-c-api cargs)
+
+if(SHERPA_ONNX_HAS_ALSA)
+  add_subdirectory(./asr-microphone-example)
+else()
+  message(WARNING "Not include ./asr-microphone-example since alsa is not available")
+endif()
diff --git a/c-api-examples/asr-microphone-example/CMakeLists.txt b/c-api-examples/asr-microphone-example/CMakeLists.txt
@@ -0,0 +1,9 @@
+
+add_executable(c-api-alsa c-api-alsa.cc alsa.cc)
+target_link_libraries(c-api-alsa sherpa-onnx-c-api cargs)
+
+if(DEFINED ENV{SHERPA_ONNX_ALSA_LIB_DIR})
+  target_link_libraries(c-api-alsa -L$ENV{SHERPA_ONNX_ALSA_LIB_DIR} -lasound)
+else()
+  target_link_libraries(c-api-alsa asound)
+endif()
diff --git a/c-api-examples/asr-microphone-example/CPPLINT.cfg b/c-api-examples/asr-microphone-example/CPPLINT.cfg
@@ -0,0 +1 @@
+exclude_files=alsa.cc|alsa.h
diff --git a/c-api-examples/asr-microphone-example/README.md b/c-api-examples/asr-microphone-example/README.md
@@ -0,0 +1,12 @@
+# Introduction
+
+This folder contains examples for real-time speech recognition from a microphone
+using sherpa-onnx C API.
+
+**Note**: You can call C API from C++ files.
+
+
+## ./c-api-alsa.cc
+
+This file uses alsa to read a microphone. It runs only on Linux. This file
+does not support macOS or Windows.
diff --git a/c-api-examples/asr-microphone-example/alsa.cc b/c-api-examples/asr-microphone-example/alsa.cc
@@ -0,0 +1 @@
+../../sherpa-onnx/csrc/alsa.cc
diff --git a/c-api-examples/asr-microphone-example/alsa.h b/c-api-examples/asr-microphone-example/alsa.h
@@ -0,0 +1 @@
+../../sherpa-onnx/csrc/alsa.h