Support streaming zipformer CTC (#496)

* Support streaming zipformer CTC * test online zipformer2 CTC * Update doc of sherpa-onnx.cc * Add Python APIs for streaming zipformer2 ctc * Add Python API examples for streaming zipformer2 ctc * Swift API for streaming zipformer2 CTC * NodeJS API for streaming zipformer2 CTC * Kotlin API for streaming zipformer2 CTC * Golang API for streaming zipformer2 CTC * C# API for streaming zipformer2 CTC * Release v1.9.6
k2-fsa · Dec 22, 2023 · e475e75 · e475e75
1 parent 7634f5f
commit e475e75
Show file tree

Hide file tree

Showing 70 changed files with 1,518 additions and 212 deletions.
diff --git a/.github/scripts/test-nodejs-npm.sh b/.github/scripts/test-nodejs-npm.sh
@@ -51,6 +51,13 @@ rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 node ./test-online-transducer.js
 rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20
 
+curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+rm sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+
+node ./test-online-zipformer2-ctc.js
+rm -rf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13
+
 # offline tts
 
 curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2

diff --git a/.github/scripts/test-online-ctc.sh b/.github/scripts/test-online-ctc.sh
@@ -13,6 +13,37 @@ echo "PATH: $PATH"
 
 which $EXE
 
+log "------------------------------------------------------------"
+log "Run streaming Zipformer2 CTC                                "
+log "------------------------------------------------------------"
+
+url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+repo=$(basename -s .tar.bz2 $url)
+curl -SL -O $url
+tar xvf $repo.tar.bz2
+rm $repo.tar.bz2
+
+log "test fp32"
+
+time $EXE \
+  --debug=1 \
+  --zipformer2-ctc-model=$repo/ctc-epoch-20-avg-1-chunk-16-left-128.onnx \
+  --tokens=$repo/tokens.txt \
+  $repo/test_wavs/DEV_T0000000000.wav \
+  $repo/test_wavs/DEV_T0000000001.wav \
+  $repo/test_wavs/DEV_T0000000002.wav
+
+log "test int8"
+
+time $EXE \
+  --debug=1 \
+  --zipformer2-ctc-model=$repo/ctc-epoch-20-avg-1-chunk-16-left-128.int8.onnx \
+  --tokens=$repo/tokens.txt \
+  $repo/test_wavs/DEV_T0000000000.wav \
+  $repo/test_wavs/DEV_T0000000001.wav \
+  $repo/test_wavs/DEV_T0000000002.wav
+
+
 log "------------------------------------------------------------"
 log "Run streaming Conformer CTC from WeNet"
 log "------------------------------------------------------------"

diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh
@@ -8,6 +8,27 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+mkdir -p /tmp/icefall-models
+dir=/tmp/icefall-models
+
+pushd $dir
+wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+rm sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+popd
+repo=$dir/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13
+
+python3 ./python-api-examples/online-decode-files.py \
+  --tokens=$repo/tokens.txt \
+  --zipformer2-ctc=$repo/ctc-epoch-20-avg-1-chunk-16-left-128.onnx \
+  $repo/test_wavs/DEV_T0000000000.wav \
+  $repo/test_wavs/DEV_T0000000001.wav \
+  $repo/test_wavs/DEV_T0000000002.wav
+
+python3 sherpa-onnx/python/tests/test_offline_recognizer.py --verbose
+
+rm -rf $dir/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13
+
 wenet_models=(
 sherpa-onnx-zh-wenet-aishell
 sherpa-onnx-zh-wenet-aishell2
@@ -17,8 +38,6 @@ sherpa-onnx-en-wenet-librispeech
 sherpa-onnx-en-wenet-gigaspeech
 )
 
-mkdir -p /tmp/icefall-models
-dir=/tmp/icefall-models
 
 for name in ${wenet_models[@]}; do
   repo_url=https://huggingface.co/csukuangfj/$name

diff --git a/.github/scripts/test-swift.sh b/.github/scripts/test-swift.sh
@@ -21,6 +21,9 @@ cat /Users/fangjun/Desktop/Obama.srt
 ./run-tts.sh
 ls -lh
 
+./run-decode-file.sh
+rm decode-file
+sed -i.bak  '20d' ./decode-file.swift
 ./run-decode-file.sh
 
 ./run-decode-file-non-streaming.sh

diff --git a/.github/workflows/export-wenet-to-onnx.yaml b/.github/workflows/export-wenet-to-onnx.yaml
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/export-wespeaker-to-onnx.yaml b/.github/workflows/export-wespeaker-to-onnx.yaml
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/export-whisper-to-onnx.yaml b/.github/workflows/export-whisper-to-onnx.yaml
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml
@@ -107,23 +107,23 @@ jobs:
           name: release-static
           path: build/bin/*
 
-      - name: Test offline Whisper
+      - name: Test online CTC
         shell: bash
         run: |
           export PATH=$PWD/build/bin:$PATH
-          export EXE=sherpa-onnx-offline
-
-          readelf -d build/bin/sherpa-onnx-offline
+          export EXE=sherpa-onnx
 
-          .github/scripts/test-offline-whisper.sh
+          .github/scripts/test-online-ctc.sh
 
-      - name: Test online CTC
+      - name: Test offline Whisper
         shell: bash
         run: |
           export PATH=$PWD/build/bin:$PATH
-          export EXE=sherpa-onnx
+          export EXE=sherpa-onnx-offline
 
-          .github/scripts/test-online-ctc.sh
+          readelf -d build/bin/sherpa-onnx-offline
+
+          .github/scripts/test-offline-whisper.sh
 
       - name: Test offline CTC
         shell: bash

diff --git a/.github/workflows/npm.yaml b/.github/workflows/npm.yaml
@@ -25,7 +25,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml
@@ -55,7 +55,7 @@ jobs:
           key: ${{ matrix.os }}-python-${{ matrix.python-version }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/style_check.yaml b/.github/workflows/style_check.yaml
@@ -49,7 +49,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-build-wheel.yaml b/.github/workflows/test-build-wheel.yaml
@@ -29,7 +29,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml
@@ -61,7 +61,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest] #, windows-latest]
         python-version: ["3.8"]
 
     steps:
@@ -70,7 +70,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -143,6 +143,7 @@ jobs:
           cd dotnet-examples/
 
           cd online-decode-files
+          ./run-zipformer2-ctc.sh
           ./run-transducer.sh
           ./run-paraformer.sh
 

diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml
@@ -53,7 +53,7 @@ jobs:
           mkdir build
           cd build
           cmake -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DBUILD_SHARED_LIBS=ON -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF ..
-          make -j
+          make -j1
           cp -v _deps/onnxruntime-src/lib/libonnxruntime*dylib ./lib/
 
           cd ../scripts/go/_internal/
@@ -153,6 +153,14 @@ jobs:
 
           git lfs install
 
+          echo "Test zipformer2 CTC"
+          wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+          tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+          rm sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+
+          ./run-zipformer2-ctc.sh
+          rm -rf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13
+
           echo "Test transducer"
           git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26
           ./run-transducer.sh

diff --git a/.github/workflows/test-nodejs-npm.yaml b/.github/workflows/test-nodejs-npm.yaml
@@ -34,7 +34,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-nodejs.yaml b/.github/workflows/test-nodejs.yaml
@@ -52,7 +52,7 @@ jobs:
           ls -lh install/lib
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml
@@ -40,7 +40,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-python-offline-websocket-server.yaml b/.github/workflows/test-python-offline-websocket-server.yaml
@@ -38,7 +38,7 @@ jobs:
           key: ${{ matrix.os }}-python-${{ matrix.python-version }}
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-python-online-websocket-server.yaml b/.github/workflows/test-python-online-websocket-server.yaml
@@ -25,7 +25,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
-        model_type: ["transducer", "paraformer"]
+        model_type: ["transducer", "paraformer", "zipformer2-ctc"]
 
     steps:
       - uses: actions/checkout@v4
@@ -38,7 +38,7 @@ jobs:
           key: ${{ matrix.os }}-python-${{ matrix.python-version }}
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -57,6 +57,26 @@ jobs:
           python3 -m pip install --no-deps --verbose .
           python3 -m pip install websockets
 
+      - name: Start server for zipformer2 CTC models
+        if: matrix.model_type == 'zipformer2-ctc'
+        shell: bash
+        run: |
+          curl -O -L https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+          tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+          rm sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
+
+          python3 ./python-api-examples/streaming_server.py \
+            --zipformer2-ctc ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13/ctc-epoch-20-avg-1-chunk-16-left-128.onnx \
+            --tokens=./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13/tokens.txt &
+          echo "sleep 10 seconds to wait the server start"
+          sleep 10
+
+      - name: Start client for zipformer2 CTC models
+        if: matrix.model_type == 'zipformer2-ctc'
+        shell: bash
+        run: |
+          python3 ./python-api-examples/online-websocket-client-decode-file.py \
+            ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13/test_wavs/DEV_T0000000000.wav
 
       - name: Start server for transducer models
         if: matrix.model_type == 'transducer'

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 project(sherpa-onnx)
 
-set(SHERPA_ONNX_VERSION "1.9.4")
+set(SHERPA_ONNX_VERSION "1.9.6")
 
 # Disable warning about
 #

diff --git a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt
@@ -26,9 +26,14 @@ data class OnlineParaformerModelConfig(
     var decoder: String = "",
 )
 
+data class OnlineZipformer2CtcModelConfig(
+    var model: String = "",
+)
+
 data class OnlineModelConfig(
     var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
     var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
+    var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
     var tokens: String,
     var numThreads: Int = 1,
     var debug: Boolean = false,

diff --git a/dotnet-examples/.gitignore b/dotnet-examples/.gitignore
@@ -1,2 +1,3 @@
 bin
 obj
+!*.sh
-Original file line number
+Diff line change
@@ -1,2 +1,3 @@
     bin
     obj
+    !*.sh