Support whisper large/large-v1/large-v2/large-v3 and distil-large-v2 (#…

…1114)
k2-fsa · Jul 12, 2024 · 117cd7b · 117cd7b
1 parent d928f77
commit 117cd7b
Show file tree

Hide file tree

Showing 23 changed files with 152 additions and 85 deletions.
diff --git a/.github/workflows/export-whisper-to-onnx.yaml b/.github/workflows/export-whisper-to-onnx.yaml
@@ -15,9 +15,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        # model: ["distil-medium.en", "distil-small.en",  "tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "large", "large-v1", "large-v2", "distil-large-v2"]
-        model: ["distil-medium.en", "distil-small.en",  "tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "medium-aishell"]
+        os: [macos-latest]
+        model: ["distil-medium.en", "distil-small.en",  "tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "medium-aishell", "large", "large-v1", "large-v2", "distil-large-v2"]
+        # model: ["large", "large-v1", "large-v2", "large-v3", "distil-large-v2"]
         python-version: ["3.8"]
 
     steps:
@@ -32,7 +32,7 @@ jobs:
         shell: bash
         run: |
           python3 -m pip install torch==1.13.0 torchaudio==0.13.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html
-          python3 -m pip install openai-whisper==20230314 onnxruntime onnx
+          python3 -m pip install openai-whisper==20231117 onnxruntime onnx soundfile librosa
 
       - name: export ${{ matrix.model }}
         shell: bash
@@ -62,7 +62,6 @@ jobs:
             rm -fv medium-aishell-decoder.onnx
           fi
 
-
           ls -lh
 
           ls -lh ~/.cache/whisper || true
@@ -74,7 +73,8 @@ jobs:
           src=sherpa-onnx-whisper-${{ matrix.model }}
 
           cd ..
-          mv whisper $src
+          mkdir $src
+          mv -v whisper/$model* $src/
 
           echo "------------------------------"
 
@@ -97,19 +97,16 @@ jobs:
           ls -lh $src
           echo "--------------------"
 
-          if [[ $model == large || $model == large-v1 || $model == large-v2 || $model == distil-large-v2 ]]; then
-            #tar cvjf - $src | split --bytes=1024MB - $src.tar.bz2.
-            tar cvjf $src.tar.bz2 $src
-            split -b 1G $src.tar.bz2 $src.tar.bz2.
-            rm $src.tar.bz2
-            # cat $src.tar.gz.* | tar xjf -
+          if [[ $model == large || $model == large-v1 || $model == large-v2 || $model == distil-large-v2  ]]; then
+            echo "Don't release model to github for large models. $model"
           else
             tar cvjf $src.tar.bz2 $src
           fi
-          ls -lh
 
+          ls -lh
 
       - name: Release
+        if: matrix.model != 'large' && matrix.model != 'large-v1' && matrix.model != 'large-v2' && matrix.model != 'large-v3' && matrix.model != 'distil-large-v2'
         uses: svenstaro/upload-release-action@v2
         with:
           file_glob: true
@@ -119,19 +116,6 @@ jobs:
           repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
           tag: asr-models
 
-      - name: Test ${{ matrix.model }}
-        shell: bash
-        run: |
-          python3 -m pip install kaldi-native-fbank
-          git checkout .
-          model=${{ matrix.model }}
-          src=sherpa-onnx-whisper-$model
-          python3 scripts/whisper/test.py \
-            --encoder $src/$model-encoder.int8.onnx \
-            --decoder $src/$model-decoder.int8.onnx \
-            --tokens $src/$model-tokens.txt \
-            $src/test_wavs/0.wav
-
       - name: Publish ${{ matrix.model }} to huggingface
         shell: bash
         env:
@@ -144,27 +128,36 @@ jobs:
 
           export GIT_CLONE_PROTECTION_ACTIVE=false
 
-          GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-${{ matrix.model }} huggingface
+          export GIT_LFS_SKIP_SMUDGE=1
+
+          git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-${{ matrix.model }} huggingface
 
           if [[ $model != medium-aishell ]]; then
             rm -rf huggingface/*
           fi
 
-          if [[ $model == large || $model == large-v1 || $model == large-v2 || $model == distil-large-v2 ]]; then
-            mv $src.tar* ./huggingface
-          else
-            cp -v $src/*.onnx ./huggingface
-            cp -v $src/*tokens* ./huggingface
-            cp -av $src/test_wavs ./huggingface
-          fi
+          cp -av $src/* ./huggingface/
 
           cd huggingface
 
           git status
           ls -lh
-          git lfs track "*gz*"
           git lfs track "*onnx*"
+          git lfs track "*weights*"
 
           git add .
           git commit -m "upload ${{ matrix.model }}"
           git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-whisper-${{ matrix.model }} main
+
+      - name: Test ${{ matrix.model }}
+        shell: bash
+        run: |
+          python3 -m pip install kaldi-native-fbank
+          git checkout .
+          model=${{ matrix.model }}
+          src=sherpa-onnx-whisper-$model
+          time python3 scripts/whisper/test.py \
+            --encoder $src/$model-encoder.onnx \
+            --decoder $src/$model-decoder.onnx \
+            --tokens $src/$model-tokens.txt \
+            $src/test_wavs/0.wav
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
-## 1.10.14 (to-be-released)
+## 1.10.14
 
+* Support whisper large v3
 * Update onnxruntime from v1.18.0 to v1.18.1
 * Fix invalid utf8 sequence from Whisper for Dart API.
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -11,7 +11,7 @@ project(sherpa-onnx)
 # ./nodejs-addon-examples
 # ./dart-api-examples/
 # ./CHANGELOG.md
-set(SHERPA_ONNX_VERSION "1.10.13")
+set(SHERPA_ONNX_VERSION "1.10.14")
 
 # Disable warning about
 #

diff --git a/cmake/kaldi-native-fbank.cmake b/cmake/kaldi-native-fbank.cmake
@@ -1,9 +1,9 @@
 function(download_kaldi_native_fbank)
   include(FetchContent)
 
-  set(kaldi_native_fbank_URL   "https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.19.3.tar.gz")
-  set(kaldi_native_fbank_URL2  "https://hub.nuaa.cf/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.19.3.tar.gz")
-  set(kaldi_native_fbank_HASH "SHA256=335fe1daf1b9bfb2a7b6bf03b64c4c4686c39077c57fb8058c02611981676638")
+  set(kaldi_native_fbank_URL   "https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.20.0.tar.gz")
+  set(kaldi_native_fbank_URL2  "https://hub.nuaa.cf/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.20.0.tar.gz")
+  set(kaldi_native_fbank_HASH "SHA256=c6195b3cf374eef824644061d3c04f6b2a9267ae554169cbaa9865c89c1fe4f9")
 
   set(KALDI_NATIVE_FBANK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
   set(KALDI_NATIVE_FBANK_BUILD_PYTHON OFF CACHE BOOL "" FORCE)
@@ -12,11 +12,11 @@ function(download_kaldi_native_fbank)
   # If you don't have access to the Internet,
   # please pre-download kaldi-native-fbank
   set(possible_file_locations
-    $ENV{HOME}/Downloads/kaldi-native-fbank-1.19.3.tar.gz
-    ${CMAKE_SOURCE_DIR}/kaldi-native-fbank-1.19.3.tar.gz
-    ${CMAKE_BINARY_DIR}/kaldi-native-fbank-1.19.3.tar.gz
-    /tmp/kaldi-native-fbank-1.19.3.tar.gz
-    /star-fj/fangjun/download/github/kaldi-native-fbank-1.19.3.tar.gz
+    $ENV{HOME}/Downloads/kaldi-native-fbank-1.20.0.tar.gz
+    ${CMAKE_SOURCE_DIR}/kaldi-native-fbank-1.20.0.tar.gz
+    ${CMAKE_BINARY_DIR}/kaldi-native-fbank-1.20.0.tar.gz
+    /tmp/kaldi-native-fbank-1.20.0.tar.gz
+    /star-fj/fangjun/download/github/kaldi-native-fbank-1.20.0.tar.gz
   )
 
   foreach(f IN LISTS possible_file_locations)

diff --git a/dart-api-examples/non-streaming-asr/pubspec.yaml b/dart-api-examples/non-streaming-asr/pubspec.yaml
@@ -10,7 +10,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.13
+  sherpa_onnx: ^1.10.14
   path: ^1.9.0
   args: ^2.5.0
 

diff --git a/dart-api-examples/streaming-asr/pubspec.yaml b/dart-api-examples/streaming-asr/pubspec.yaml
@@ -11,7 +11,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.13
+  sherpa_onnx: ^1.10.14
   path: ^1.9.0
   args: ^2.5.0
 

diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml
@@ -8,7 +8,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.13
+  sherpa_onnx: ^1.10.14
   path: ^1.9.0
   args: ^2.5.0
 

diff --git a/dart-api-examples/vad/pubspec.yaml b/dart-api-examples/vad/pubspec.yaml
@@ -9,7 +9,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.10.13
+  sherpa_onnx: ^1.10.14
   path: ^1.9.0
   args: ^2.5.0
 

diff --git a/flutter-examples/streaming_asr/pubspec.yaml b/flutter-examples/streaming_asr/pubspec.yaml
@@ -5,7 +5,7 @@ description: >
 
 publish_to: 'none'
 
-version: 1.10.13
+version: 1.10.14
 
 topics:
   - speech-recognition
@@ -30,7 +30,7 @@ dependencies:
   record: ^5.1.0
   url_launcher: ^6.2.6
 
-  sherpa_onnx: ^1.10.13
+  sherpa_onnx: ^1.10.14
   # sherpa_onnx:
     # path: ../../flutter/sherpa_onnx
 

diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml
@@ -17,7 +17,7 @@ dependencies:
   cupertino_icons: ^1.0.6
   path_provider: ^2.1.3
   path: ^1.9.0
-  sherpa_onnx: ^1.10.13
+  sherpa_onnx: ^1.10.14
   url_launcher: ^6.2.6
   audioplayers: ^5.0.0
 

diff --git a/flutter/sherpa_onnx/pubspec.yaml b/flutter/sherpa_onnx/pubspec.yaml
@@ -17,7 +17,7 @@ topics:
   - voice-activity-detection
 
 # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec
-version: 1.10.13
+version: 1.10.14
 
 homepage: https://github.com/k2-fsa/sherpa-onnx
 
@@ -30,19 +30,19 @@ dependencies:
   flutter:
     sdk: flutter
 
-  sherpa_onnx_android: ^1.10.13
+  sherpa_onnx_android: ^1.10.14
     # path: ../sherpa_onnx_android
 
-  sherpa_onnx_macos: ^1.10.13
+  sherpa_onnx_macos: ^1.10.14
     # path: ../sherpa_onnx_macos
 
-  sherpa_onnx_linux: ^1.10.13
+  sherpa_onnx_linux: ^1.10.14
     # path: ../sherpa_onnx_linux
     #
-  sherpa_onnx_windows: ^1.10.13
+  sherpa_onnx_windows: ^1.10.14
     # path: ../sherpa_onnx_windows
 
-  sherpa_onnx_ios: ^1.10.13
+  sherpa_onnx_ios: ^1.10.14
   # sherpa_onnx_ios:
     # path: ../sherpa_onnx_ios
 

diff --git a/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec b/flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec
@@ -7,7 +7,7 @@
 # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c
 Pod::Spec.new do |s|
   s.name             = 'sherpa_onnx_ios'
-  s.version          = '1.10.13'
+  s.version          = '1.10.14'
   s.summary          = 'A new Flutter FFI plugin project.'
   s.description      = <<-DESC
 A new Flutter FFI plugin project.

diff --git a/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec b/flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec
@@ -4,7 +4,7 @@
 #
 Pod::Spec.new do |s|
   s.name             = 'sherpa_onnx_macos'
-  s.version          = '1.10.13'
+  s.version          = '1.10.14'
   s.summary          = 'sherpa-onnx Flutter FFI plugin project.'
   s.description      = <<-DESC
 sherpa-onnx Flutter FFI plugin project.

diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json
@@ -1,5 +1,5 @@
 {
   "dependencies": {
-    "sherpa-onnx-node": "^1.10.13"
+    "sherpa-onnx-node": "^1.10.14"
   }
 }
diff --git a/scripts/dart/sherpa-onnx-pubspec.yaml b/scripts/dart/sherpa-onnx-pubspec.yaml
@@ -17,7 +17,7 @@ topics:
   - voice-activity-detection
 
 # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx.podspec
-version: 1.10.13
+version: 1.10.14
 
 homepage: https://github.com/k2-fsa/sherpa-onnx
 

diff --git a/scripts/whisper/.gitignore b/scripts/whisper/.gitignore
@@ -2,3 +2,9 @@
 *.config
 *.ort
 *-tokens.txt
+*.bias
+*.weights
+*.weight
+*.*embedding
+_Const*
+onnx__*