Merge branch 'k2-fsa:master' into master

k2-fsa · May 27, 2024 · f5f7b27 · f5f7b27
2 parents 72a45c2 + 49ea59d
commit f5f7b27
Show file tree

Hide file tree

Showing 220 changed files with 7,588 additions and 705 deletions.
diff --git a/.github/scripts/node-addon/run.sh b/.github/scripts/node-addon/run.sh
@@ -18,7 +18,7 @@ fi
 SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
 echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
 
-# SHERPA_ONNX_VERSION=1.0.24
+# SHERPA_ONNX_VERSION=1.0.25
 
 if [ -z $owner ]; then
   owner=k2-fsa

diff --git a/.github/scripts/test-offline-ctc.sh b/.github/scripts/test-offline-ctc.sh
@@ -8,6 +8,8 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+export GIT_CLONE_PROTECTION_ACTIVE=false
+
 echo "EXE is $EXE"
 echo "PATH: $PATH"
 
@@ -122,15 +124,13 @@ sherpa-onnx-en-wenet-librispeech
 # sherpa-onnx-en-wenet-gigaspeech
 )
 for name in ${wenet_models[@]}; do
-  repo_url=https://huggingface.co/csukuangfj/$name
+  repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$name.tar.bz2
   log "Start testing ${repo_url}"
-  repo=$(basename $repo_url)
+  repo=$name
   log "Download pretrained model and test-data from $repo_url"
-  GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-  pushd $repo
-  git lfs pull --include "*.onnx"
-  ls -lh *.onnx
-  popd
+  curl -SL -O $repo_url
+  tar xvf $name.tar.bz2
+  rm $name.tar.bz2
 
   log "test float32 models"
   time $EXE \
@@ -154,16 +154,13 @@ done
 log "------------------------------------------------------------"
 log "Run tdnn yesno (Hebrew)"
 log "------------------------------------------------------------"
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-tdnn-yesno
-log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model and test-data from $repo_url"
-
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
+url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-tdnn-yesno.tar.bz2
+curl -SL -O $url
+tar xvf sherpa-onnx-tdnn-yesno.tar.bz2
+rm sherpa-onnx-tdnn-yesno.tar.bz2
+log "Start testing ${url}"
+repo=sherpa-onnx-tdnn-yesno
+log "Download pretrained model and test-data from $url"
 
 log "test float32 models"
 time $EXE \
@@ -199,17 +196,14 @@ log "------------------------------------------------------------"
 log "Run Citrinet (stt_en_citrinet_512, English)"
 log "------------------------------------------------------------"
 
-repo_url=http://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-citrinet-512
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2
+curl -SL -O $repo_url
+tar xvf sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2
+rm sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
+repo=sherpa-onnx-nemo-ctc-en-citrinet-512
 log "Download pretrained model and test-data from $repo_url"
 
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
-
 time $EXE \
   --tokens=$repo/tokens.txt \
   --nemo-ctc-model=$repo/model.onnx \
@@ -231,18 +225,14 @@ rm -rf $repo
 log "------------------------------------------------------------"
 log "Run Librispeech zipformer CTC H/HL/HLG decoding (English)   "
 log "------------------------------------------------------------"
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-ctc-en-2023-10-02
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2
+curl -SL -O $repo_url
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
+tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2
+rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2
+repo=sherpa-onnx-zipformer-ctc-en-2023-10-02
 log "Download pretrained model and test-data from $repo_url"
 
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-git lfs pull --include "*.fst"
-ls -lh
-popd
-
 graphs=(
 $repo/H.fst
 $repo/HL.fst

diff --git a/.github/scripts/test-offline-transducer.sh b/.github/scripts/test-offline-transducer.sh
@@ -8,6 +8,8 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+export GIT_CLONE_PROTECTION_ACTIVE=false
+
 echo "EXE is $EXE"
 echo "PATH: $PATH"
 
@@ -116,17 +118,14 @@ log "------------------------------------------------------------"
 log "Run Conformer transducer (English)"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-conformer-en-2023-03-18
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-conformer-en-2023-03-18.tar.bz2
+curl -SL -O $repo_url
+tar xvf sherpa-onnx-conformer-en-2023-03-18.tar.bz2
+rm sherpa-onnx-conformer-en-2023-03-18.tar.bz2
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
+repo=sherpa-onnx-conformer-en-2023-03-18
 log "Download pretrained model and test-data from $repo_url"
 
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
-
 time $EXE \
   --tokens=$repo/tokens.txt \
   --encoder=$repo/encoder-epoch-99-avg-1.onnx \
@@ -153,16 +152,12 @@ log "------------------------------------------------------------"
 log "Run Zipformer transducer (English)"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-en-2023-03-30
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-03-30.tar.bz2
+curl -SL -O $repo_url
+tar xvf sherpa-onnx-zipformer-en-2023-03-30.tar.bz2
+rm sherpa-onnx-zipformer-en-2023-03-30.tar.bz2
+repo=sherpa-onnx-zipformer-en-2023-03-30
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model and test-data from $repo_url"
-
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
 
 time $EXE \
   --tokens=$repo/tokens.txt \
@@ -190,16 +185,12 @@ log "------------------------------------------------------------"
 log "Run Paraformer (Chinese)"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2
+curl -SL -O $repo_url
+tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2
+rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2
+repo=sherpa-onnx-paraformer-zh-2023-03-28
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model and test-data from $repo_url"
-
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
 
 time $EXE \
   --tokens=$repo/tokens.txt \
@@ -227,16 +218,13 @@ log "------------------------------------------------------------"
 log "Run Paraformer (Chinese) with timestamps"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-zh-2023-09-14
-log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model and test-data from $repo_url"
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2
+curl -SL -O $repo_url
+tar xvf sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2
+rm sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2
+repo=sherpa-onnx-paraformer-zh-2023-09-14
 
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
+log "Start testing ${repo_url}"
 
 time $EXE \
   --tokens=$repo/tokens.txt \

diff --git a/.github/scripts/test-offline-tts.sh b/.github/scripts/test-offline-tts.sh
@@ -8,6 +8,8 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+export GIT_CLONE_PROTECTION_ACTIVE=false
+
 echo "EXE is $EXE"
 echo "PATH: $PATH"
 
@@ -38,15 +40,13 @@ log "------------------------------------------------------------"
 log "vits-ljs test"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/vits-ljs
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-ljs.tar.bz2
+curl -SL -O $repo_url
+tar xvf vits-ljs.tar.bz2
+rm vits-ljs.tar.bz2
+repo=vits-ljs
+
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
 
 $EXE \
   --vits-model=$repo/vits-ljs.onnx \
@@ -63,15 +63,13 @@ log "------------------------------------------------------------"
 log "vits-vctk test"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/vits-vctk
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-vctk.tar.bz2
+curl -SL -O $repo_url
+tar xvf vits-vctk.tar.bz2
+rm vits-vctk.tar.bz2
+repo=vits-vctk
+
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
 
 for sid in 0 10 90; do
   $EXE \
@@ -91,15 +89,13 @@ log "------------------------------------------------------------"
 log "vits-zh-aishell3"
 log "------------------------------------------------------------"
 
-repo_url=https://huggingface.co/csukuangfj/vits-zh-aishell3
+repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2
+curl -SL -O $repo_url
+tar xvf vits-zh-aishell3.tar.bz2
+rm vits-zh-aishell3.tar.bz2
+repo=vits-zh-aishell3
+
 log "Start testing ${repo_url}"
-repo=$(basename $repo_url)
-log "Download pretrained model from $repo_url"
-GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-pushd $repo
-git lfs pull --include "*.onnx"
-ls -lh *.onnx
-popd
 
 for sid in 0 10 90; do
   $EXE \

diff --git a/.github/scripts/test-offline-whisper.sh b/.github/scripts/test-offline-whisper.sh
@@ -8,6 +8,8 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+export GIT_CLONE_PROTECTION_ACTIVE=false
+
 echo "EXE is $EXE"
 echo "PATH: $PATH"
 
@@ -31,17 +33,12 @@ for name in ${names[@]}; do
   log "Run $name"
   log "------------------------------------------------------------"
 
-  repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-whisper-$name
+  repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-$name.tar.bz2
+  curl -SL -O $repo_url
+  tar xvf sherpa-onnx-whisper-$name.tar.bz2
+  rm sherpa-onnx-whisper-$name.tar.bz2
+  repo=sherpa-onnx-whisper-$name
   log "Start testing ${repo_url}"
-  repo=$(basename $repo_url)
-  log "Download pretrained model and test-data from $repo_url"
-
-  GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-  pushd $repo
-  git lfs pull --include "*.onnx"
-  # git lfs pull --include "*.ort"
-  ls -lh *.onnx
-  popd
 
   log "test fp32 onnx"
 

diff --git a/.github/scripts/test-online-ctc.sh b/.github/scripts/test-online-ctc.sh
@@ -8,6 +8,8 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+export GIT_CLONE_PROTECTION_ACTIVE=false
+
 echo "EXE is $EXE"
 echo "PATH: $PATH"
 
@@ -53,7 +55,7 @@ $EXE \
   $repo/test_wavs/1.wav \
   $repo/test_wavs/8k.wav
 
-rm -rf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18
+rm -rf $repo
 
 log "------------------------------------------------------------"
 log "Run streaming Zipformer2 CTC                                "
@@ -85,28 +87,26 @@ time $EXE \
   $repo/test_wavs/DEV_T0000000001.wav \
   $repo/test_wavs/DEV_T0000000002.wav
 
+rm -rf $repo
 
 log "------------------------------------------------------------"
 log "Run streaming Conformer CTC from WeNet"
 log "------------------------------------------------------------"
 wenet_models=(
 sherpa-onnx-zh-wenet-aishell
-sherpa-onnx-zh-wenet-aishell2
-sherpa-onnx-zh-wenet-wenetspeech
-sherpa-onnx-zh-wenet-multi-cn
+# sherpa-onnx-zh-wenet-aishell2
+# sherpa-onnx-zh-wenet-wenetspeech
+# sherpa-onnx-zh-wenet-multi-cn
 sherpa-onnx-en-wenet-librispeech
-sherpa-onnx-en-wenet-gigaspeech
+# sherpa-onnx-en-wenet-gigaspeech
 )
 for name in ${wenet_models[@]}; do
-  repo_url=https://huggingface.co/csukuangfj/$name
+  repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$name.tar.bz2
+  curl -SL -O $repo_url
+  tar xvf $name.tar.bz2
+  rm $name.tar.bz2
+  repo=$name
   log "Start testing ${repo_url}"
-  repo=$(basename $repo_url)
-  log "Download pretrained model and test-data from $repo_url"
-  GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url
-  pushd $repo
-  git lfs pull --include "*.onnx"
-  ls -lh *.onnx
-  popd
 
   log "test float32 models"
   time $EXE \