diff --git a/.github/scripts/test-offline-ctc.sh b/.github/scripts/test-offline-ctc.sh index 69a948044..4a5955464 100755 --- a/.github/scripts/test-offline-ctc.sh +++ b/.github/scripts/test-offline-ctc.sh @@ -124,15 +124,13 @@ sherpa-onnx-en-wenet-librispeech # sherpa-onnx-en-wenet-gigaspeech ) for name in ${wenet_models[@]}; do - repo_url=https://huggingface.co/csukuangfj/$name + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$name.tar.bz2 log "Start testing ${repo_url}" - repo=$(basename $repo_url) + repo=$name log "Download pretrained model and test-data from $repo_url" - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - pushd $repo - git lfs pull --include "*.onnx" - ls -lh *.onnx - popd + curl -SL -O $repo_url + tar xvf $name.tar.bz2 + rm $name.tar.bz2 log "test float32 models" time $EXE \ @@ -156,16 +154,13 @@ done log "------------------------------------------------------------" log "Run tdnn yesno (Hebrew)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-tdnn-yesno -log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" - -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd +url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-tdnn-yesno.tar.bz2 +curl -SL -O $url +tar xvf sherpa-onnx-tdnn-yesno.tar.bz2 +rm sherpa-onnx-tdnn-yesno.tar.bz2 +log "Start testing ${url}" +repo=sherpa-onnx-tdnn-yesno +log "Download pretrained model and test-data from $url" log "test float32 models" time $EXE \ @@ -201,17 +196,14 @@ log "------------------------------------------------------------" log "Run Citrinet (stt_en_citrinet_512, English)" log "------------------------------------------------------------" -repo_url=http://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-citrinet-512 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 +rm sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 log "Start testing ${repo_url}" -repo=$(basename $repo_url) +repo=sherpa-onnx-nemo-ctc-en-citrinet-512 log "Download pretrained model and test-data from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd - time $EXE \ --tokens=$repo/tokens.txt \ --nemo-ctc-model=$repo/model.onnx \ @@ -233,18 +225,14 @@ rm -rf $repo log "------------------------------------------------------------" log "Run Librispeech zipformer CTC H/HL/HLG decoding (English) " log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-ctc-en-2023-10-02 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +curl -SL -O $repo_url log "Start testing ${repo_url}" -repo=$(basename $repo_url) +tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +repo=sherpa-onnx-zipformer-ctc-en-2023-10-02 log "Download pretrained model and test-data from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -git lfs pull --include "*.fst" -ls -lh -popd - graphs=( $repo/H.fst $repo/HL.fst diff --git a/.github/scripts/test-offline-transducer.sh b/.github/scripts/test-offline-transducer.sh index f43614f57..c8047bab6 100755 --- a/.github/scripts/test-offline-transducer.sh +++ b/.github/scripts/test-offline-transducer.sh @@ -118,17 +118,14 @@ log "------------------------------------------------------------" log "Run Conformer transducer (English)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-conformer-en-2023-03-18 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-conformer-en-2023-03-18.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-conformer-en-2023-03-18.tar.bz2 +rm sherpa-onnx-conformer-en-2023-03-18.tar.bz2 log "Start testing ${repo_url}" -repo=$(basename $repo_url) +repo=sherpa-onnx-conformer-en-2023-03-18 log "Download pretrained model and test-data from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd - time $EXE \ --tokens=$repo/tokens.txt \ --encoder=$repo/encoder-epoch-99-avg-1.onnx \ @@ -155,16 +152,12 @@ log "------------------------------------------------------------" log "Run Zipformer transducer (English)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-en-2023-03-30 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-03-30.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-zipformer-en-2023-03-30.tar.bz2 +rm sherpa-onnx-zipformer-en-2023-03-30.tar.bz2 +repo=sherpa-onnx-zipformer-en-2023-03-30 log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" - -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd time $EXE \ --tokens=$repo/tokens.txt \ @@ -192,16 +185,12 @@ log "------------------------------------------------------------" log "Run Paraformer (Chinese)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 +rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 +repo=sherpa-onnx-paraformer-zh-2023-03-28 log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" - -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd time $EXE \ --tokens=$repo/tokens.txt \ @@ -229,16 +218,13 @@ log "------------------------------------------------------------" log "Run Paraformer (Chinese) with timestamps" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-zh-2023-09-14 -log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 +rm sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 +repo=sherpa-onnx-paraformer-zh-2023-09-14 -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd +log "Start testing ${repo_url}" time $EXE \ --tokens=$repo/tokens.txt \ diff --git a/.github/scripts/test-offline-tts.sh b/.github/scripts/test-offline-tts.sh index e611c3d2e..d3d35df2c 100755 --- a/.github/scripts/test-offline-tts.sh +++ b/.github/scripts/test-offline-tts.sh @@ -40,15 +40,13 @@ log "------------------------------------------------------------" log "vits-ljs test" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/vits-ljs +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-ljs.tar.bz2 +curl -SL -O $repo_url +tar xvf vits-ljs.tar.bz2 +rm vits-ljs.tar.bz2 +repo=vits-ljs + log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd $EXE \ --vits-model=$repo/vits-ljs.onnx \ @@ -65,15 +63,13 @@ log "------------------------------------------------------------" log "vits-vctk test" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/vits-vctk +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-vctk.tar.bz2 +curl -SL -O $repo_url +tar xvf vits-vctk.tar.bz2 +rm vits-vctk.tar.bz2 +repo=vits-vctk + log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd for sid in 0 10 90; do $EXE \ @@ -93,15 +89,13 @@ log "------------------------------------------------------------" log "vits-zh-aishell3" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/vits-zh-aishell3 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 +curl -SL -O $repo_url +tar xvf vits-zh-aishell3.tar.bz2 +rm vits-zh-aishell3.tar.bz2 +repo=vits-zh-aishell3 + log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd for sid in 0 10 90; do $EXE \ diff --git a/.github/scripts/test-offline-whisper.sh b/.github/scripts/test-offline-whisper.sh index e2987b5de..d5a60890a 100755 --- a/.github/scripts/test-offline-whisper.sh +++ b/.github/scripts/test-offline-whisper.sh @@ -33,17 +33,12 @@ for name in ${names[@]}; do log "Run $name" log "------------------------------------------------------------" - repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-whisper-$name + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-$name.tar.bz2 + curl -SL -O $repo_url + tar xvf sherpa-onnx-whisper-$name.tar.bz2 + rm sherpa-onnx-whisper-$name.tar.bz2 + repo=sherpa-onnx-whisper-$name log "Start testing ${repo_url}" - repo=$(basename $repo_url) - log "Download pretrained model and test-data from $repo_url" - - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - pushd $repo - git lfs pull --include "*.onnx" - # git lfs pull --include "*.ort" - ls -lh *.onnx - popd log "test fp32 onnx" diff --git a/.github/scripts/test-online-ctc.sh b/.github/scripts/test-online-ctc.sh index a81b6ff22..fe5716f0a 100755 --- a/.github/scripts/test-online-ctc.sh +++ b/.github/scripts/test-online-ctc.sh @@ -55,7 +55,7 @@ $EXE \ $repo/test_wavs/1.wav \ $repo/test_wavs/8k.wav -rm -rf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18 +rm -rf $repo log "------------------------------------------------------------" log "Run streaming Zipformer2 CTC " @@ -87,28 +87,26 @@ time $EXE \ $repo/test_wavs/DEV_T0000000001.wav \ $repo/test_wavs/DEV_T0000000002.wav +rm -rf $repo log "------------------------------------------------------------" log "Run streaming Conformer CTC from WeNet" log "------------------------------------------------------------" wenet_models=( sherpa-onnx-zh-wenet-aishell -sherpa-onnx-zh-wenet-aishell2 -sherpa-onnx-zh-wenet-wenetspeech -sherpa-onnx-zh-wenet-multi-cn +# sherpa-onnx-zh-wenet-aishell2 +# sherpa-onnx-zh-wenet-wenetspeech +# sherpa-onnx-zh-wenet-multi-cn sherpa-onnx-en-wenet-librispeech -sherpa-onnx-en-wenet-gigaspeech +# sherpa-onnx-en-wenet-gigaspeech ) for name in ${wenet_models[@]}; do - repo_url=https://huggingface.co/csukuangfj/$name + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$name.tar.bz2 + curl -SL -O $repo_url + tar xvf $name.tar.bz2 + rm $name.tar.bz2 + repo=$name log "Start testing ${repo_url}" - repo=$(basename $repo_url) - log "Download pretrained model and test-data from $repo_url" - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - pushd $repo - git lfs pull --include "*.onnx" - ls -lh *.onnx - popd log "test float32 models" time $EXE \ diff --git a/.github/scripts/test-online-paraformer.sh b/.github/scripts/test-online-paraformer.sh index 6b5d3cf60..af2df5316 100755 --- a/.github/scripts/test-online-paraformer.sh +++ b/.github/scripts/test-online-paraformer.sh @@ -19,16 +19,13 @@ log "------------------------------------------------------------" log "Run streaming Paraformer" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 +rm sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 +repo=sherpa-onnx-streaming-paraformer-bilingual-zh-en + log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" - -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd time $EXE \ --tokens=$repo/tokens.txt \ diff --git a/.github/scripts/test-online-transducer.sh b/.github/scripts/test-online-transducer.sh index f8a76d763..e894358a5 100755 --- a/.github/scripts/test-online-transducer.sh +++ b/.github/scripts/test-online-transducer.sh @@ -19,16 +19,13 @@ log "------------------------------------------------------------" log "Run LSTM transducer (English)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-lstm-en-2023-02-17 -log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-lstm-en-2023-02-17.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-lstm-en-2023-02-17.tar.bz2 +rm sherpa-onnx-lstm-en-2023-02-17.tar.bz2 +repo=sherpa-onnx-lstm-en-2023-02-17 -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd +log "Start testing ${repo_url}" waves=( $repo/test_wavs/0.wav @@ -62,16 +59,13 @@ log "------------------------------------------------------------" log "Run LSTM transducer (Chinese)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-lstm-zh-2023-02-20 -log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-lstm-zh-2023-02-20.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-lstm-zh-2023-02-20.tar.bz2 +rm sherpa-onnx-lstm-zh-2023-02-20.tar.bz2 +repo=sherpa-onnx-lstm-zh-2023-02-20 -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd +log "Start testing ${repo_url}" waves=( $repo/test_wavs/0.wav @@ -105,16 +99,13 @@ log "------------------------------------------------------------" log "Run streaming Zipformer transducer (English)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-02-21 -log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-2023-02-21.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-streaming-zipformer-en-2023-02-21.tar.bz2 +rm sherpa-onnx-streaming-zipformer-en-2023-02-21.tar.bz2 +repo=sherpa-onnx-streaming-zipformer-en-2023-02-21 -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd +log "Start testing ${repo_url}" waves=( $repo/test_wavs/0.wav @@ -150,16 +141,13 @@ log "------------------------------------------------------------" log "Run streaming Zipformer transducer (Bilingual, Chinese + English)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 -log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 +repo=sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd +log "Start testing ${repo_url}" waves=( $repo/test_wavs/0.wav @@ -216,16 +204,13 @@ log "------------------------------------------------------------" log "Run streaming Conformer transducer (English)" log "------------------------------------------------------------" -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-streaming-conformer-en-2023-05-09 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-conformer-en-2023-05-09.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-streaming-conformer-en-2023-05-09.tar.bz2 +rm sherpa-onnx-streaming-conformer-en-2023-05-09.tar.bz2 +repo=sherpa-onnx-streaming-conformer-en-2023-05-09 + log "Start testing ${repo_url}" -repo=$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" - -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -pushd $repo -git lfs pull --include "*.onnx" -ls -lh *.onnx -popd waves=( $repo/test_wavs/0.wav diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index e6d4c17a4..4f7508a0b 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -102,20 +102,16 @@ wenet_models=( # sherpa-onnx-zh-wenet-wenetspeech # sherpa-onnx-zh-wenet-multi-cn sherpa-onnx-en-wenet-librispeech -sherpa-onnx-en-wenet-gigaspeech +# sherpa-onnx-en-wenet-gigaspeech ) for name in ${wenet_models[@]}; do - repo_url=https://huggingface.co/csukuangfj/$name + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$name.tar.bz2 + curl -SL -O $repo_url + tar xvf $name.tar.bz2 + rm $name.tar.bz2 + repo=$name log "Start testing ${repo_url}" - repo=$dir/$(basename $repo_url) - log "Download pretrained model and test-data from $repo_url" - pushd $dir - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - cd $repo - git lfs pull --include "*.onnx" - ls -lh *.onnx - popd python3 ./python-api-examples/offline-decode-files.py \ --tokens=$repo/tokens.txt \ @@ -206,16 +202,14 @@ log "Test streaming transducer models" if [[ x$OS != x'windows-latest' ]]; then echo "OS: $OS" pushd $dir - repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + curl -SL -O $repo_url + tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + repo=sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 log "Start testing ${repo_url}" - repo=$dir/$(basename $repo_url) - log "Download pretrained model and test-data from $repo_url" - - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - cd $repo - git lfs pull --include "*.onnx" - popd + repo=$dir/$repo python3 -c "import sherpa_onnx; print(sherpa_onnx.__file__)" sherpa_onnx_version=$(python3 -c "import sherpa_onnx; print(sherpa_onnx.__version__)") @@ -255,15 +249,14 @@ fi log "Test non-streaming transducer models" pushd $dir -repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-en-2023-04-01 - -log "Start testing ${repo_url}" -repo=$dir/$(basename $repo_url) +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 log "Download pretrained model and test-data from $repo_url" -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -cd $repo -git lfs pull --include "*.onnx" +curl -SL -O $repo_url +tar xvf sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 +rm sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 +repo=$dir/sherpa-onnx-zipformer-en-2023-04-01 + popd ls -lh $repo @@ -295,16 +288,13 @@ log "Test non-streaming paraformer models" if [[ x$OS != x'windows-latest' ]]; then echo "OS: $OS" pushd $dir - repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28 + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + curl -SL -O $repo_url + tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 log "Start testing ${repo_url}" - repo=$dir/$(basename $repo_url) - log "Download pretrained model and test-data from $repo_url" - - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - cd $repo - git lfs pull --include "*.onnx" - popd + repo=$dir/sherpa-onnx-paraformer-zh-2023-03-28 ls -lh $repo @@ -332,16 +322,13 @@ fi log "Test non-streaming NeMo CTC models" pushd $dir -repo_url=http://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-citrinet-512 +repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 +curl -SL -O $repo_url +tar xvf sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 +rm sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 log "Start testing ${repo_url}" -repo=$dir/$(basename $repo_url) -log "Download pretrained model and test-data from $repo_url" - -GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url -cd $repo -git lfs pull --include "*.onnx" -popd +repo=$dir/sherpa-onnx-nemo-ctc-en-citrinet-512 ls -lh $repo diff --git a/.github/scripts/test-spoken-language-identification.sh b/.github/scripts/test-spoken-language-identification.sh index fec3d4382..851e34220 100755 --- a/.github/scripts/test-spoken-language-identification.sh +++ b/.github/scripts/test-spoken-language-identification.sh @@ -73,18 +73,13 @@ for name in ${names[@]}; do log "------------------------------------------------------------" log "Run $name" log "------------------------------------------------------------" + repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-$name.tar.bz2 + curl -SL -O $repo_url + tar xvf sherpa-onnx-whisper-$name.tar.bz2 + rm sherpa-onnx-whisper-$name.tar.bz2 - repo_url=https://huggingface.co/csukuangfj/sherpa-onnx-whisper-$name log "Start testing ${repo_url}" - repo=$(basename $repo_url) - log "Download pretrained model and test-data from $repo_url" - - GIT_LFS_SKIP_SMUDGE=1 git clone $repo_url - pushd $repo - git lfs pull --include "*.onnx" - # git lfs pull --include "*.ort" - ls -lh *.onnx - popd + repo=sherpa-onnx-whisper-$name for wav in ${waves[@]}; do log "test fp32 onnx" diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index 01d601aba..7f907cf99 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -192,6 +192,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 27579ea1e..94bcfdb00 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -183,6 +183,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index 452e2cb3c..9fb400d4c 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -120,6 +120,7 @@ jobs: ls -lh rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/apk-asr-2pass.yaml b/.github/workflows/apk-asr-2pass.yaml index 36272cc38..a79a28af3 100644 --- a/.github/workflows/apk-asr-2pass.yaml +++ b/.github/workflows/apk-asr-2pass.yaml @@ -158,6 +158,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-asr.yaml b/.github/workflows/apk-asr.yaml index c822c0ec8..d381cb1af 100644 --- a/.github/workflows/apk-asr.yaml +++ b/.github/workflows/apk-asr.yaml @@ -158,6 +158,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-audio-tagging-wearos.yaml b/.github/workflows/apk-audio-tagging-wearos.yaml index 67fb1ea38..b2d2d959a 100644 --- a/.github/workflows/apk-audio-tagging-wearos.yaml +++ b/.github/workflows/apk-audio-tagging-wearos.yaml @@ -158,6 +158,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-audio-tagging.yaml b/.github/workflows/apk-audio-tagging.yaml index 8d18241fe..db739021f 100644 --- a/.github/workflows/apk-audio-tagging.yaml +++ b/.github/workflows/apk-audio-tagging.yaml @@ -158,6 +158,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-kws.yaml b/.github/workflows/apk-kws.yaml index 30a284f2a..8c348e5a3 100644 --- a/.github/workflows/apk-kws.yaml +++ b/.github/workflows/apk-kws.yaml @@ -155,6 +155,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-speaker-identification.yaml b/.github/workflows/apk-speaker-identification.yaml index 9a4bc2194..2d674c78c 100644 --- a/.github/workflows/apk-speaker-identification.yaml +++ b/.github/workflows/apk-speaker-identification.yaml @@ -96,6 +96,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-spoken-language-identification.yaml b/.github/workflows/apk-spoken-language-identification.yaml index 39e1e1b7f..dd4322156 100644 --- a/.github/workflows/apk-spoken-language-identification.yaml +++ b/.github/workflows/apk-spoken-language-identification.yaml @@ -158,6 +158,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-tts-engine.yaml b/.github/workflows/apk-tts-engine.yaml index 6ed8ef05d..89013a59c 100644 --- a/.github/workflows/apk-tts-engine.yaml +++ b/.github/workflows/apk-tts-engine.yaml @@ -159,6 +159,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-tts.yaml b/.github/workflows/apk-tts.yaml index eedcf5dc9..1b51368ed 100644 --- a/.github/workflows/apk-tts.yaml +++ b/.github/workflows/apk-tts.yaml @@ -159,6 +159,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-vad-asr.yaml b/.github/workflows/apk-vad-asr.yaml index 09e8c73d4..aea8028d7 100644 --- a/.github/workflows/apk-vad-asr.yaml +++ b/.github/workflows/apk-vad-asr.yaml @@ -158,6 +158,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/apk-vad.yaml b/.github/workflows/apk-vad.yaml index e11edadfd..ddd8c6afb 100644 --- a/.github/workflows/apk-vad.yaml +++ b/.github/workflows/apk-vad.yaml @@ -155,6 +155,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface cd huggingface diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index 762239880..0f47de078 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -92,6 +92,7 @@ jobs: shell: bash run: | git lfs install + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/arm-linux-gcc ls -lh arm-linux-gcc diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index 4ecc7a415..971324e9b 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -93,6 +93,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-wheels-armv7l.yaml b/.github/workflows/build-wheels-armv7l.yaml index 000c698b5..a720a39f4 100644 --- a/.github/workflows/build-wheels-armv7l.yaml +++ b/.github/workflows/build-wheels-armv7l.yaml @@ -96,6 +96,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index 0443074c2..7a4f59011 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -92,6 +92,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml index 7b9182427..2cdea3f78 100644 --- a/.github/workflows/build-wheels-macos-arm64.yaml +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -62,6 +62,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml index 755eb7605..13dc292dc 100644 --- a/.github/workflows/build-wheels-macos-x64.yaml +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -79,6 +79,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-wheels-win32.yaml b/.github/workflows/build-wheels-win32.yaml index bd34d1e59..752fbef32 100644 --- a/.github/workflows/build-wheels-win32.yaml +++ b/.github/workflows/build-wheels-win32.yaml @@ -61,6 +61,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-wheels-win64.yaml b/.github/workflows/build-wheels-win64.yaml index 670ead539..ee9f020a8 100644 --- a/.github/workflows/build-wheels-win64.yaml +++ b/.github/workflows/build-wheels-win64.yaml @@ -62,6 +62,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface @@ -91,6 +92,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface diff --git a/.github/workflows/build-xcframework.yaml b/.github/workflows/build-xcframework.yaml index d4e8c8967..97eb33515 100644 --- a/.github/workflows/build-xcframework.yaml +++ b/.github/workflows/build-xcframework.yaml @@ -134,6 +134,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/export-ced-to-onnx.yaml b/.github/workflows/export-ced-to-onnx.yaml index 506abe513..70c4cc5fb 100644 --- a/.github/workflows/export-ced-to-onnx.yaml +++ b/.github/workflows/export-ced-to-onnx.yaml @@ -65,6 +65,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 d=sherpa-onnx-ced-$m-audio-tagging-2024-04-19 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/k2-fsa/$d huggingface mv -v $d/* huggingface cd huggingface diff --git a/.github/workflows/export-wenet-to-onnx.yaml b/.github/workflows/export-wenet-to-onnx.yaml index 4f0b492ee..165017648 100644 --- a/.github/workflows/export-wenet-to-onnx.yaml +++ b/.github/workflows/export-wenet-to-onnx.yaml @@ -47,6 +47,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-aishell huggingface cd huggingface @@ -88,6 +89,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-aishell2 huggingface cd huggingface @@ -129,6 +131,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-multi-cn huggingface cd huggingface @@ -170,6 +173,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-wenetspeech huggingface cd huggingface @@ -211,6 +215,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-en-wenet-librispeech huggingface cd huggingface @@ -253,6 +258,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/csukuangfj/sherpa-onnx-en-wenet-gigaspeech huggingface cd huggingface diff --git a/.github/workflows/export-whisper-to-onnx.yaml b/.github/workflows/export-whisper-to-onnx.yaml index 4aa8a9d1a..2dd6aa283 100644 --- a/.github/workflows/export-whisper-to-onnx.yaml +++ b/.github/workflows/export-whisper-to-onnx.yaml @@ -142,6 +142,8 @@ jobs: git config --global user.email "csukuangfj@gmail.com" git config --global user.name "Fangjun Kuang" + export GIT_CLONE_PROTECTION_ACTIVE=false + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-${{ matrix.model }} huggingface if [[ $model != medium-aishell ]]; then diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 154d6d774..ff06206e5 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -69,6 +69,7 @@ jobs: run: | echo "pwd: $PWD" ls -lh + du -h -d1 . - name: Build sherpa-onnx uses: addnab/docker-run-action@v3 @@ -109,6 +110,7 @@ jobs: - name: Display dependencies of sherpa-onnx for linux shell: bash run: | + du -h -d1 . sudo chown -R $USER ./build ls -lh build/bin ls -lh build/_deps/onnxruntime-src/lib/ @@ -128,121 +130,148 @@ jobs: name: release-${{ matrix.build_type }}-with-shared-lib-${{ matrix.shared_lib }}-with-tts-${{ matrix.with_tts }} path: install/* - - name: Test online CTC + - name: Test online transducer shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx - .github/scripts/test-online-ctc.sh + .github/scripts/test-online-transducer.sh + du -h -d1 . - - name: Test offline transducer + - name: Test online transducer (C API) + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export EXE=decode-file-c-api + + .github/scripts/test-online-transducer.sh + du -h -d1 . + + - name: Test offline CTC shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline - .github/scripts/test-offline-transducer.sh + .github/scripts/test-offline-ctc.sh + du -h -d1 . - name: Test spoken language identification (C++ API) shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline-language-identification .github/scripts/test-spoken-language-identification.sh + du -h -d1 . - - name: Test C API + - name: Test online CTC shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH - export SLID_EXE=spoken-language-identification-c-api - export SID_EXE=speaker-identification-c-api - export AT_EXE=audio-tagging-c-api - export PUNCT_EXE=add-punctuation-c-api + export EXE=sherpa-onnx - .github/scripts/test-c-api.sh + .github/scripts/test-online-ctc.sh + du -h -d1 . - - name: Test offline CTC + - name: Test offline transducer shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline - .github/scripts/test-offline-ctc.sh + .github/scripts/test-offline-transducer.sh + du -h -d1 . + + - name: Test C API + shell: bash + run: | + du -h -d1 . + export PATH=$PWD/build/bin:$PATH + export SLID_EXE=spoken-language-identification-c-api + export SID_EXE=speaker-identification-c-api + export AT_EXE=audio-tagging-c-api + export PUNCT_EXE=add-punctuation-c-api + + .github/scripts/test-c-api.sh + du -h -d1 . - name: Test offline punctuation shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline-punctuation .github/scripts/test-offline-punctuation.sh + du -h -d1 . - name: Test Audio tagging shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline-audio-tagging .github/scripts/test-audio-tagging.sh + du -h -d1 . - name: Test transducer kws shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-keyword-spotter .github/scripts/test-kws.sh + du -h -d1 . - name: Test offline Whisper if: matrix.build_type != 'Debug' shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline readelf -d build/bin/sherpa-onnx-offline .github/scripts/test-offline-whisper.sh + du -h -d1 . - name: Test offline TTS if: matrix.with_tts == 'ON' shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx-offline-tts .github/scripts/test-offline-tts.sh + du -h -d1 . - name: Test online paraformer shell: bash run: | + du -h -d1 . export PATH=$PWD/build/bin:$PATH export EXE=sherpa-onnx .github/scripts/test-online-paraformer.sh - - - name: Test online transducer - shell: bash - run: | - export PATH=$PWD/build/bin:$PATH - export EXE=sherpa-onnx - - .github/scripts/test-online-transducer.sh - - - name: Test online transducer (C API) - shell: bash - run: | - export PATH=$PWD/build/bin:$PATH - export EXE=decode-file-c-api - - .github/scripts/test-online-transducer.sh + du -h -d1 . - name: Copy files shell: bash if: matrix.build_type == 'Release' run: | + du -h -d1 . SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) if [[ ${{ matrix.shared_lib }} == 'ON' ]]; then @@ -265,6 +294,7 @@ jobs: tree $dst tar cjvf ${dst}.tar.bz2 $dst + du -h -d1 . - name: Publish to huggingface if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && matrix.build_type == 'Release' @@ -276,11 +306,14 @@ jobs: timeout_seconds: 200 shell: bash command: | + du -h -d1 . git config --global user.email "csukuangfj@gmail.com" git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + du -h -d1 . cd huggingface git lfs pull @@ -296,6 +329,7 @@ jobs: git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}" git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + du -h -d1 . - name: Release pre-compiled binaries and libs for linux x64 if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.build_type == 'Release' diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml index a5869a4b0..0f29cd3d9 100644 --- a/.github/workflows/riscv64-linux.yaml +++ b/.github/workflows/riscv64-linux.yaml @@ -239,6 +239,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface diff --git a/.github/workflows/test-python-offline-websocket-server.yaml b/.github/workflows/test-python-offline-websocket-server.yaml index 18183f482..aeb10881e 100644 --- a/.github/workflows/test-python-offline-websocket-server.yaml +++ b/.github/workflows/test-python-offline-websocket-server.yaml @@ -73,10 +73,9 @@ jobs: if: matrix.model_type == 'transducer' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-en-2023-06-26 - cd sherpa-onnx-zipformer-en-2023-06-26 - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-06-26.tar.bz2 + tar xvf sherpa-onnx-zipformer-en-2023-06-26.tar.bz2 + rm sherpa-onnx-zipformer-en-2023-06-26.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --encoder ./sherpa-onnx-zipformer-en-2023-06-26/encoder-epoch-99-avg-1.onnx \ @@ -105,14 +104,13 @@ jobs: if: matrix.model_type == 'paraformer' && matrix.os != 'windows-latest' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en - cd sherpa-onnx-paraformer-bilingual-zh-en - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ - --paraformer ./sherpa-onnx-paraformer-bilingual-zh-en/model.int8.onnx \ - --tokens ./sherpa-onnx-paraformer-bilingual-zh-en/tokens.txt & + --paraformer ./sherpa-onnx-paraformer-zh-2023-03-28/model.int8.onnx \ + --tokens ./sherpa-onnx-paraformer-zh-2023-03-28/tokens.txt & echo "sleep 10 seconds to wait the server start" sleep 10 @@ -122,25 +120,24 @@ jobs: shell: bash run: | python3 ./python-api-examples/offline-websocket-client-decode-files-paralell.py \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/0.wav \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/1.wav \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/2.wav \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/8k.wav + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/0.wav \ + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/1.wav \ + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/2.wav \ + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/8k.wav python3 ./python-api-examples/offline-websocket-client-decode-files-sequential.py \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/0.wav \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/1.wav \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/2.wav \ - ./sherpa-onnx-paraformer-bilingual-zh-en/test_wavs/8k.wav + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/0.wav \ + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/1.wav \ + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/2.wav \ + ./sherpa-onnx-paraformer-zh-2023-03-28/test_wavs/8k.wav - name: Start server for nemo_ctc models if: matrix.model_type == 'nemo_ctc' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-conformer-medium - cd sherpa-onnx-nemo-ctc-en-conformer-medium - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 + tar xvf sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 + rm sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --nemo-ctc ./sherpa-onnx-nemo-ctc-en-conformer-medium/model.onnx \ @@ -167,10 +164,9 @@ jobs: if: matrix.model_type == 'whisper' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-tiny.en - cd sherpa-onnx-whisper-tiny.en - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 + rm sherpa-onnx-whisper-tiny.en.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --whisper-encoder=./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.onnx \ @@ -198,10 +194,9 @@ jobs: if: matrix.model_type == 'tdnn' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-tdnn-yesno - cd sherpa-onnx-tdnn-yesno - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-tdnn-yesno.tar.bz2 + tar xvf sherpa-onnx-tdnn-yesno.tar.bz2 + rm sherpa-onnx-tdnn-yesno.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --tdnn-model=./sherpa-onnx-tdnn-yesno/model-epoch-14-avg-2.onnx \ diff --git a/.github/workflows/test-python-online-websocket-server.yaml b/.github/workflows/test-python-online-websocket-server.yaml index 888af36ae..9ef57dbe7 100644 --- a/.github/workflows/test-python-online-websocket-server.yaml +++ b/.github/workflows/test-python-online-websocket-server.yaml @@ -94,10 +94,9 @@ jobs: if: matrix.model_type == 'transducer' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26 - cd sherpa-onnx-streaming-zipformer-en-2023-06-26 - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 + tar xvf sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 + rm sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 python3 ./python-api-examples/streaming_server.py \ --encoder ./sherpa-onnx-streaming-zipformer-en-2023-06-26/encoder-epoch-99-avg-1-chunk-16-left-128.onnx \ @@ -118,10 +117,9 @@ jobs: if: matrix.model_type == 'paraformer' shell: bash run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en - cd sherpa-onnx-streaming-paraformer-bilingual-zh-en - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 + tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 + rm sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 python3 ./python-api-examples/streaming_server.py \ --tokens ./sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt \ diff --git a/.github/workflows/wasm-simd-hf-space-de-tts.yaml b/.github/workflows/wasm-simd-hf-space-de-tts.yaml index 31943dff4..2c1d978aa 100644 --- a/.github/workflows/wasm-simd-hf-space-de-tts.yaml +++ b/.github/workflows/wasm-simd-hf-space-de-tts.yaml @@ -88,6 +88,7 @@ jobs: rm -rf ms export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone http://www.modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-de.git ms @@ -126,6 +127,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de huggingface cd huggingface diff --git a/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml b/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml index cd9ba7863..e123e65f1 100644 --- a/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-en-asr-zipformer.yaml @@ -90,6 +90,7 @@ jobs: rm -rf ms export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://www.modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-en.git ms cd ms @@ -126,6 +127,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-en huggingface cd huggingface diff --git a/.github/workflows/wasm-simd-hf-space-en-tts.yaml b/.github/workflows/wasm-simd-hf-space-en-tts.yaml index 68c065955..09d42911b 100644 --- a/.github/workflows/wasm-simd-hf-space-en-tts.yaml +++ b/.github/workflows/wasm-simd-hf-space-en-tts.yaml @@ -86,6 +86,7 @@ jobs: rm -rf ms export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://www.modelscope.cn/studios/k2-fsa/web-assembly-tts-sherpa-onnx-en.git ms cd ms @@ -122,6 +123,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-en huggingface cd huggingface diff --git a/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml b/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml index 53e50cadc..1c7625655 100644 --- a/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-zh-cantonese-en-asr-paraformer.yaml @@ -96,6 +96,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer huggingface cd huggingface @@ -133,6 +134,7 @@ jobs: rm -rf ms export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://www.modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-cantonese-en-paraformer.git ms cd ms diff --git a/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml b/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml index 6fdd35cca..a67385b12 100644 --- a/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-zh-en-asr-paraformer.yaml @@ -97,6 +97,7 @@ jobs: rm -rf ms export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://www.modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer.git ms cd ms @@ -133,6 +134,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en-paraformer huggingface cd huggingface diff --git a/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml b/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml index 89c2e987e..4dea90f49 100644 --- a/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml +++ b/.github/workflows/wasm-simd-hf-space-zh-en-asr-zipformer.yaml @@ -88,6 +88,7 @@ jobs: rm -rf ms export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://www.modelscope.cn/studios/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en.git ms cd ms @@ -124,6 +125,7 @@ jobs: rm -rf huggingface export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/spaces/k2-fsa/web-assembly-asr-sherpa-onnx-zh-en huggingface cd huggingface diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index d160e475e..7d26157b7 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -215,6 +215,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index c476ab107..c140dad8a 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -217,6 +217,7 @@ jobs: git config --global user.name "Fangjun Kuang" rm -rf huggingface + export GIT_CLONE_PROTECTION_ACTIVE=false GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface cd huggingface diff --git a/dotnet-examples/offline-decode-files/run-nemo-ctc.sh b/dotnet-examples/offline-decode-files/run-nemo-ctc.sh index 44e4ee0b6..b65da2c25 100755 --- a/dotnet-examples/offline-decode-files/run-nemo-ctc.sh +++ b/dotnet-examples/offline-decode-files/run-nemo-ctc.sh @@ -3,10 +3,9 @@ set -ex if [ ! -d ./sherpa-onnx-nemo-ctc-en-conformer-medium ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-conformer-medium - cd sherpa-onnx-nemo-ctc-en-conformer-medium - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 + tar xvf sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 + rm sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 fi dotnet run \ diff --git a/dotnet-examples/offline-decode-files/run-paraformer.sh b/dotnet-examples/offline-decode-files/run-paraformer.sh index 4fb72f418..acd8e9d45 100755 --- a/dotnet-examples/offline-decode-files/run-paraformer.sh +++ b/dotnet-examples/offline-decode-files/run-paraformer.sh @@ -3,10 +3,9 @@ set -ex if [ ! -d ./sherpa-onnx-paraformer-zh-2023-03-28 ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28 - cd sherpa-onnx-paraformer-zh-2023-03-28 - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 + rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 fi dotnet run \ diff --git a/dotnet-examples/offline-decode-files/run-tdnn-yesno.sh b/dotnet-examples/offline-decode-files/run-tdnn-yesno.sh index c2d8e9fdd..a424d7504 100755 --- a/dotnet-examples/offline-decode-files/run-tdnn-yesno.sh +++ b/dotnet-examples/offline-decode-files/run-tdnn-yesno.sh @@ -3,10 +3,9 @@ set -ex if [ ! -d ./sherpa-onnx-tdnn-yesno ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-tdnn-yesno - cd sherpa-onnx-tdnn-yesno - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-tdnn-yesno.tar.bz2 + tar xvf sherpa-onnx-tdnn-yesno.tar.bz2 + rm sherpa-onnx-tdnn-yesno.tar.bz2 fi dotnet run \ diff --git a/dotnet-examples/offline-decode-files/run-whisper.sh b/dotnet-examples/offline-decode-files/run-whisper.sh index 6f5e26c64..934b7363d 100755 --- a/dotnet-examples/offline-decode-files/run-whisper.sh +++ b/dotnet-examples/offline-decode-files/run-whisper.sh @@ -3,10 +3,9 @@ set -ex if [ ! -d ./sherpa-onnx-whisper-tiny.en ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-tiny.en - cd sherpa-onnx-whisper-tiny.en - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 + tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 + rm sherpa-onnx-whisper-tiny.en.tar.bz2 fi dotnet run \ diff --git a/dotnet-examples/online-decode-files/run-paraformer.sh b/dotnet-examples/online-decode-files/run-paraformer.sh index 5731d0138..122d9a062 100755 --- a/dotnet-examples/online-decode-files/run-paraformer.sh +++ b/dotnet-examples/online-decode-files/run-paraformer.sh @@ -6,10 +6,9 @@ set -ex if [ ! -d ./sherpa-onnx-streaming-paraformer-bilingual-zh-en ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en - cd sherpa-onnx-streaming-paraformer-bilingual-zh-en - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 + tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 + rm sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 fi dotnet run -c Release \ diff --git a/dotnet-examples/online-decode-files/run-transducer.sh b/dotnet-examples/online-decode-files/run-transducer.sh index b99fc1e53..b3ca7c7c4 100755 --- a/dotnet-examples/online-decode-files/run-transducer.sh +++ b/dotnet-examples/online-decode-files/run-transducer.sh @@ -8,6 +8,7 @@ set -ex if [ ! -d ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 fi dotnet run -c Release \ diff --git a/dotnet-examples/speech-recognition-from-microphone/run-paraformer.sh b/dotnet-examples/speech-recognition-from-microphone/run-paraformer.sh index ec4e15d78..52628c840 100755 --- a/dotnet-examples/speech-recognition-from-microphone/run-paraformer.sh +++ b/dotnet-examples/speech-recognition-from-microphone/run-paraformer.sh @@ -6,10 +6,9 @@ set -ex if [ ! -d ./sherpa-onnx-streaming-paraformer-bilingual-zh-en ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en - cd sherpa-onnx-streaming-paraformer-bilingual-zh-en - git lfs pull --include "*.onnx" - cd .. + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 + tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 + rm sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 fi dotnet run -c Release \ diff --git a/dotnet-examples/speech-recognition-from-microphone/run-transducer.sh b/dotnet-examples/speech-recognition-from-microphone/run-transducer.sh index 1c1dc11af..e6184b4fc 100755 --- a/dotnet-examples/speech-recognition-from-microphone/run-transducer.sh +++ b/dotnet-examples/speech-recognition-from-microphone/run-transducer.sh @@ -9,15 +9,14 @@ set -ex export LD_LIBRARY_PATH=$PWD:$LD_LIBRARY_PATH export DYLD_LIBRARY_PATH=$PWD:$DYLD_LIBRARY_PATH -if [ ! -d ./icefall-asr-zipformer-streaming-wenetspeech-20230615 ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/pkufool/icefall-asr-zipformer-streaming-wenetspeech-20230615 - cd icefall-asr-zipformer-streaming-wenetspeech-20230615 - git lfs pull --include "*.onnx" - cd .. +if [ ! -d ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 fi dotnet run -c Release \ - --tokens ./icefall-asr-zipformer-streaming-wenetspeech-20230615/data/lang_char/tokens.txt \ - --encoder ./icefall-asr-zipformer-streaming-wenetspeech-20230615/exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx \ - --decoder ./icefall-asr-zipformer-streaming-wenetspeech-20230615/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx \ - --joiner ./icefall-asr-zipformer-streaming-wenetspeech-20230615/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx + --tokens ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt \ + --encoder ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.int8.onnx \ + --decoder ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.int8.onnx \ + --joiner ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.int8.onnx diff --git a/python-api-examples/non_streaming_server.py b/python-api-examples/non_streaming_server.py index 034061def..2b91813e2 100755 --- a/python-api-examples/non_streaming_server.py +++ b/python-api-examples/non_streaming_server.py @@ -23,10 +23,9 @@ (1) Use a non-streaming transducer model cd /path/to/sherpa-onnx -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-zipformer-en-2023-06-26 -cd sherpa-onnx-zipformer-en-2023-06-26 -git lfs pull --include "*.onnx" -cd .. +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 +tar xvf sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 +rm sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --encoder ./sherpa-onnx-zipformer-en-2023-06-26/encoder-epoch-99-avg-1.onnx \ @@ -37,22 +36,20 @@ (2) Use a non-streaming paraformer cd /path/to/sherpa-onnx -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en -cd sherpa-onnx-paraformer-bilingual-zh-en/ -git lfs pull --include "*.onnx" -cd .. +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 +tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 +rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ - --paraformer ./sherpa-onnx-paraformer-bilingual-zh-en/model.int8.onnx \ - --tokens ./sherpa-onnx-paraformer-bilingual-zh-en/tokens.txt + --paraformer ./sherpa-onnx-paraformer-zh-2023-03-28/model.int8.onnx \ + --tokens ./sherpa-onnx-paraformer-zh-2023-03-28/tokens.txt (3) Use a non-streaming CTC model from NeMo cd /path/to/sherpa-onnx -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-conformer-medium -cd sherpa-onnx-nemo-ctc-en-conformer-medium -git lfs pull --include "*.onnx" -cd .. +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 +tar xvf sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 +rm sherpa-onnx-nemo-ctc-en-conformer-medium.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --nemo-ctc ./sherpa-onnx-nemo-ctc-en-conformer-medium/model.onnx \ @@ -61,10 +58,9 @@ (4) Use a non-streaming CTC model from WeNet cd /path/to/sherpa-onnx -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-wenetspeech -cd sherpa-onnx-zh-wenet-wenetspeech -git lfs pull --include "*.onnx" -cd .. +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zh-wenet-wenetspeech.tar.bz2 +tar xvf sherpa-onnx-zh-wenet-wenetspeech.tar.bz2 +rm sherpa-onnx-zh-wenet-wenetspeech.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --wenet-ctc ./sherpa-onnx-zh-wenet-wenetspeech/model.onnx \ @@ -73,10 +69,9 @@ (5) Use a Whisper model cd /path/to/sherpa-onnx -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-whisper-tiny.en -cd sherpa-onnx-whisper-tiny.en -git lfs pull --include "*.onnx" -cd .. +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 +tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 +rm sherpa-onnx-whisper-tiny.en.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --whisper-encoder=./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.onnx \ @@ -87,9 +82,9 @@ cd /path/to/sherpa-onnx -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-tdnn-yesno -cd sherpa-onnx-tdnn-yesno -git lfs pull --include "*.onnx" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-tdnn-yesno.tar.bz2 +tar xvf sherpa-onnx-tdnn-yesno.tar.bz2 +rm sherpa-onnx-tdnn-yesno.tar.bz2 python3 ./python-api-examples/non_streaming_server.py \ --sample-rate=8000 \ @@ -383,6 +378,7 @@ def add_hotwords_args(parser: argparse.ArgumentParser): """, ) + def add_blank_penalty_args(parser: argparse.ArgumentParser): parser.add_argument( "--blank-penalty", diff --git a/python-api-examples/online-decode-files.py b/python-api-examples/online-decode-files.py index b9ab3b981..0188f0491 100755 --- a/python-api-examples/online-decode-files.py +++ b/python-api-examples/online-decode-files.py @@ -8,9 +8,9 @@ (1) Streaming transducer -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26 -cd sherpa-onnx-streaming-zipformer-en-2023-06-26 -git lfs pull --include "*.onnx" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 +tar xvf sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 +rm sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2 ./python-api-examples/online-decode-files.py \ --tokens=./sherpa-onnx-streaming-zipformer-en-2023-06-26/tokens.txt \ @@ -23,9 +23,9 @@ (2) Streaming paraformer -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en -cd sherpa-onnx-streaming-paraformer-bilingual-zh-en -git lfs pull --include "*.onnx" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 +tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 +rm sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 ./python-api-examples/online-decode-files.py \ --tokens=./sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt \ @@ -52,9 +52,9 @@ (4) Streaming Conformer CTC from WeNet -GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-zh-wenet-wenetspeech -cd sherpa-onnx-zh-wenet-wenetspeech -git lfs pull --include "*.onnx" +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zh-wenet-wenetspeech.tar.bz2 +tar xvf sherpa-onnx-zh-wenet-wenetspeech.tar.bz2 +rm sherpa-onnx-zh-wenet-wenetspeech.tar.bz2 ./python-api-examples/online-decode-files.py \ --tokens=./sherpa-onnx-zh-wenet-wenetspeech/tokens.txt \ diff --git a/scripts/wespeaker/run.sh b/scripts/wespeaker/run.sh index be7aa8401..5c2bf8133 100755 --- a/scripts/wespeaker/run.sh +++ b/scripts/wespeaker/run.sh @@ -4,6 +4,7 @@ set -ex echo "Downloading models" export GIT_LFS_SKIP_SMUDGE=1 +export GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/openspeech/wespeaker-models cd wespeaker-models git lfs pull --include "*.onnx"