From e472180f2cdc07a58fc6f3173f91532ecb4f91f5 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 20 Jul 2024 17:09:23 +0800 Subject: [PATCH] Add C# API for SenseVoice models (#1151) --- .github/scripts/test-dot-net.sh | 21 ++++++++-------- .../offline-decode-files/Program.cs | 11 +++++++++ .../run-sense-voice-ctc.sh | 14 +++++++++++ scripts/dart/release.sh | 6 ++--- scripts/dotnet/OfflineModelConfig.cs | 7 +++--- .../dotnet/OfflineSenseVoiceModelConfig.cs | 24 +++++++++++++++++++ scripts/dotnet/run.sh | 10 ++++---- scripts/go/release.sh | 14 +++++------ 8 files changed, 79 insertions(+), 28 deletions(-) create mode 100755 dotnet-examples/offline-decode-files/run-sense-voice-ctc.sh create mode 100644 scripts/dotnet/OfflineSenseVoiceModelConfig.cs diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index a4559f7b0..2b40669bd 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -2,16 +2,8 @@ cd dotnet-examples/ -cd ./keyword-spotting-from-files -./run.sh - -cd ../online-decode-files -./run-transducer-itn.sh -./run-zipformer2-ctc.sh -./run-transducer.sh -./run-paraformer.sh - -cd ../offline-decode-files +cd ./offline-decode-files +./run-sense-voice-ctc.sh ./run-paraformer-itn.sh ./run-telespeech-ctc.sh ./run-nemo-ctc.sh @@ -21,6 +13,15 @@ cd ../offline-decode-files ./run-whisper.sh ./run-tdnn-yesno.sh +cd ../keyword-spotting-from-files +./run.sh + +cd ../online-decode-files +./run-transducer-itn.sh +./run-zipformer2-ctc.sh +./run-transducer.sh +./run-paraformer.sh + cd ../vad-non-streaming-asr-paraformer ./run.sh diff --git a/dotnet-examples/offline-decode-files/Program.cs b/dotnet-examples/offline-decode-files/Program.cs index 400b18f00..d971becd3 100644 --- a/dotnet-examples/offline-decode-files/Program.cs +++ b/dotnet-examples/offline-decode-files/Program.cs @@ -61,6 +61,12 @@ class Options [Option("telespeech-ctc", Required = false, HelpText = "Path to model.onnx. Used only for TeleSpeech CTC models")] public string TeleSpeechCtc { get; set; } = ""; + [Option("sense-voice-model", Required = false, HelpText = "Path to model.onnx. Used only for SenseVoice CTC models")] + public string SenseVoiceModel { get; set; } = ""; + + [Option("sense-voice-use-itn", Required = false, HelpText = "1 to use inverse text normalization for sense voice.")] + public int SenseVoiceUseItn { get; set; } = 1; + [Option("num-threads", Required = false, Default = 1, HelpText = "Number of threads for computation")] public int NumThreads { get; set; } = 1; @@ -225,6 +231,11 @@ private static void Run(Options options) { config.ModelConfig.Tdnn.Model = options.TdnnModel; } + else if (!String.IsNullOrEmpty(options.SenseVoiceModel)) + { + config.ModelConfig.SenseVoice.Model = options.SenseVoiceModel; + config.ModelConfig.SenseVoice.UseInverseTextNormalization = options.SenseVoiceUseItn; + } else { Console.WriteLine("Please provide a model"); diff --git a/dotnet-examples/offline-decode-files/run-sense-voice-ctc.sh b/dotnet-examples/offline-decode-files/run-sense-voice-ctc.sh new file mode 100755 index 000000000..a75fd10f1 --- /dev/null +++ b/dotnet-examples/offline-decode-files/run-sense-voice-ctc.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 + tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 + rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 +fi + +dotnet run \ + --sense-voice-model=./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/model.int8.onnx \ + --tokens=./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/tokens.txt \ + --files ./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/test_wavs/zh.wav diff --git a/scripts/dart/release.sh b/scripts/dart/release.sh index e3f8208a4..bf6b0879a 100755 --- a/scripts/dart/release.sh +++ b/scripts/dart/release.sh @@ -36,7 +36,7 @@ windows_x64_wheel=$src_dir/$windows_x64_wheel_filename function process_linux() { mkdir -p t cd t - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$linux_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$linux_wheel_filename unzip $linux_wheel_filename cp -v sherpa_onnx/lib/*.so* ../linux cd .. @@ -50,7 +50,7 @@ function process_linux() { function process_windows_x64() { mkdir -p t cd t - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$windows_x64_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$windows_x64_wheel_filename unzip $windows_x64_wheel_filename cp -v sherpa_onnx-${SHERPA_ONNX_VERSION}.data/data/bin/*.dll ../windows cd .. @@ -60,7 +60,7 @@ function process_windows_x64() { function process_macos() { mkdir -p t cd t - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$macos_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$macos_wheel_filename unzip $macos_wheel_filename cp -v sherpa_onnx/lib/*.dylib ../macos cd .. diff --git a/scripts/dotnet/OfflineModelConfig.cs b/scripts/dotnet/OfflineModelConfig.cs index ddb489d48..b24aeaf89 100644 --- a/scripts/dotnet/OfflineModelConfig.cs +++ b/scripts/dotnet/OfflineModelConfig.cs @@ -23,6 +23,7 @@ public OfflineModelConfig() ModelingUnit = "cjkchar"; BpeVocab = ""; TeleSpeechCtc = ""; + SenseVoice = new OfflineSenseVoiceModelConfig(); } public OfflineTransducerModelConfig Transducer; public OfflineParaformerModelConfig Paraformer; @@ -51,7 +52,7 @@ public OfflineModelConfig() [MarshalAs(UnmanagedType.LPStr)] public string TeleSpeechCtc; - } - -} \ No newline at end of file + public OfflineSenseVoiceModelConfig SenseVoice; + } +} diff --git a/scripts/dotnet/OfflineSenseVoiceModelConfig.cs b/scripts/dotnet/OfflineSenseVoiceModelConfig.cs new file mode 100644 index 000000000..3ab0b637f --- /dev/null +++ b/scripts/dotnet/OfflineSenseVoiceModelConfig.cs @@ -0,0 +1,24 @@ +/// Copyright (c) 2024 Xiaomi Corporation (authors: Fangjun Kuang) + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + [StructLayout(LayoutKind.Sequential)] + public struct OfflineSenseVoiceModelConfig + { + public OfflineSenseVoiceModelConfig() + { + Model = ""; + Language = ""; + UseInverseTextNormalization = 0; + } + [MarshalAs(UnmanagedType.LPStr)] + public string Model; + + [MarshalAs(UnmanagedType.LPStr)] + public string Language; + + public int UseInverseTextNormalization; + } +} diff --git a/scripts/dotnet/run.sh b/scripts/dotnet/run.sh index 5b0000126..8fe448a90 100755 --- a/scripts/dotnet/run.sh +++ b/scripts/dotnet/run.sh @@ -55,7 +55,7 @@ if [ ! -f $src_dir/linux-x64/libsherpa-onnx-c-api.so ]; then if [ -f $linux_x64_wheel ]; then cp -v $linux_x64_wheel . else - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$linux_x64_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$linux_x64_wheel_filename fi unzip $linux_x64_wheel_filename cp -v sherpa_onnx/lib/*.so* ../ @@ -73,7 +73,7 @@ if [ ! -f $src_dir/linux-arm64/libsherpa-onnx-c-api.so ]; then if [ -f $linux_arm64_wheel ]; then cp -v $linux_arm64_wheel . else - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$linux_arm64_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$linux_arm64_wheel_filename fi unzip $linux_arm64_wheel_filename cp -v sherpa_onnx/lib/*.so* ../ @@ -91,7 +91,7 @@ if [ ! -f $src_dir/macos-x64/libsherpa-onnx-c-api.dylib ]; then if [ -f $macos_x64_wheel ]; then cp -v $macos_x64_wheel . else - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$macos_x64_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$macos_x64_wheel_filename fi unzip $macos_x64_wheel_filename cp -v sherpa_onnx/lib/*.dylib ../ @@ -111,7 +111,7 @@ if [ ! -f $src_dir/macos-arm64/libsherpa-onnx-c-api.dylib ]; then if [ -f $macos_arm64_wheel ]; then cp -v $macos_arm64_wheel . else - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$macos_arm64_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$macos_arm64_wheel_filename fi unzip $macos_arm64_wheel_filename cp -v sherpa_onnx/lib/*.dylib ../ @@ -131,7 +131,7 @@ if [ ! -f $src_dir/windows-x64/sherpa-onnx-c-api.dll ]; then if [ -f $windows_x64_wheel ]; then cp -v $windows_x64_wheel . else - curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/$windows_x64_wheel_filename + curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/$windows_x64_wheel_filename fi unzip $windows_x64_wheel_filename cp -v sherpa_onnx-${SHERPA_ONNX_VERSION}.data/data/bin/*.dll ../ diff --git a/scripts/go/release.sh b/scripts/go/release.sh index f09ede5f5..6d4b4c721 100755 --- a/scripts/go/release.sh +++ b/scripts/go/release.sh @@ -27,7 +27,7 @@ function linux() { dst=$(realpath sherpa-onnx-go-linux/lib/x86_64-unknown-linux-gnu) mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl cp -v sherpa_onnx/lib/*.so* $dst @@ -39,7 +39,7 @@ function linux() { dst=$(realpath sherpa-onnx-go-linux/lib/aarch64-unknown-linux-gnu) mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl cp -v sherpa_onnx/lib/*.so* $dst @@ -51,7 +51,7 @@ function linux() { dst=$(realpath sherpa-onnx-go-linux/lib/arm-unknown-linux-gnueabihf) mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-linux_armv7l.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-linux_armv7l.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-linux_armv7l.whl cp -v sherpa_onnx/lib/*.so* $dst @@ -84,7 +84,7 @@ function osx() { mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp39-cp39-macosx_11_0_x86_64.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp39-cp39-macosx_11_0_x86_64.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp39-cp39-macosx_11_0_x86_64.whl cp -v sherpa_onnx/lib/*.dylib $dst/ @@ -102,7 +102,7 @@ function osx() { mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp39-cp39-macosx_11_0_arm64.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp39-cp39-macosx_11_0_arm64.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp39-cp39-macosx_11_0_arm64.whl cp -v sherpa_onnx/lib/*.dylib $dst/ @@ -137,7 +137,7 @@ function windows() { dst=$(realpath sherpa-onnx-go-windows/lib/x86_64-pc-windows-gnu) mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win_amd64.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win_amd64.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win_amd64.whl cp -v sherpa_onnx-${SHERPA_ONNX_VERSION}.data/data/bin/*.dll $dst @@ -149,7 +149,7 @@ function windows() { dst=$(realpath sherpa-onnx-go-windows/lib/i686-pc-windows-gnu) mkdir t cd t - wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win32.whl + wget -q https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/cpu/$SHERPA_ONNX_VERSION/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win32.whl unzip ./sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win32.whl cp -v sherpa_onnx-${SHERPA_ONNX_VERSION}.data/data/bin/*.dll $dst