diff --git a/harmony-os/README.md b/harmony-os/README.md index 5b14bb7bb..566e1a74c 100644 --- a/harmony-os/README.md +++ b/harmony-os/README.md @@ -10,5 +10,8 @@ VAD + Non-streaming ASR for speech recognition. Please see the doc at +- [./SherpaOnnxStreamingAsr](./SherpaOnnxStreamingAsr) It shows how to use + streaming ASR models for real-time on-device speech recognition. + - [./SherpaOnnxTts](./SherpaOnnxTts) It shows how to run on-device text-to-speech. Please see the doc at diff --git a/harmony-os/SherpaOnnxStreamingAsr/.gitignore b/harmony-os/SherpaOnnxStreamingAsr/.gitignore new file mode 100644 index 000000000..d2ff20141 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/.gitignore @@ -0,0 +1,12 @@ +/node_modules +/oh_modules +/local.properties +/.idea +**/build +/.hvigor +.cxx +/.clangd +/.clang-format +/.clang-tidy +**/.test +/.appanalyzer \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 b/harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 new file mode 100644 index 000000000..88d26d63f --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/AppScope/app.json5 @@ -0,0 +1,10 @@ +{ + "app": { + "bundleName": "com.k2fsa.sherpa.onnx.streaming.asr", + "vendor": "example", + "versionCode": 1000000, + "versionName": "1.0.0", + "icon": "$media:app_icon", + "label": "$string:app_name" + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json new file mode 100644 index 000000000..1bd22c6e6 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string": [ + { + "name": "app_name", + "value": "SherpaOnnxStreamingAsr" + } + ] +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/media/app_icon.png b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/media/app_icon.png new file mode 100644 index 000000000..a39445dc8 Binary files /dev/null and b/harmony-os/SherpaOnnxStreamingAsr/AppScope/resources/base/media/app_icon.png differ diff --git a/harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 b/harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 new file mode 100644 index 000000000..8e63d9768 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/build-profile.json5 @@ -0,0 +1,40 @@ +{ + "app": { + "signingConfigs": [], + "products": [ + { + "name": "default", + "signingConfig": "default", + "compatibleSdkVersion": "4.0.0(10)", + "runtimeOS": "HarmonyOS", + "buildOption": { + "strictMode": { + "caseSensitiveCheck": true, + } + } + } + ], + "buildModeSet": [ + { + "name": "debug", + }, + { + "name": "release" + } + ] + }, + "modules": [ + { + "name": "entry", + "srcPath": "./entry", + "targets": [ + { + "name": "default", + "applyToProducts": [ + "default" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 b/harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 new file mode 100644 index 000000000..77b31b517 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/code-linter.json5 @@ -0,0 +1,20 @@ +{ + "files": [ + "**/*.ets" + ], + "ignore": [ + "**/src/ohosTest/**/*", + "**/src/test/**/*", + "**/src/mock/**/*", + "**/node_modules/**/*", + "**/oh_modules/**/*", + "**/build/**/*", + "**/.preview/**/*" + ], + "ruleSet": [ + "plugin:@performance/recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore b/harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore new file mode 100644 index 000000000..e2713a277 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/.gitignore @@ -0,0 +1,6 @@ +/node_modules +/oh_modules +/.preview +/build +/.cxx +/.test \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 new file mode 100644 index 000000000..4e4aa2bc1 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/build-profile.json5 @@ -0,0 +1,33 @@ +{ + "apiType": "stageMode", + "buildOption": { + "sourceOption": { + "workers": [ + './src/main/ets/workers/StreamingAsrWorker.ets' + ] + } + }, + "buildOptionSet": [ + { + "name": "release", + "arkOptions": { + "obfuscation": { + "ruleOptions": { + "enable": false, + "files": [ + "./obfuscation-rules.txt" + ] + } + } + } + }, + ], + "targets": [ + { + "name": "default" + }, + { + "name": "ohosTest", + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts b/harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts new file mode 100644 index 000000000..c6edcd904 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/hvigorfile.ts @@ -0,0 +1,6 @@ +import { hapTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: hapTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt b/harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt new file mode 100644 index 000000000..272efb6ca --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/obfuscation-rules.txt @@ -0,0 +1,23 @@ +# Define project specific obfuscation rules here. +# You can include the obfuscation configuration files in the current module's build-profile.json5. +# +# For more details, see +# https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5 + +# Obfuscation options: +# -disable-obfuscation: disable all obfuscations +# -enable-property-obfuscation: obfuscate the property names +# -enable-toplevel-obfuscation: obfuscate the names in the global scope +# -compact: remove unnecessary blank spaces and all line feeds +# -remove-log: remove all console.* statements +# -print-namecache: print the name cache that contains the mapping from the old names to new names +# -apply-namecache: reuse the given cache file + +# Keep options: +# -keep-property-name: specifies property names that you want to keep +# -keep-global-name: specifies names that you want to keep in the global scope + +-enable-property-obfuscation +-enable-toplevel-obfuscation +-enable-filename-obfuscation +-enable-export-obfuscation \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 new file mode 100644 index 000000000..ce848c049 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package-lock.json5 @@ -0,0 +1,29 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "sherpa_onnx@1.10.33": "sherpa_onnx@1.10.33" + }, + "packages": { + "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": { + "name": "libsherpa_onnx.so", + "version": "1.0.0", + "resolved": "../oh_modules/.ohpm/sherpa_onnx@1.10.33/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx", + "registryType": "local" + }, + "sherpa_onnx@1.10.33": { + "name": "sherpa_onnx", + "version": "1.10.33", + "integrity": "sha512-cmZ8zwOMx4qmDvOjF1/PL6/suBgReanSf5XdQTuMWWZ6qN74rynODHrt4C+Qz754MTXg0q/phAKeVjGA4rHHSA==", + "resolved": "https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.33.har", + "registryType": "ohpm", + "dependencies": { + "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx" + } + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 new file mode 100644 index 000000000..e57a8e40e --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/oh-package.json5 @@ -0,0 +1,12 @@ +{ + "name": "entry", + "version": "1.0.0", + "description": "Please describe the basic information.", + "main": "", + "author": "", + "license": "", + "dependencies": { + "sherpa_onnx": "1.10.33", + } +} + diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets new file mode 100644 index 000000000..679d91453 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entryability/EntryAbility.ets @@ -0,0 +1,43 @@ +import AbilityConstant from '@ohos.app.ability.AbilityConstant'; +import hilog from '@ohos.hilog'; +import UIAbility from '@ohos.app.ability.UIAbility'; +import Want from '@ohos.app.ability.Want'; +import window from '@ohos.window'; + +export default class EntryAbility extends UIAbility { + onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate'); + } + + onDestroy(): void { + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy'); + } + + onWindowStageCreate(windowStage: window.WindowStage): void { + // Main window is created, set main page for this ability + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate'); + + windowStage.loadContent('pages/Index', (err) => { + if (err.code) { + hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? ''); + return; + } + hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.'); + }); + } + + onWindowStageDestroy(): void { + // Main window is destroyed, release UI related resources + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy'); + } + + onForeground(): void { + // Ability has brought to foreground + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground'); + } + + onBackground(): void { + // Ability has back to background + hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground'); + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets new file mode 100644 index 000000000..d2c48b421 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets @@ -0,0 +1,12 @@ +import hilog from '@ohos.hilog'; +import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility'; + +export default class EntryBackupAbility extends BackupExtensionAbility { + async onBackup() { + hilog.info(0x0000, 'testTag', 'onBackup ok'); + } + + async onRestore(bundleVersion: BundleVersion) { + hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion)); + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets new file mode 100644 index 000000000..3a08b9213 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Index.ets @@ -0,0 +1,428 @@ +import { LengthUnit } from '@kit.ArkUI'; +import worker, { MessageEvents } from '@ohos.worker'; +import { BusinessError } from '@kit.BasicServicesKit'; +import { picker } from '@kit.CoreFileKit'; +import systemTime from '@ohos.systemTime'; +import { Permissions } from '@kit.AbilityKit'; +import { allAllowed, requestPermissions } from './Permission'; +import { audio } from '@kit.AudioKit'; +import fs from '@ohos.file.fs'; + + +function savePcmToWav(filename: string, samples: Int16Array, sampleRate: number) { + const fp = fs.openSync(filename, fs.OpenMode.READ_WRITE | fs.OpenMode.CREATE); + + const header = new ArrayBuffer(44); + const view = new DataView(header); + + // http://soundfile.sapp.org/doc/WaveFormat/ + // F F I R + view.setUint32(0, 0x46464952, true); // chunkID + view.setUint32(4, 36 + samples.length * 2, true); // chunkSize // E V A W + view.setUint32(8, 0x45564157, true); // format // // t m f + view.setUint32(12, 0x20746d66, true); // subchunk1ID + view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM + view.setUint32(20, 1, true); // audioFormat, 1 for PCM + view.setUint16(22, 1, true); // numChannels: 1 channel + view.setUint32(24, sampleRate, true); // sampleRate + view.setUint32(28, sampleRate * 2, true); // byteRate + view.setUint16(32, 2, true); // blockAlign + view.setUint16(34, 16, true); // bitsPerSample + view.setUint32(36, 0x61746164, true); // Subchunk2ID + view.setUint32(40, samples.length * 2, true); // subchunk2Size + + fs.writeSync(fp.fd, new Uint8Array(header).buffer, { length: header.byteLength }); + fs.writeSync(fp.fd, samples.buffer, { length: samples.buffer.byteLength }); + + fs.closeSync(fp.fd); +} + +function toInt16Samples(samples: Float32Array): Int16Array { + const int16Samples = new Int16Array(samples.length); + for (let i = 0; i < samples.length; ++i) { + let s = samples[i] * 32767; + s = s > 32767 ? 32767 : s; + s = s < -32768 ? -32768 : s; + int16Samples[i] = s; + } + + return int16Samples; +} + + +@Entry +@Component +struct Index { + @State title: string = 'Next-gen Kaldi: Real-time speech recognition'; + @State titleFontSize: number = 15; + @State currentIndex: number = 0; + @State lang: string = 'English'; + @State resultForFile: string = '' + @State resultForMic: string = '' + @State selectFileBtnEnabled: boolean = false; + @State micBtnCaption: string = 'Start'; + @State micStarted: boolean = false; + @State micAllowed: boolean = false; + @State micBtnEnabled: boolean = false; + @State micSaveBtnCaption: string = 'Save recorded audio'; + @State micSaveBtnEnabled: boolean = false; + @State info: string = ''; + @State micInfo: string = ''; + @State micInitDone: boolean = false; + private resultListForMic: string[] = []; + private controller: TabsController = new TabsController(); + private workerInstance?: worker.ThreadWorker + private readonly scriptURL: string = 'entry/ets/workers/StreamingAsrWorker.ets' + private startTime: number = 0; + private stopTime: number = 0; + private sampleRate: number = 48000; + private sampleList: Float32Array[] = [] + private mic?: audio.AudioCapturer; + + flatten(samples: Float32Array[]): Float32Array { + let n = 0; + for (let i = 0; i < samples.length; ++i) { + n += samples[i].length; + } + + const ans: Float32Array = new Float32Array(n); + let offset: number = 0; + for (let i = 0; i < samples.length; ++i) { + ans.set(samples[i], offset); + offset += samples[i].length; + } + + return ans; + } + + async initMic() { + const permissions: Permissions[] = ["ohos.permission.MICROPHONE"]; + let allowed: boolean = await allAllowed(permissions); + if (!allowed) { + console.log("request to access the microphone"); + const status: boolean = await requestPermissions(permissions); + + if (!status) { + console.error('access to microphone is denied') + this.resultForMic = "Failed to get microphone permission. Please retry"; + return; + } + + allowed = await allAllowed(permissions); + if (!allowed) { + console.error('failed to get microphone permission'); + this.resultForMic = "Failed to get microphone permission. Please retry"; + return; + } + this.micAllowed = true; + } else { + console.log("allowed to access microphone"); + this.micAllowed = true; + } + + const audioStreamInfo: audio.AudioStreamInfo = { + samplingRate: this.sampleRate, + channels: audio.AudioChannel.CHANNEL_1, + sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE, + encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW, + }; + + const audioCapturerInfo: audio.AudioCapturerInfo = { + source: audio.SourceType.SOURCE_TYPE_MIC, capturerFlags: 0 + }; + + const audioCapturerOptions: audio.AudioCapturerOptions = { + streamInfo: audioStreamInfo, capturerInfo: audioCapturerInfo + + }; + audio.createAudioCapturer(audioCapturerOptions, (err, data) => { + if (err) { + console.error(`error code is ${err.code}, error message is ${err.message}`); + this.resultForMic = 'Failed to init microphone'; + } else { + console.info(`init mic successfully`); + this.mic = data; + this.mic.on('readData', this.micCallback); + } + }); + } + + async aboutToAppear() { + this.workerInstance = new worker.ThreadWorker(this.scriptURL, { + name: 'Streaming ASR worker' + }); + + this.workerInstance.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + console.log(`received msg from worker: ${msgType}`); + + if (msgType == 'init-streaming-asr-done') { + this.selectFileBtnEnabled = true; + this.micBtnEnabled = true; + this.info = `Initializing done.\n\nPlease select a wave file of 16kHz in language ${this.lang}`; + this.micInfo = `Initializing done.\n\nPlease click Start and speak`; + } + + if (msgType == 'streaming-asr-decode-file-done') { + const text = e.data['text'] as string; + this.resultForFile = text; + this.selectFileBtnEnabled = true; + + systemTime.getRealTime((err, data) => { + if (err) { + console.log('Failed to get stop time'); + } else { + this.stopTime = data; + + const audioDuration = e.data['duration'] as number; + const elapsedSeconds = (this.stopTime - this.startTime) / 1000; + const RTF = elapsedSeconds / audioDuration; + this.info = `Audio duration: ${audioDuration.toFixed(2)} s +Elapsed: ${elapsedSeconds.toFixed(2)} s +RTF = ${elapsedSeconds.toFixed(2)}/${audioDuration.toFixed(2)} = ${RTF.toFixed(3)} +`; + } + }); + } + + if (msgType == 'streaming-asr-decode-mic-result') { + const text = e.data['text'] as string; + if (text.trim() == '') { + return; + } + + const isEndpoint = e.data['isEndpoint'] as boolean; + + let s = ''; + let i = 0; + for (; i < this.resultListForMic.length; ++i) { + s += `${i}: ${this.resultListForMic[i]}\n` + } + + s += `${i}: ${text}`; + this.resultForMic = s; + + if (isEndpoint) { + this.resultListForMic.push(text); + } + } + }; + + const context = getContext(); + this.workerInstance.postMessage({ msgType: 'init-streaming-asr', context }); + this.info = 'Initializing ASR model.\nPlease wait'; + this.micInfo = 'Initializing ASR model.\nPlease wait'; + + await this.initMic(); + } + + @Builder + TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) { + Column() { + Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 }) + Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a') + }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => { + this.currentIndex = targetIndex; + this.controller.changeIndex(this.currentIndex); + }) + } + + build() { + Column() { + Tabs({ barPosition: BarPosition.End, controller: this.controller }) { + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + Button('Select .wav file (16kHz) ') + .enabled(this.selectFileBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + this.resultForFile = ''; + this.info = ''; + this.selectFileBtnEnabled = false; + + const documentSelectOptions = new picker.DocumentSelectOptions(); + documentSelectOptions.maxSelectNumber = 1; + documentSelectOptions.fileSuffixFilters = ['.wav']; + const documentViewPicker = new picker.DocumentViewPicker(); + + documentViewPicker.select(documentSelectOptions).then((result: Array) => { + console.log(`select file result: ${result}`); + + if (!result[0]) { + this.resultForFile = 'Please select a file to decode'; + this.selectFileBtnEnabled = true; + return; + } + + if (this.workerInstance) { + systemTime.getRealTime((err, data) => { + if (err) { + console.log('Failed to get start time'); + } else { + this.startTime = data; + } + }); + + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-file', filename: result[0], + }); + this.info = `Decoding ${result[0]} ... ...`; + } else { + console.log(`this worker instance is undefined ${this.workerInstance}`); + } + + }).catch((err: BusinessError) => { + console.error(`Failed to select file, code is ${err.code}, message is ${err.message}`); + this.selectFileBtnEnabled = true; + }) + }) + + Text(`Supported languages: ${this.lang}`); + if (this.info != '') { + TextArea({ text: this.info }).focusable(false); + } + TextArea({ text: this.resultForFile }) + .width('100%') + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .height('100%'); + } + }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc'))) + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + Button(this.micBtnCaption) + .enabled(this.micBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + this.micInfo = ''; + if (this.mic) { + if (this.micStarted) { + this.micStarted = false; + this.micBtnCaption = 'Start'; + this.mic.stop(); + this.micSaveBtnEnabled = true; + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-mic-stop' + }); + } + } else { + this.micStarted = true; + this.micSaveBtnEnabled = false; + this.micBtnCaption = 'Stop'; + this.resultForMic = ''; + this.resultListForMic = []; + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-mic-start' + }); + } + + this.sampleList = []; + this.mic.start(); + } + } + }); + Button(this.micSaveBtnCaption) + .enabled(this.micSaveBtnEnabled) + .fontSize(13) + .width(296) + .height(60) + .onClick(() => { + if (this.sampleList.length == 0) { + this.micSaveBtnEnabled = false; + return; + } + + const samples = this.flatten(this.sampleList); + + if (samples.length == 0) { + this.micSaveBtnEnabled = false; + return; + } + + + let uri: string = ''; + + + const audioOptions = new picker.AudioSaveOptions(); // audioOptions.newFileNames = ['o.wav']; + + const audioViewPicker = new picker.AudioViewPicker(); + + audioViewPicker.save(audioOptions).then((audioSelectResult: Array) => { + uri = audioSelectResult[0]; + savePcmToWav(uri, toInt16Samples(samples), this.sampleRate); + console.log(`Saved to ${uri}`); + this.micInfo += `\nSaved to ${uri}`; + }); + + }) + + + Text(`Supported languages: ${this.lang}`) + + if (this.micInfo != '') { + TextArea({ text: this.micInfo }) + .focusable(false); + } + + TextArea({ text: this.resultForMic }) + .width('100%') + .lineSpacing({ value: 10, unit: LengthUnit.VP }) + .width('100%') + .height('100%'); + } + }.tabBar(this.TabBuilder('From mic', 1, $r('app.media.icon_mic'), $r('app.media.icon_mic'))) + + + TabContent() { + Column({ space: 10 }) { + Text(this.title).fontSize(this.titleFontSize).fontWeight(FontWeight.Bold); + TextArea({ + text: ` +Everyting is open-sourced. + +It runs locally, without accessing the network + +See also https://github.com/k2-fsa/sherpa-onnx + +新一代 Kaldi QQ 和微信交流群: 请看 + +https://k2-fsa.github.io/sherpa/social-groups.html + +微信公众号: 新一代 Kaldi + ` + }).width('100%').height('100%').focusable(false) + }.justifyContent(FlexAlign.Start) + }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info'), $r('app.media.info'))) + }.scrollable(false) + }.width('100%') + } + + private micCallback = (buffer: ArrayBuffer) => { + const view: Int16Array = new Int16Array(buffer); + + const samplesFloat: Float32Array = new Float32Array(view.length); + for (let i = 0; i < view.length; ++i) { + samplesFloat[i] = view[i] / 32768.0; + } + + this.sampleList.push(samplesFloat); + + if (this.workerInstance) { + this.workerInstance.postMessage({ + msgType: 'streaming-asr-decode-mic-samples', + samples: samplesFloat, + sampleRate: this.sampleRate, + }) + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets new file mode 100644 index 000000000..40ef391ad --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/pages/Permission.ets @@ -0,0 +1,26 @@ +// This file is modified from +// https://gitee.com/ukSir/hmchat2/blob/master/entry/src/main/ets/utils/permissionMananger.ets +import { abilityAccessCtrl, bundleManager, common, Permissions } from '@kit.AbilityKit'; + +export function allAllowed(permissions: Permissions[]): boolean { + if (permissions.length == 0) { + return false; + } + + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + + const bundleInfo = bundleManager.getBundleInfoForSelfSync(bundleManager.BundleFlag.GET_BUNDLE_INFO_WITH_APPLICATION); + + let tokenID: number = bundleInfo.appInfo.accessTokenId; + + return permissions.every(permission => abilityAccessCtrl.GrantStatus.PERMISSION_GRANTED == + mgr.checkAccessTokenSync(tokenID, permission)); +} + +export async function requestPermissions(permissions: Permissions[]): Promise { + const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager(); + const context: Context = getContext() as common.UIAbilityContext; + + const result = await mgr.requestPermissionsFromUser(context, permissions); + return result.authResults.length > 0 && result.authResults.every(authResults => authResults == 0); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets new file mode 100644 index 000000000..d521cc3e4 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/ets/workers/StreamingAsrWorker.ets @@ -0,0 +1,294 @@ +import worker, { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope } from '@ohos.worker'; +import { + OnlineModelConfig, + OnlineRecognizer, + OnlineRecognizerConfig, + OnlineStream, + readWaveFromBinary, + Samples +} from 'sherpa_onnx'; +import { fileIo } from '@kit.CoreFileKit'; + +const workerPort: ThreadWorkerGlobalScope = worker.workerPort; + + +let recognizer: OnlineRecognizer; +let micStream: OnlineStream; + +function getModelConfig(type: number): OnlineModelConfig { + const modelConfig = new OnlineModelConfig(); + switch (type) { + case 0: { + const modelDir = 'sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 1: { + const modelDir = 'sherpa-onnx-lstm-zh-2023-02-20'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-11-avg-1.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-11-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-11-avg-1.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'lstm'; + break; + } + + case 2: { + const modelDir = 'sherpa-onnx-lstm-en-2023-02-17'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'lstm'; + break; + } + + case 3: { + const modelDir = 'icefall-asr-zipformer-streaming-wenetspeech-20230615'; + modelConfig.transducer.encoder = `${modelDir}/exp/encoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.transducer.joiner = `${modelDir}/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.tokens = `${modelDir}/data/lang_char/tokens.txt`; + modelConfig.modelType = 'zipformer2'; + break; + } + + case 4: { + const modelDir = 'icefall-asr-zipformer-streaming-wenetspeech-20230615'; + modelConfig.transducer.encoder = `${modelDir}/exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.transducer.decoder = `${modelDir}/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.transducer.joiner = `${modelDir}/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx`; + modelConfig.tokens = `${modelDir}/data/lang_char/tokens.txt`; + modelConfig.modelType = 'zipformer2'; + break; + } + + case 5: { + const modelDir = 'sherpa-onnx-streaming-paraformer-bilingual-zh-en'; + modelConfig.paraformer.encoder = `${modelDir}/encoder.int8.onnx`; + modelConfig.paraformer.decoder = `${modelDir}/decoder.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'paraformer'; + break; + } + + case 6: { + const modelDir = 'sherpa-onnx-streaming-zipformer-en-2023-06-26'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1-chunk-16-left-128.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1-chunk-16-left-128.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer2'; + break; + } + + case 7: { + const modelDir = 'sherpa-onnx-streaming-zipformer-fr-2023-04-14'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-29-avg-9-with-averaged-model.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-29-avg-9-with-averaged-model.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-29-avg-9-with-averaged-model.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 8: { + const modelDir = 'sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 9: { + const modelDir = 'sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23' + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 10: { + const modelDir = 'sherpa-onnx-streaming-zipformer-en-20M-2023-02-17'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + + case 14: { + const modelDir = 'sherpa-onnx-streaming-zipformer-korean-2024-06-16'; + modelConfig.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`; + modelConfig.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`; + modelConfig.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`; + modelConfig.tokens = `${modelDir}/tokens.txt`; + modelConfig.modelType = 'zipformer'; + break; + } + default: { + console.log(`Please specify a supported type. Given type ${type}`); + } + } + return modelConfig; +} + +function initStreamingAsr(context: Context): OnlineRecognizer { + let type: number; + + /* + +If you use type = 8, then you should have the following directory structure in the rawfile directory + +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ pwd +/Users/fangjun/open-source/sherpa-onnx/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ ls +sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 +(py38) fangjuns-MacBook-Pro:rawfile fangjun$ tree . +. +└── sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + ├── decoder-epoch-99-avg-1.onnx + ├── encoder-epoch-99-avg-1.int8.onnx + ├── joiner-epoch-99-avg-1.int8.onnx + └── tokens.txt + +1 directory, 4 files + +You can download model files from +https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models + +Note that please delete files that are not used. Otherwise, you APP will be very large +due to containing unused large files. + + */ + type = 8; + + const config: OnlineRecognizerConfig = new OnlineRecognizerConfig(); + config.modelConfig = getModelConfig(type); + config.modelConfig.debug = true; + config.modelConfig.numThreads = 2; + config.enableEndpoint = true; + + return new OnlineRecognizer(config, context.resourceManager); +} + +interface DecodeFileResult { + text: string; + duration: number; +} + +function decodeFile(filename: string): DecodeFileResult { + const fp = fileIo.openSync(filename); + const stat = fileIo.statSync(fp.fd); + const arrayBuffer = new ArrayBuffer(stat.size); + fileIo.readSync(fp.fd, arrayBuffer); + const data: Uint8Array = new Uint8Array(arrayBuffer); + const wave: Samples = readWaveFromBinary(data) as Samples; + console.log(`Sample rate: ${wave.sampleRate}`); + + const stream = recognizer.createStream(); + stream.acceptWaveform(wave); + const tailPadding = new Float32Array(0.5 * wave.sampleRate); + tailPadding.fill(0); + + stream.acceptWaveform({ samples: tailPadding, sampleRate: wave.sampleRate }); + + while (recognizer.isReady(stream)) { + recognizer.decode(stream); + } + + const audioDuration = wave.samples.length / wave.sampleRate; + + return { text: recognizer.getResult(stream).text, duration: audioDuration }; +} + +/** + * Defines the event handler to be called when the worker thread receives a message sent by the host thread. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessage = (e: MessageEvents) => { + const msgType = e.data['msgType'] as string; + + if (msgType != 'streaming-asr-decode-mic-samples') { + console.log(`from the main thread, msg-type: ${msgType}`); + } + + if (msgType == 'init-streaming-asr' && !recognizer) { + console.log('initializing streaming ASR...'); + const context = e.data['context'] as Context; + recognizer = initStreamingAsr(context); + console.log('streaming ASR is initialized. '); + workerPort.postMessage({ 'msgType': 'init-streaming-asr-done' }); + } + + if (msgType == 'streaming-asr-decode-file') { + const filename = e.data['filename'] as string; + console.log(`decoding ${filename}`); + const result = decodeFile(filename); + workerPort.postMessage({ + 'msgType': 'streaming-asr-decode-file-done', text: result.text, duration: result.duration + }); + } + + if (msgType == 'streaming-asr-decode-mic-start') { + micStream = recognizer.createStream(); + } + + if (msgType == 'streaming-asr-decode-mic-stop') { // nothing to do + } + + if (msgType == 'streaming-asr-decode-mic-samples') { + const samples = e.data['samples'] as Float32Array; + const sampleRate = e.data['sampleRate'] as number; + + micStream.acceptWaveform({ samples, sampleRate }); + while (recognizer.isReady(micStream)) { + recognizer.decode(micStream); + + let isEndpoint = false; + let text = recognizer.getResult(micStream).text; + + if (recognizer.isEndpoint(micStream)) { + isEndpoint = true; + recognizer.reset(micStream); + } + + if (text.trim() != '') { + workerPort.postMessage({ + 'msgType': 'streaming-asr-decode-mic-result', text: text, isEndpoint: isEndpoint, + }); + } + } + } + +} + +/** + * Defines the event handler to be called when the worker receives a message that cannot be deserialized. + * The event handler is executed in the worker thread. + * + * @param e message data + */ +workerPort.onmessageerror = (e: MessageEvents) => { +} + +/** + * Defines the event handler to be called when an exception occurs during worker execution. + * The event handler is executed in the worker thread. + * + * @param e error message + */ +workerPort.onerror = (e: ErrorEvent) => { +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 new file mode 100644 index 000000000..80e93ca6a --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/module.json5 @@ -0,0 +1,64 @@ +{ + "module": { + "name": "entry", + "type": "entry", + "description": "$string:module_desc", + "mainElement": "EntryAbility", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false, + "pages": "$profile:main_pages", + "abilities": [ + { + "name": "EntryAbility", + "srcEntry": "./ets/entryability/EntryAbility.ets", + "description": "$string:EntryAbility_desc", + "icon": "$media:layered_image", + "label": "$string:EntryAbility_label", + "startWindowIcon": "$media:startIcon", + "startWindowBackground": "$color:start_window_background", + "exported": true, + "skills": [ + { + "entities": [ + "entity.system.home" + ], + "actions": [ + "action.system.home" + ] + } + ] + } + ], + "extensionAbilities": [ + { + "name": "EntryBackupAbility", + "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets", + "type": "backup", + "exported": false, + "metadata": [ + { + "name": "ohos.extension.backup", + "resource": "$profile:backup_config" + } + ], + } + ], + "requestPermissions": [ + { + "name": "ohos.permission.MICROPHONE", + "reason": "$string:mic_reason", + "usedScene": { + "abilities": [ + "EntryAbility", + ], + "when": "inuse", + } + } + ] + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json new file mode 100644 index 000000000..3c712962d --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/color.json @@ -0,0 +1,8 @@ +{ + "color": [ + { + "name": "start_window_background", + "value": "#FFFFFF" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json new file mode 100644 index 000000000..8679134da --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Real-time ASR" + }, + { + "name": "mic_reason", + "value": "access the microhone for on-device real-time speech recognition with Next-gen Kaldi" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/background.png b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/background.png new file mode 100644 index 000000000..f939c9fa8 Binary files /dev/null and b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/background.png differ diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/foreground.png b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/foreground.png new file mode 100644 index 000000000..4483ddad1 Binary files /dev/null and b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/foreground.png differ diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/home.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/home.svg new file mode 100644 index 000000000..504af3400 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/home.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg new file mode 100644 index 000000000..ab6b1fd76 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_doc.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg new file mode 100644 index 000000000..0aeb30d63 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/icon_mic.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json new file mode 100644 index 000000000..fb4992044 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/layered_image.json @@ -0,0 +1,7 @@ +{ + "layered-image": + { + "background" : "$media:background", + "foreground" : "$media:foreground" + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/startIcon.png b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/startIcon.png new file mode 100644 index 000000000..205ad8b5a Binary files /dev/null and b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/media/startIcon.png differ diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json new file mode 100644 index 000000000..78f40ae7c --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/backup_config.json @@ -0,0 +1,3 @@ +{ + "allowToBackupRestore": true +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 000000000..1898d94f5 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "pages/Index" + ] +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json new file mode 100644 index 000000000..8679134da --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/en_US/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_desc", + "value": "On-device real-time speech recognition with Next-gen Kaldi" + }, + { + "name": "EntryAbility_label", + "value": "Real-time ASR" + }, + { + "name": "mic_reason", + "value": "access the microhone for on-device real-time speech recognition with Next-gen Kaldi" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile/.gitkeep b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/rawfile/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json new file mode 100644 index 000000000..e4ff4f4e3 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/main/resources/zh_CN/element/string.json @@ -0,0 +1,20 @@ +{ + "string": [ + { + "name": "module_desc", + "value": "新一代Kaldi: 本地实时语音识别" + }, + { + "name": "EntryAbility_desc", + "value": "新一代Kaldi: 本地实时语音识别" + }, + { + "name": "EntryAbility_label", + "value": "实时语音识别" + }, + { + "name": "mic_reason", + "value": "使用新一代Kaldi, 访问麦克风进行本地实时语音识别 (不需要联网)" + } + ] +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets new file mode 100644 index 000000000..8aa374977 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/Ability.test.ets @@ -0,0 +1,35 @@ +import hilog from '@ohos.hilog'; +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function abilityTest() { + describe('ActsAbilityTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }) + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }) + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }) + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }) + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + hilog.info(0x0000, 'testTag', '%{public}s', 'it begin'); + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }) + }) +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets new file mode 100644 index 000000000..794c7dc4e --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/ets/test/List.test.ets @@ -0,0 +1,5 @@ +import abilityTest from './Ability.test'; + +export default function testsuite() { + abilityTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 new file mode 100644 index 000000000..55725a929 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/ohosTest/module.json5 @@ -0,0 +1,13 @@ +{ + "module": { + "name": "entry_test", + "type": "feature", + "deviceTypes": [ + "phone", + "tablet", + "2in1" + ], + "deliveryWithInstall": true, + "installationFree": false + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets new file mode 100644 index 000000000..bb5b5c373 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/List.test.ets @@ -0,0 +1,5 @@ +import localUnitTest from './LocalUnit.test'; + +export default function testsuite() { + localUnitTest(); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets new file mode 100644 index 000000000..165fc1615 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/entry/src/test/LocalUnit.test.ets @@ -0,0 +1,33 @@ +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium'; + +export default function localUnitTest() { + describe('localUnitTest', () => { + // Defines a test suite. Two parameters are supported: test suite name and test suite function. + beforeAll(() => { + // Presets an action, which is performed only once before all test cases of the test suite start. + // This API supports only one parameter: preset action function. + }); + beforeEach(() => { + // Presets an action, which is performed before each unit test case starts. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: preset action function. + }); + afterEach(() => { + // Presets a clear action, which is performed after each unit test case ends. + // The number of execution times is the same as the number of test cases defined by **it**. + // This API supports only one parameter: clear action function. + }); + afterAll(() => { + // Presets a clear action, which is performed after all test cases of the test suite end. + // This API supports only one parameter: clear action function. + }); + it('assertContain', 0, () => { + // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function. + let a = 'abc'; + let b = 'b'; + // Defines a variety of assertion methods, which are used to declare expected boolean conditions. + expect(a).assertContain(b); + expect(a).assertEqual(a); + }); + }); +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 b/harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 new file mode 100644 index 000000000..06b278367 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/hvigor/hvigor-config.json5 @@ -0,0 +1,22 @@ +{ + "modelVersion": "5.0.0", + "dependencies": { + }, + "execution": { + // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */ + // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ + // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ + // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ + // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ + }, + "logging": { + // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ + }, + "debugging": { + // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ + }, + "nodeOptions": { + // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ + // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ + } +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts b/harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts new file mode 100644 index 000000000..f3cb9f1a8 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/hvigorfile.ts @@ -0,0 +1,6 @@ +import { appTasks } from '@ohos/hvigor-ohos-plugin'; + +export default { + system: appTasks, /* Built-in plugin of Hvigor. It cannot be modified. */ + plugins:[] /* Custom plugin to extend the functionality of Hvigor. */ +} diff --git a/harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 b/harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 new file mode 100644 index 000000000..f538ae290 --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/oh-package-lock.json5 @@ -0,0 +1,19 @@ +{ + "meta": { + "stableOrder": true + }, + "lockfileVersion": 3, + "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", + "specifiers": { + "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19" + }, + "packages": { + "@ohos/hypium@1.0.19": { + "name": "@ohos/hypium", + "version": "1.0.19", + "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==", + "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har", + "registryType": "ohpm" + } + } +} \ No newline at end of file diff --git a/harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 b/harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 new file mode 100644 index 000000000..a79d5300e --- /dev/null +++ b/harmony-os/SherpaOnnxStreamingAsr/oh-package.json5 @@ -0,0 +1,9 @@ +{ + "modelVersion": "5.0.0", + "description": "Please describe the basic information.", + "dependencies": { + }, + "devDependencies": { + "@ohos/hypium": "1.0.19" + } +} diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets index 2675c7b77..5bd4b092f 100644 --- a/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets @@ -229,7 +229,7 @@ struct Index { .lineSpacing({ value: 10, unit: LengthUnit.VP }) .height('100%'); }.alignItems(HorizontalAlign.Center).justifyContent(FlexAlign.Start) - }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc_default'))) + }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc'))) TabContent() { Column({ space: 10 }) { @@ -278,8 +278,8 @@ struct Index { .height('100%'); }.alignItems(HorizontalAlign.Center).justifyContent(FlexAlign.Start) } - .tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'), - $r('app.media.ic_public_input_voice_default'))) + .tabBar(this.TabBuilder('From mic', 1, $r('app.media.icon_mic'), + $r('app.media.icon_mic'))) TabContent() { Column({ space: 10 }) { @@ -300,7 +300,7 @@ https://k2-fsa.github.io/sherpa/social-groups.html ` }).width('100%').height('100%').focusable(false) }.justifyContent(FlexAlign.Start) - }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info_circle'), $r('app.media.info_circle_default'))) + }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info'), $r('app.media.info'))) }.scrollable(false) }.width('100%').justifyContent(FlexAlign.Start) diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png deleted file mode 100644 index 37fa8591f..000000000 Binary files a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice.png and /dev/null differ diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png deleted file mode 100644 index f842d5707..000000000 Binary files a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/ic_public_input_voice_default.png and /dev/null differ diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.png deleted file mode 100644 index 3a6e5f2c6..000000000 Binary files a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.png and /dev/null differ diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg new file mode 100644 index 000000000..ab6b1fd76 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png deleted file mode 100644 index 3a6e5f2c6..000000000 Binary files a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_doc_default.png and /dev/null differ diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg new file mode 100644 index 000000000..0aeb30d63 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/icon_mic.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg new file mode 100644 index 000000000..2210223f4 --- /dev/null +++ b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png deleted file mode 100644 index 1bb7ed86d..000000000 Binary files a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle.png and /dev/null differ diff --git a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle_default.png b/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle_default.png deleted file mode 100644 index 1bb7ed86d..000000000 Binary files a/harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/info_circle_default.png and /dev/null differ diff --git a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc index 437600262..e572feeda 100644 --- a/sherpa-onnx/csrc/online-zipformer-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer-transducer-model.cc @@ -118,15 +118,24 @@ void OnlineZipformerTransducerModel::InitEncoder(void *model_data, for (auto i : v) { os << i << " "; } +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif }; print(encoder_dims_, "encoder_dims"); print(attention_dims_, "attention_dims"); print(num_encoder_layers_, "num_encoder_layers"); print(cnn_module_kernels_, "cnn_module_kernels"); print(left_context_len_, "left_context_len"); +#if __OHOS__ + SHERPA_ONNX_LOGE("T: %{public}d", T_); + SHERPA_ONNX_LOGE("decode_chunk_len_: %{public}d", decode_chunk_len_); +#else SHERPA_ONNX_LOGE("T: %d", T_); SHERPA_ONNX_LOGE("decode_chunk_len_: %d", decode_chunk_len_); +#endif } } @@ -147,7 +156,11 @@ void OnlineZipformerTransducerModel::InitDecoder(void *model_data, std::ostringstream os; os << "---decoder---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } Ort::AllocatorWithDefaultOptions allocator; // used in the macro below @@ -172,7 +185,11 @@ void OnlineZipformerTransducerModel::InitJoiner(void *model_data, std::ostringstream os; os << "---joiner---\n"; PrintModelMetadata(os, meta_data); +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif } } diff --git a/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc b/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc index 85a32ec3c..bd79cfc4a 100644 --- a/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc +++ b/sherpa-onnx/csrc/online-zipformer2-transducer-model.cc @@ -126,7 +126,11 @@ void OnlineZipformer2TransducerModel::InitEncoder(void *model_data, for (auto i : v) { os << i << " "; } +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s\n", os.str().c_str()); +#else SHERPA_ONNX_LOGE("%s\n", os.str().c_str()); +#endif }; print(encoder_dims_, "encoder_dims"); print(query_head_dims_, "query_head_dims"); @@ -135,8 +139,14 @@ void OnlineZipformer2TransducerModel::InitEncoder(void *model_data, print(num_encoder_layers_, "num_encoder_layers"); print(cnn_module_kernels_, "cnn_module_kernels"); print(left_context_len_, "left_context_len"); + +#if __OHOS__ + SHERPA_ONNX_LOGE("T: %{public}d", T_); + SHERPA_ONNX_LOGE("decode_chunk_len_: %{public}d", decode_chunk_len_); +#else SHERPA_ONNX_LOGE("T: %d", T_); SHERPA_ONNX_LOGE("decode_chunk_len_: %d", decode_chunk_len_); +#endif } }