Skip to content

Commit

Permalink
Add microphone demo about VAD+ASR for HarmonyOS (#1581)
Browse files Browse the repository at this point in the history
  • Loading branch information
@csukuangfj
csukuangfj authored Nov 30, 2024
1 parent 299f239 commit c9d3b6c
Show file tree
Hide file tree
Showing 7 changed files with 276 additions and 23 deletions.
177 changes: 161 additions & 16 deletions harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,59 +3,154 @@ import worker, { MessageEvents } from '@ohos.worker';
import { BusinessError } from '@kit.BasicServicesKit';
import { picker } from '@kit.CoreFileKit';

import { Permissions } from '@kit.AbilityKit';
import { allAllowed, requestPermissions } from './Permission';
import { audio } from '@kit.AudioKit';


@Entry
@Component
struct Index {
@State currentIndex: number = 0;
@State resultFromFile: string = '';
@State resultForFile: string = '';
@State progressForFile: number = 0;
@State selectFileBtnEnabled: boolean = false;
@State message: string = 'To be implemented';
@State lang: string = 'English';
@State resultForMic: string = '';
@State micStarted: boolean = false;
@State message: string = 'Start recording';
@State micInitDone: boolean = false;
private controller: TabsController = new TabsController();
private workerInstance?: worker.ThreadWorker
private readonly scriptURL: string = 'entry/ets/workers/NonStreamingAsrWithVadWorker.ets'
private mic?: audio.AudioCapturer;
private sampleList: Float32Array[] = []

flatten(samples: Float32Array[]): Float32Array {
let n = 0;
for (let i = 0; i < samples.length; ++i) {
n += samples[i].length;
}

const ans: Float32Array = new Float32Array(n);
let offset: number = 0;
for (let i = 0; i < samples.length; ++i) {
ans.set(samples[i], offset);
offset += samples[i].length;
}

return ans;
}

async initMic() {
const permissions: Permissions[] = ["ohos.permission.MICROPHONE"];
let allowed: boolean = await allAllowed(permissions);
if (!allowed) {
requestPermissions(permissions);
console.log("request to access the microphone");

allowed = await allAllowed(permissions);
if (!allowed) {
console.error('failed to get microphone permission');
this.resultForMic = "Failed to get microphone permission. Please retry";
return;
}
} else {
console.log("allowed to access microphone");
}

aboutToAppear(): void {
const audioStreamInfo: audio.AudioStreamInfo = {
samplingRate: audio.AudioSamplingRate.SAMPLE_RATE_16000,
channels: audio.AudioChannel.CHANNEL_1,
sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE,
encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW,
};

const audioCapturerInfo: audio.AudioCapturerInfo = {
source: audio.SourceType.SOURCE_TYPE_MIC,
capturerFlags: 0
};

const audioCapturerOptions: audio.AudioCapturerOptions = {
streamInfo: audioStreamInfo,
capturerInfo: audioCapturerInfo

};
audio.createAudioCapturer(audioCapturerOptions, (err, data) => {
if (err) {
console.error(`error code is ${err.code}, error message is ${err.message}`);
this.resultForMic = 'Failed to init microphone';
} else {
console.info(`init mic successfully`);
this.mic = data;
this.mic.on('readData', this.micCallback);

if (this.workerInstance) {
this.workerInstance.postMessage({ msgType: 'init-vad-mic', context: getContext() });
}
}
});
}

async aboutToAppear() {
this.workerInstance = new worker.ThreadWorker(this.scriptURL, {
name: 'NonStreaming ASR worker'
});

this.workerInstance.onmessage = (e: MessageEvents) => {
const msgType = e.data['msgType'] as string;
console.log(`received data ${msgType}`);
console.log(`received msg from worker: ${msgType}`);

if (msgType == 'init-vad-mic-done') {
this.micInitDone = true;
}

if (msgType == 'init-non-streaming-asr-done') {
this.selectFileBtnEnabled = true;
this.resultForFile = `Initializing done.\n\nPlease select a wave file of 16kHz in language ${this.lang}`;
}

if (msgType == 'non-streaming-asr-vad-decode-done') {
this.resultFromFile = e.data['text'] as string + '\n';
this.resultForFile = e.data['text'] as string + '\n';
}

if (msgType == 'non-streaming-asr-vad-decode-partial') {
if (this.resultFromFile == '') {
this.resultFromFile = e.data['text'] as string;
if (this.resultForFile == '') {
this.resultForFile = e.data['text'] as string;
} else {
this.resultFromFile += '\n\n' + e.data['text'] as string;
this.resultForFile += '\n\n' + e.data['text'] as string;
}
}

if (msgType == 'non-streaming-asr-vad-decode-error') {
this.resultFromFile = e.data['text'] as string;
this.resultForFile = e.data['text'] as string;
}

if (msgType == 'non-streaming-asr-vad-decode-progress') {
this.progressForFile = e.data['progress'] as number;

this.selectFileBtnEnabled = this.progressForFile >= 100;
}

if (msgType == 'non-streaming-asr-vad-mic-partial') {
if (this.resultForMic == '') {
this.resultForMic = e.data['text'] as string;
} else {
this.resultForMic += '\n\n' + e.data['text'] as string;
}
}

if (msgType == 'non-streaming-asr-vad-mic-error') {
this.resultForMic = e.data['text'] as string;
}
}

const context = getContext();
this.resultForFile = 'Initializing models';
this.workerInstance.postMessage({ msgType: 'init-vad', context });
this.workerInstance.postMessage({ msgType: 'init-non-streaming-asr', context });

await this.initMic();
}

@Builder
Expand Down Expand Up @@ -86,13 +181,13 @@ struct Index {
.lineHeight(41)
.fontWeight(500)

Button('Select .wav file ')
Button('Select .wav file (16kHz) ')
.enabled(this.selectFileBtnEnabled)
.fontSize(13)
.width(296)
.height(60)
.onClick(() => {
this.resultFromFile = '';
this.resultForFile = '';
this.progressForFile = 0;

const documentSelectOptions = new picker.DocumentSelectOptions();
Expand All @@ -103,7 +198,7 @@ struct Index {
console.log(`Result: ${result}`);

if (!result[0]) {
this.resultFromFile = 'Please select a file to decode';
this.resultForFile = 'Please select a file to decode';
this.selectFileBtnEnabled = true;
return;
}
Expand Down Expand Up @@ -135,7 +230,7 @@ struct Index {
}.width('100%').justifyContent(FlexAlign.Center)
}

TextArea({ text: this.resultFromFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP });
TextArea({ text: this.resultForFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP });

}
.alignItems(HorizontalAlign.Center)
Expand All @@ -144,10 +239,50 @@ struct Index {

TabContent() {
Column() {
Text(this.message)
.fontSize(50)
.fontWeight(FontWeight.Bold);
Button(this.message)
.enabled(this.micInitDone)
.onClick(() => {
console.log('clicked mic button');
this.resultForMic = '';
if (this.mic) {
if (this.micStarted) {
this.mic.stop();
this.message = "Start recording";
this.micStarted = false;
console.log('mic stopped');

const samples = this.flatten(this.sampleList);
let s = 0;
for (let i = 0; i < samples.length; ++i) {
s += samples[i];
}
console.log(`samples ${samples.length}, sum: ${s}`);

if (this.workerInstance) {
console.log('decode mic');
this.workerInstance.postMessage({
msgType: 'non-streaming-asr-vad-mic',
samples,
});
} else {
console.log(`this worker instance is undefined ${this.workerInstance}`);
}
} else {
this.sampleList = [];
this.mic.start();
this.message = "Stop recording";
this.micStarted = true;
console.log('mic started');
}
}
});

Text(`Supported languages: ${this.lang}`)

TextArea({ text: this.resultForMic }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP });
}
.alignItems(HorizontalAlign.Center)
.justifyContent(FlexAlign.Start)
}
.tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'),
$r('app.media.ic_public_input_voice_default')))
Expand All @@ -170,4 +305,14 @@ struct Index {
.width('100%')
.justifyContent(FlexAlign.Start)
}

private micCallback = (buffer: ArrayBuffer) => {
const view: Int16Array = new Int16Array(buffer);

const samplesFloat: Float32Array = new Float32Array(view.length);
for (let i = 0; i < view.length; ++i) {
samplesFloat[i] = view[i] / 32768.0;
}
this.sampleList.push(samplesFloat);
}
}
5 changes: 3 additions & 2 deletions harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,10 @@ export function getOfflineModelConfig(type: number): OfflineModelConfig {

break;
}
default: {
console.log(`Please specify a supported type. Given type ${type}`);
}
}

console.log(`Please specify a supported type. Given type ${type}`);

return c;
}
26 changes: 26 additions & 0 deletions harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Permission.ets
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// This file is modified from
// https://gitee.com/ukSir/hmchat2/blob/master/entry/src/main/ets/utils/permissionMananger.ets
import { abilityAccessCtrl, bundleManager, common, Permissions } from '@kit.AbilityKit';

export function allAllowed(permissions: Permissions[]): boolean {
if (permissions.length == 0) {
return false;
}

const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager();

const bundleInfo = bundleManager.getBundleInfoForSelfSync(bundleManager.BundleFlag.GET_BUNDLE_INFO_WITH_APPLICATION);

let tokenID: number = bundleInfo.appInfo.accessTokenId;

return permissions.every(permission => abilityAccessCtrl.GrantStatus.PERMISSION_GRANTED ==
mgr.checkAccessTokenSync(tokenID, permission));
}

export async function requestPermissions(permissions: Permissions[]): Promise<boolean> {
const mgr: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager();
const context: Context = getContext() as common.UIAbilityContext;

const result = await mgr.requestPermissionsFromUser(context, permissions);
return result.authResults.length > 0 && result.authResults.every(authResults => authResults == 0);
}
Loading

0 comments on commit c9d3b6c

Please sign in to comment.