diff --git a/package.json b/package.json index aeb3382..58fca60 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "vscode-test": "^1.5.0" }, "dependencies": { + "axios": "^1.6.7", "express": "^4.18.3", "ws": "^8.16.0" } diff --git a/src/extension.ts b/src/extension.ts index 1eadd85..ab68802 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -76,6 +76,9 @@ export function activate(context: ExtensionContext) { () => { console.log(`[OBS Audio Assistant: Show] opening connection`); + const audioPanel = AudioAssistantPanel.render(context.extensionUri); + console.log(`[OBS Audio Assistant: Show] registered`); + // The code you place here will be executed every time your command is executed const wss = new WebSocketServer({ port: websocketPort }); console.log(`[OBS Audio Assistant: Show] openned socket`); @@ -89,26 +92,15 @@ export function activate(context: ExtensionContext) { console.log(`[OBS Audio Assistant] New WebSocket message: ${phrase.toString()}`); commandQueue += phrase + "\n"; - // TODO: insert code to handle command in `phrase` here const vscodeCommand = await getVscodeCommandFromUserQuery(phrase.toString()); console.log(vscodeCommand); + if (vscodeCommand.error) { + // don't do it + } else { + audioPanel?.playObs(vscodeCommand.data); + } }); }); - - /* Workflow - * display webview - * invoke STT - * wait for "got-stop-recording" - * got raw text - * send raw text to miniParser / LLM - * get back command - * get mp3 file name - * send to player - * Grammar - * [ check | play ] [ story 1-50 ] [ frame 1-10 ] - */ - AudioAssistantPanel.render(context.extensionUri); - console.log(`[OBS Audio Assistant: Show] registered`); } ); diff --git a/src/panels/AudioAssistantPanel.ts b/src/panels/AudioAssistantPanel.ts index 5cb4a31..a4c31a8 100644 --- a/src/panels/AudioAssistantPanel.ts +++ b/src/panels/AudioAssistantPanel.ts @@ -16,6 +16,13 @@ type CommandToFunctionMap = Record void>; */ export class AudioAssistantPanel { public static currentPanel: AudioAssistantPanel | undefined; + public playObs: ({ + story_number, + frame_number, + }: { + story_number: string; + frame_number: string; + }) => void; private readonly _panel: WebviewPanel; private _disposables: Disposable[] = []; @@ -37,6 +44,18 @@ export class AudioAssistantPanel { // Set an event listener to listen for messages passed from the webview context this._setWebviewMessageListener(this._panel.webview); + + this.playObs = ({ story_number, frame_number }) => { + const lang = "en/en"; + const domain = "https://cdn.door43.org/obs/mp3/1/"; + const scope = digits(story_number) + "-" + digits(frame_number); + const url = domain + lang + "-obs-v6/en_obs_" + scope + "_128kbps.mp3"; + + this._panel.webview.postMessage({ + command: "play", + data: { url: url }, + }); + }; } /** @@ -71,6 +90,7 @@ export class AudioAssistantPanel { ); AudioAssistantPanel.currentPanel = new AudioAssistantPanel(panel, extensionUri); + return AudioAssistantPanel.currentPanel; } } @@ -140,58 +160,79 @@ export class AudioAssistantPanel { private _setWebviewMessageListener(webview: Webview) { webview.onDidReceiveMessage( (message: any) => { - const { command, text } = message; - - const commandToFunctionMapping: CommandToFunctionMap = { - ["spoke"]: this._generateVscodeCommand, - }; - - commandToFunctionMapping[command](text); + // const { command, text } = message; + // const commandToFunctionMapping: CommandToFunctionMap = { + // ["spoke"]: this._generateVscodeCommand, + // }; + // commandToFunctionMapping[command](text); }, undefined, this._disposables ); } - /** - * @TODO Pass the user's speech text into the LLM to generate vscode command to run - * @Spidel - */ - private _generateVscodeCommand(text: string) { - // TODO: Generate this - const vscodeCommand = null; - - // TODO: Pass in the generated command - this._runVscodeCommand("", {}); - } - - /** - * @TODO Runs a LLM-generated vscode command - * @Kintsoogi - * @Spidel - */ - private _runVscodeCommand(command: string, data: any) { - const { storyNum, frameNum } = data; - - const commandToFunctionMapping: CommandToFunctionMap = { - ["play"]: this._playObs, - }; + // /** + // * @TODO Pass the user's speech text into the LLM to generate vscode command to run + // * @Spidel + // */ + // private _generateVscodeCommand(text: string) { + // // TODO: Generate this + // const vscodeCommand = null; + + // // TODO: Pass in the generated command + // this._runVscodeCommand("", {}); + // } + + // /** + // * @TODO Runs a LLM-generated vscode command + // * @Kintsoogi + // * @Spidel + // */ + // private _runVscodeCommand(command: string, data: any) { + // const { storyNum, frameNum } = data; + + // const commandToFunctionMapping: CommandToFunctionMap = { + // ["play"]: this._playObs, + // }; + + // commandToFunctionMapping[command](data); + // } +} - commandToFunctionMapping[command](data); +function digits(inp: string) { + let out = ""; + + switch (inp) { + case "one": + out = "01"; + break; + case "two": + out = "02"; + break; + case "three": + out = "03"; + break; + case "four": + out = "04"; + break; + case "five": + out = "05"; + break; + case "six": + out = "06"; + break; + case "seven": + out = "07"; + break; + case "eight": + out = "08"; + break; + case "nine": + out = "09"; + break; + default: + out = ("0" + inp).slice(-2); } - /** - * @TODO Play an OBS audio file given story and frame number - * @Rich - */ - private _playObs({ storyNum, frameNum }: { storyNum: number; frameNum: number }) { - // TODO: Get .mp3 data given storyNum and frameNum - console.log(storyNum, frameNum); - - // TODO: Signal to webview to play mp3 - this._panel.webview.postMessage({ - command: "", - data: {}, - }); - } + return out; } diff --git a/src/utilities/parser.ts b/src/utilities/parser.ts new file mode 100644 index 0000000..5f5e7c4 --- /dev/null +++ b/src/utilities/parser.ts @@ -0,0 +1,20 @@ +export function parser(cli: string) { + let trimmed = cli.replace(/\s+/g, ' ').trim(); // remove extra white space + let parts = trimmed.split(" "); // tokenize + let command: string = "c"; + let story: number = 1; + let frame: number = 1; + + for (var a in parts) { + switch (parts[a]) { + case "play": + case "check": command = "c"; break; + case "story": story = parseInt(parts[parts.indexOf("story") + 1], 10); break; + case "frame": frame = parseInt(parts[parts.indexOf("frame") + 1], 10); + } + } + + return [command, story, frame]; +} + + diff --git a/src/utilities/picker.ts b/src/utilities/picker.ts new file mode 100644 index 0000000..c359545 --- /dev/null +++ b/src/utilities/picker.ts @@ -0,0 +1,10 @@ +import { parser } from "../utilities/parser"; + +export function picker(speech: string) { // construct url to OBS frame clip + var [cmd, story, frame] = parser("check story 1 frame 2" + speech); + // "en/en-obs-v6/en_obs_02-05_128kbps.mp3" + const lang = "en/en"; + const domain = "https://cdn.door43.org/obs/mp3/1/"; + const scope = ("0" + story).slice(-2) + "_" + ("0" + frame).slice(-2); + return domain + lang + "_obs_" + scope + "_128kbps.mp3"; +} diff --git a/webview-ui/.gitignore b/webview-ui/.gitignore index 133f67c..31d07ec 100644 --- a/webview-ui/.gitignore +++ b/webview-ui/.gitignore @@ -5,3 +5,5 @@ dist-ssr build build-ssr *.local +parser.ts +picker.ts diff --git a/webview-ui/icon_mic_preview_4b61.png b/webview-ui/icon_mic_preview_4b61.png new file mode 100644 index 0000000..e5a986f Binary files /dev/null and b/webview-ui/icon_mic_preview_4b61.png differ diff --git a/webview-ui/src/App.tsx b/webview-ui/src/App.tsx index 3845e18..2e0545f 100644 --- a/webview-ui/src/App.tsx +++ b/webview-ui/src/App.tsx @@ -1,4 +1,5 @@ import { vscode } from "./utilities/vscode"; +import { useState, useEffect } from "react"; import "./App.css"; function App() { @@ -6,6 +7,48 @@ function App() { * @TODO Send text message of user speech to extension.js * @Bruce.MCL */ + const [obsLink, setObsLink] = useState(""); + + useEffect(() => { + const micImg = document.getElementById("micImg"); + + if (micImg) { + micImg.addEventListener("mousedown", (event) => { + console.log("start recording"); + vscode.postMessage({ + command: "extension.microphoneState", + enabled: true, + }); + }); + + micImg.addEventListener("mouseup", (event) => { + console.log("stop recording"); + vscode.postMessage({ + command: "extension.microphoneState", + enabled: false, + }); + }); + } + + window.addEventListener("message", (event) => { + const message = event.data; // The JSON data our extension sent + const { command, data } = message; + + switch (command) { + case "play": + const aud = new Audio(data.url); + + if (aud) { + aud.muted = true; + aud.play(); + aud.muted = false; + } + setObsLink(data.url); + break; + } + }); + }, []); + function handleSpeechInput(command: string) { vscode.postMessage({ command: "spoke", @@ -13,13 +56,14 @@ function App() { }); } + const renderedLink = obsLink ? OBS Story : null; + return (

Audio OBS Navigator

-
- - -
+

+ +

{obsLink}

); }