added mic icon, ability to play obs frame, begin hooking up sequence

package.json

@@ -41,6 +41,7 @@
     "vscode-test": "^1.5.0"
   },
   "dependencies": {
+    "axios": "^1.6.7",
     "express": "^4.18.3",
     "ws": "^8.16.0"
   }

src/extension.ts

@@ -76,6 +76,9 @@ export function activate(context: ExtensionContext) {
     () => {
       console.log(`[OBS Audio Assistant: Show] opening connection`);
 
+      const audioPanel = AudioAssistantPanel.render(context.extensionUri);
+      console.log(`[OBS Audio Assistant: Show] registered`);
+
       // The code you place here will be executed every time your command is executed
       const wss = new WebSocketServer({ port: websocketPort });
       console.log(`[OBS Audio Assistant: Show] openned socket`);
@@ -89,26 +92,15 @@ export function activate(context: ExtensionContext) {
           console.log(`[OBS Audio Assistant] New WebSocket message: ${phrase.toString()}`);
           commandQueue += phrase + "\n";
 
-          // TODO: insert code to handle command in `phrase` here
           const vscodeCommand = await getVscodeCommandFromUserQuery(phrase.toString());
           console.log(vscodeCommand);
+          if (vscodeCommand.error) {
+            // don't do it
+          } else {
+            audioPanel?.playObs(vscodeCommand.data);
+          }
         });
       });
-
-      /* Workflow
-       *   display webview
-       *   invoke STT
-       *   wait for "got-stop-recording"
-       *   got raw text
-       *   send raw text to miniParser / LLM
-       *   get back command
-       *   get mp3 file name
-       *   send to player
-       * Grammar
-       *   [ check | play ] [ story 1-50 ] [ frame 1-10 ]
-       */
-      AudioAssistantPanel.render(context.extensionUri);
-      console.log(`[OBS Audio Assistant: Show] registered`);
     }
   );
 

src/panels/AudioAssistantPanel.ts

@@ -16,6 +16,13 @@ type CommandToFunctionMap = Record<string, (data: any) => void>;
  */
 export class AudioAssistantPanel {
   public static currentPanel: AudioAssistantPanel | undefined;
+  public playObs: ({
+    story_number,
+    frame_number,
+  }: {
+    story_number: string;
+    frame_number: string;
+  }) => void;
   private readonly _panel: WebviewPanel;
   private _disposables: Disposable[] = [];
 
@@ -37,6 +44,18 @@ export class AudioAssistantPanel {
 
     // Set an event listener to listen for messages passed from the webview context
     this._setWebviewMessageListener(this._panel.webview);
+
+    this.playObs = ({ story_number, frame_number }) => {
+      const lang = "en/en";
+      const domain = "https://cdn.door43.org/obs/mp3/1/";
+      const scope = digits(story_number) + "-" + digits(frame_number);
+      const url = domain + lang + "-obs-v6/en_obs_" + scope + "_128kbps.mp3";
+
+      this._panel.webview.postMessage({
+        command: "play",
+        data: { url: url },
+      });
+    };
   }
 
   /**
@@ -71,6 +90,7 @@ export class AudioAssistantPanel {
       );
 
       AudioAssistantPanel.currentPanel = new AudioAssistantPanel(panel, extensionUri);
+      return AudioAssistantPanel.currentPanel;
     }
   }
 
@@ -140,58 +160,79 @@ export class AudioAssistantPanel {
   private _setWebviewMessageListener(webview: Webview) {
     webview.onDidReceiveMessage(
       (message: any) => {
-        const { command, text } = message;
-
-        const commandToFunctionMapping: CommandToFunctionMap = {
-          ["spoke"]: this._generateVscodeCommand,
-        };
-
-        commandToFunctionMapping[command](text);
+        // const { command, text } = message;
+        // const commandToFunctionMapping: CommandToFunctionMap = {
+        //   ["spoke"]: this._generateVscodeCommand,
+        // };
+        // commandToFunctionMapping[command](text);
       },
       undefined,
       this._disposables
     );
   }
 
-  /**
-   * @TODO Pass the user's speech text into the LLM to generate vscode command to run
-   * @Spidel
-   */
-  private _generateVscodeCommand(text: string) {
-    // TODO: Generate this
-    const vscodeCommand = null;
-
-    // TODO: Pass in the generated command
-    this._runVscodeCommand("", {});
-  }
-
-  /**
-   * @TODO Runs a LLM-generated vscode command
-   * @Kintsoogi
-   * @Spidel
-   */
-  private _runVscodeCommand(command: string, data: any) {
-    const { storyNum, frameNum } = data;
-
-    const commandToFunctionMapping: CommandToFunctionMap = {
-      ["play"]: this._playObs,
-    };
+  // /**
+  //  * @TODO Pass the user's speech text into the LLM to generate vscode command to run
+  //  * @Spidel
+  //  */
+  // private _generateVscodeCommand(text: string) {
+  //   // TODO: Generate this
+  //   const vscodeCommand = null;
+
+  //   // TODO: Pass in the generated command
+  //   this._runVscodeCommand("", {});
+  // }
+
+  // /**
+  //  * @TODO Runs a LLM-generated vscode command
+  //  * @Kintsoogi
+  //  * @Spidel
+  //  */
+  // private _runVscodeCommand(command: string, data: any) {
+  //   const { storyNum, frameNum } = data;
+
+  //   const commandToFunctionMapping: CommandToFunctionMap = {
+  //     ["play"]: this._playObs,
+  //   };
+
+  //   commandToFunctionMapping[command](data);
+  // }
+}
 
-    commandToFunctionMapping[command](data);
+function digits(inp: string) {
+  let out = "";
+
+  switch (inp) {
+    case "one":
+      out = "01";
+      break;
+    case "two":
+      out = "02";
+      break;
+    case "three":
+      out = "03";
+      break;
+    case "four":
+      out = "04";
+      break;
+    case "five":
+      out = "05";
+      break;
+    case "six":
+      out = "06";
+      break;
+    case "seven":
+      out = "07";
+      break;
+    case "eight":
+      out = "08";
+      break;
+    case "nine":
+      out = "09";
+      break;
+    default:
+      out = ("0" + inp).slice(-2);
   }
 
-  /**
-   * @TODO Play an OBS audio file given story and frame number
-   * @Rich
-   */
-  private _playObs({ storyNum, frameNum }: { storyNum: number; frameNum: number }) {
-    // TODO: Get .mp3 data given storyNum and frameNum
-    console.log(storyNum, frameNum);
-
-    // TODO: Signal to webview to play mp3
-    this._panel.webview.postMessage({
-      command: "",
-      data: {},
-    });
-  }
+  return out;
 }

src/utilities/parser.ts

@@ -0,0 +1,20 @@
+export function parser(cli: string) {
+    let trimmed = cli.replace(/\s+/g, ' ').trim(); // remove extra white space
+    let parts = trimmed.split(" "); // tokenize
+    let command: string = "c";
+    let story: number = 1;
+    let frame: number = 1;
+
+    for (var a in parts) {
+        switch (parts[a]) {
+            case "play":
+            case "check": command = "c"; break;
+            case "story": story = parseInt(parts[parts.indexOf("story") + 1], 10); break;
+            case "frame": frame = parseInt(parts[parts.indexOf("frame") + 1], 10);
+        }
+    }
+
+    return [command, story, frame];
+}
+
+

src/utilities/picker.ts

@@ -0,0 +1,10 @@
+import { parser } from "../utilities/parser";
+
+export function picker(speech: string) {  // construct url to OBS frame clip
+    var [cmd, story, frame] = parser("check story 1 frame 2" + speech);
+    //  "en/en-obs-v6/en_obs_02-05_128kbps.mp3"
+    const lang = "en/en";
+    const domain = "https://cdn.door43.org/obs/mp3/1/";
+    const scope = ("0" + story).slice(-2) + "_" + ("0" + frame).slice(-2);
+    return domain + lang + "_obs_" + scope + "_128kbps.mp3";
+}

webview-ui/.gitignore

@@ -5,3 +5,5 @@ dist-ssr
 build
 build-ssr
 *.local
+parser.ts
+picker.ts

webview-ui/src/App.tsx

@@ -1,25 +1,69 @@
 import { vscode } from "./utilities/vscode";
+import { useState, useEffect } from "react";
 import "./App.css";
 
 function App() {
   /**
    * @TODO Send text message of user speech to extension.js
    * @Bruce.MCL
    */
+  const [obsLink, setObsLink] = useState("");
+
+  useEffect(() => {
+    const micImg = document.getElementById("micImg");
+
+    if (micImg) {
+      micImg.addEventListener("mousedown", (event) => {
+        console.log("start recording");
+        vscode.postMessage({
+          command: "extension.microphoneState",
+          enabled: true,
+        });
+      });
+
+      micImg.addEventListener("mouseup", (event) => {
+        console.log("stop recording");
+        vscode.postMessage({
+          command: "extension.microphoneState",
+          enabled: false,
+        });
+      });
+    }
+
+    window.addEventListener("message", (event) => {
+      const message = event.data; // The JSON data our extension sent
+      const { command, data } = message;
+
+      switch (command) {
+        case "play":
+          const aud = new Audio(data.url);
+
+          if (aud) {
+            aud.muted = true;
+            aud.play();
+            aud.muted = false;
+          }
+          setObsLink(data.url);
+          break;
+      }
+    });
+  }, []);
+
   function handleSpeechInput(command: string) {
     vscode.postMessage({
       command: "spoke",
       text: `User spoke: ${command}`,
     });
   }
 
+  const renderedLink = obsLink ? <a href={obsLink}>OBS Story</a> : null;
+
   return (
     <main>
       <h1>Audio OBS Navigator</h1>
-      <div className="recordBtns">
-        <i className="codicon codicon-mic"></i>
-        <i className="codicon codicon-stop"></i>
-      </div>
+      <br></br>
+      <img id="micImg" src="../icon_mic_preview_4b61.png" width="50"></img>
+      <h3>{obsLink}</h3>
     </main>
   );
 }