Add Strudel fft function on Hydra (#310)

This also wires through the instance of the `StrudelWrapper`, just like
#309 .

- Patches strudels webaudio module through to be accessible from
anywhere.
- Adds `fft(index: number, buckets: number = 8, options?: { min?:
number; max?: number, scale?: number }): number`.
Currently this hooks into the webaudio module from strudel and takes
fft-data from there. In the future, we could add different fft sources,
and make the source be switchable (or make it use all per default).
- Automatically adds an `.analyze('flok-master')` to all strudel
patterns, if fft is used an a hydra pane

README.md

@@ -123,7 +123,7 @@ run Flok in secure mode by passing the `--secure` parameter:
 
 ```sh
 npx flok-web@latest --secure
-``` 
+```
 
 #### Note about remote users (not LAN)
 
@@ -159,7 +159,7 @@ object, like this:
 
 #### Sardine
 
-Use `flok-repl` with the `-t sardine` parameter. In order to make it work, 
+Use `flok-repl` with the `-t sardine` parameter. In order to make it work,
 the `sardine` REPL must be included to your PATH. It should already be the
 case if you followed a regular install.
 
@@ -217,14 +217,56 @@ installing and using it.
 
 #### Hydra
 
-[Hydra](https://hydra.ojack.xyz/) is a video synth and coding environment, inspired in 
-analog video synthesis, that runs directly in the browser and is already included in 
+[Hydra](https://hydra.ojack.xyz/) is a video synth and coding environment, inspired in
+analog video synthesis, that runs directly in the browser and is already included in
 the web App.  You don't need to install anything as it runs on the browser.  Just use
 the `hydra` target to execute Hydra code.
 
-You can also use [p5.js](https://p5js.org/) within a `hydra` target, like you would in 
+You can also use [p5.js](https://p5js.org/) within a `hydra` target, like you would in
 the official Hydra editor.
 
+##### `fft()` function
+
+The `fft()` function is a special function that allows you to get the FFT data
+from web targets.
+
+**Note: Only Strudel is supported at the moment.**
+
+**You can disable the FFT visualizer in the display settings. This might help with performance.**
+
+```ts
+fft(index: number,
+    buckets: number = 8,
+    options?: { min?: number; max?: number, scale?: number, analyzerId?: string }): number
+```
+
+Parameters:
+- `index: number` : The index of the bucket to return the value from.
+- `buckets: number`: The number of buckets to combine the underlying FFT data
+  too. Defaults to 8.
+- `options?: { min?: number; max?: number, scale?: number }`:
+    - `min?: number`: Minimum clamp value of the underlying data. Defaults to
+      -150.
+    - `max?: number`: Maximum clamp value of the underlying data. Defaults to 0.
+    - `scale?: number`: Scale of the output. Defaults to 1 (so the output is
+       from 0 to 1)
+    - `analyzerId?: string`: Which Strudel analyser to listen to. Defaults to
+      `flok-master`, which is also automatically added to all strudel patterns.
+      Can be used to route different patterns to different parts of the hydra
+      visualiser
+
+Example:
+```js
+solid(() => fft(0,1), 0)
+  .mask(shape(5,.05))
+  .rotate(() => 50 *  fft(0, 40)) // we need to supply a function
+                                  // for the parameter, for it to update automaticaly.
+```
+
+**Caveat**: Because of how we setup the analyze node on Strudel, every Strudel pane
+needs a re-eval after the Hydra code decides that we need to get the fft data.
+This does not happen automatically, manual re-eval is necessary.
+
 #### Mercury
 
 [Mercury](https://github.com/tmhglnd/mercury) is a minimal and human readable

packages/web/src/components/display-settings-dialog.tsx

@@ -77,6 +77,26 @@ export default function DisplaySettingsDialog({
                 }
               />
             </div>
+            <div className="">
+              <div className="grid grid-cols-4 items-center gap-4">
+                <Label htmlFor="name" className="text-right">
+                  Use FFT for visualization
+                </Label>
+                <input
+                  id="showCanvas"
+                  type="checkbox"
+                  checked={unsavedSettings.enableFft ?? true}
+                  className="w-5"
+                  onChange={(e) =>
+                    sanitizeAndSetUnsavedSettings({
+                      ...unsavedSettings,
+                      enableFft: e.target.checked,
+                    })
+                  }
+                />
+              </div>
+              <p>You need to reload the page to apply changes to fft</p>
+            </div>
           </div>
           <DialogFooter>
             <Button type="submit">Save changes</Button>

packages/web/src/lib/display-settings.ts

@@ -1,11 +1,13 @@
 export interface DisplaySettings {
   canvasPixelSize: number;
   showCanvas: boolean;
+  enableFft: boolean;
 }
 
 export const defaultDisplaySettings: DisplaySettings = {
   canvasPixelSize: 1,
   showCanvas: true,
+  enableFft: true,
 }
 
 export function sanitizeDisplaySettings(settings: DisplaySettings): DisplaySettings {
@@ -16,6 +18,7 @@ export function sanitizeDisplaySettings(settings: DisplaySettings): DisplaySetti
   // canvas; should be low enough
   const maxPixelSize = 50;
 
+
   return {
     ...settings,
       canvasPixelSize: Math.max(

packages/web/src/lib/hydra-wrapper.ts

@@ -1,12 +1,14 @@
 import Hydra from "hydra-synth";
 import { isWebglSupported } from "@/lib/webgl-detector.js";
+import {DisplaySettings} from "@/lib/display-settings.ts";
 
 declare global {
   interface Window {
     global: Window;
     src: Function;
     H: Function;
     P5: Function;
+    fft: (index: number, buckets: number) => number;
   }
 }
 
@@ -19,19 +21,27 @@ export class HydraWrapper {
   protected _hydra: any;
   protected _onError: ErrorHandler;
   protected _onWarning: ErrorHandler;
+  protected _displaySettings: DisplaySettings;
 
   constructor({
     canvas,
     onError,
     onWarning,
+    displaySettings,
   }: {
     canvas: HTMLCanvasElement;
     onError?: ErrorHandler;
     onWarning?: ErrorHandler;
+    displaySettings: DisplaySettings;
   }) {
     this._canvas = canvas;
     this._onError = onError || (() => {});
     this._onWarning = onWarning || (() => {});
+    this._displaySettings = displaySettings;
+  }
+
+  setDisplaySettings(displaySettings: DisplaySettings) {
+    this._displaySettings = displaySettings;
   }
 
   async initialize() {
@@ -64,6 +74,45 @@ export class HydraWrapper {
 
     window.H = this._hydra;
 
+    const clamp = (num: number, min: number, max: number) => Math.min(Math.max(num, min), max);
+
+
+    // Enables Hydra to use Strudel frequency data
+    // with `.scrollX(() => fft(1,0)` it will influence the x-axis, according to the fft data
+    // first number is the index of the bucket, second is the number of buckets to aggregate the number too
+    window.fft = (index: number, buckets: number = 8, options?: { min?: number; max?: number, scale?: number, analyzerId?: string }) => {
+      const analyzerId = options?.analyzerId ?? "flok-master"
+      const min = options?.min ?? -150;
+      const scale = options?.scale ?? 1
+      const max = options?.max ?? 0
+
+      // Strudel is not initialized yet, so we just return a default value
+      if(window.strudel == undefined) return .5;
+
+      // If display settings are not enabled, we just return a default value
+      if(!(this._displaySettings.enableFft ?? true)) return .5;
+
+      // Enable auto-analyze
+      window.strudel.enableAutoAnalyze = true;
+
+      // If the analyzerId is not defined, we just return a default value
+      if(window.strudel.webaudio.analysers[analyzerId] == undefined) {
+        return .5
+      }
+
+      const freq = window.strudel.webaudio.getAnalyzerData("frequency", analyzerId) as Array<number>;
+      const bucketSize = (freq.length) / buckets
+
+      // inspired from https://github.com/tidalcycles/strudel/blob/a7728e3d81fb7a0a2dff9f2f4bd9e313ddf138cd/packages/webaudio/scope.mjs#L53
+      const normalized = freq.map((it: number) => {
+        const norm = clamp((it - min) / (max - min), 0, 1);
+        return norm * scale;
+      })
+
+      return normalized.slice(bucketSize * index, bucketSize * (index + 1))
+        .reduce((a, b) => a + b, 0) / bucketSize
+    }
+
     this.initialized = true;
     console.log("Hydra initialized");
   }

packages/web/src/lib/strudel-wrapper.ts

@@ -33,6 +33,19 @@ export class StrudelWrapper {
   protected _docPatterns: any;
   protected _audioInitialized: boolean;
   protected framer?: any;
+  protected webaudio?: any;
+
+  enableAutoAnalyze = false;
+  hapAnalyzeSnippet = `
+    all(x => 
+      x.fmap(hap => {
+        if(hap.analyze == undefined) {
+          hap.analyze = 'flok-master';
+        }
+        return hap
+      })
+    )
+    `;
 
   constructor({
     onError,
@@ -49,6 +62,9 @@ export class StrudelWrapper {
 
   async importModules() {
     // import desired modules and add them to the eval scope
+
+    this.webaudio = await import("@strudel/webaudio");
+
     await evalScope(
       import("@strudel/core"),
       import("@strudel/midi"),
@@ -57,7 +73,7 @@ export class StrudelWrapper {
       import("@strudel/osc"),
       import("@strudel/serial"),
       import("@strudel/soundfonts"),
-      import("@strudel/webaudio"),
+      this.webaudio,
       controls
     );
     try {
@@ -145,12 +161,14 @@ export class StrudelWrapper {
     }
   }
 
+
   async tryEval(msg: EvalMessage) {
     if (!this.initialized) await this.initialize();
     try {
-      const { body: code, docId } = msg;
+      const {body: code, docId} = msg;
       // little hack that injects the docId at the end of the code to make it available in afterEval
-      const pattern = await this._repl.evaluate(`${code}//${docId}`);
+      // also add ann analyser node to all patterns, for fft data in hydra
+      const pattern = await this._repl.evaluate(`${code}\n${this.enableAutoAnalyze ? this.hapAnalyzeSnippet : ""}\n//${docId}`);
       if (pattern) {
         this._docPatterns[docId] = pattern.docId(docId); // docId is needed for highlighting
         const allPatterns = stack(...Object.values(this._docPatterns));

packages/web/src/routes/frames/hydra.tsx

@@ -43,6 +43,7 @@ export function Component() {
         onWarning: (msg) => {
           sendToast("warning", "Hydra warning", msg);
         },
+        displaySettings: displaySettings,
       });
 
       await hydra.initialize();
@@ -56,9 +57,14 @@ export function Component() {
   useAnimationFrame(
     useCallback(() => {
       window.m = window.parent?.mercury?.m;
+      window.strudel = window.parent?.strudel?.strudel;
     }, [])
   );
 
+  useEffect(() => {
+    instance?.setDisplaySettings(displaySettings);
+  }, [displaySettings]);
+
   useEvalHandler(
     useCallback(
       (msg) => {

packages/web/src/routes/frames/strudel.tsx

@@ -34,6 +34,8 @@ export function Component() {
       }
     };
 
+    window.strudel = instance;
+
     window.addEventListener("message", handleWindowMessage);
 
     return () => {