Allow users to generate pipelines from YAML files (#563)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Cody Baker <[email protected]>

.storybook/main.js → .storybook/main.ts

@@ -1,5 +1,8 @@
-/** @type { import('@storybook/web-components-vite').StorybookConfig } */
-const config = {
+import type { StorybookConfig } from '@storybook/web-components-vite';
+
+import ViteYaml from "@modyfi/vite-plugin-yaml";
+
+const config: StorybookConfig = {
     stories: ["../src/**/*.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
     addons: ["@storybook/addon-links", "@storybook/addon-essentials"],
     framework: {
@@ -9,5 +12,12 @@ const config = {
     docs: {
         autodocs: "tag",
     },
+  async viteFinal(config, { configType }) {
+
+    config.plugins.push(ViteYaml()) // Add YAML Loader to vite config
+
+    return config
+  },
 };
+
 export default config;

electron.vite.config.js

@@ -1,8 +1,12 @@
 import { defineConfig } from "electron-vite";
 
+import ViteYaml from "@modyfi/vite-plugin-yaml";
+
 // electron.vite.config.js
 export default defineConfig({
     main: {},
     preload: {},
-    renderer: {},
+    renderer: {
+        plugins: [ViteYaml()],
+    },
 });

guide_testing_suite.yml

@@ -0,0 +1,19 @@
+# Follows pattern
+# pipelines:
+#   pipeline_name:
+#     DataInterfaceName:
+#       file_or_folder_path: <relative path>
+#       other_source_data_inputs: <insert if needed>
+#     AdditionalDataInterface: ...
+
+pipelines:
+
+  SpikeGLX-Phy:
+    SpikeGLXRecordingInterface:
+      file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin
+    PhySortingInterface:
+      folder_path: ephy_testing_data/phy/phy_example_0
+
+  SpikeGLX-v1-SingleProbe-AP:
+    SpikeGLXRecordingInterface:
+      file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin

package.json

@@ -159,11 +159,12 @@
   "devDependencies": {
     "@electron-toolkit/utils": "^2.0.1",
     "@electron/notarize": "^2.2.0",
-    "@storybook/addon-essentials": "^7.0.0-beta.62",
-    "@storybook/addon-links": "^7.0.0-beta.62",
-    "@storybook/blocks": "^7.0.0-beta.62",
-    "@storybook/web-components": "^7.0.0-beta.62",
-    "@storybook/web-components-vite": "^7.0.0-beta.62",
+    "@modyfi/vite-plugin-yaml": "^1.0.4",
+    "@storybook/addon-essentials": "^7.6.10",
+    "@storybook/addon-links": "^7.6.10",
+    "@storybook/blocks": "^7.6.10",
+    "@storybook/web-components": "^7.6.10",
+    "@storybook/web-components-vite": "^7.6.10",
     "@vitest/coverage-v8": "^0.34.1",
     "chromatic": "^6.17.1",
     "electron": "^26.2.2",
@@ -175,7 +176,7 @@
     "prettier": "^2.3.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "storybook": "^7.0.0-beta.62",
+    "storybook": "^7.6.10",
     "vite": "^4.1.4",
     "vitest": "^0.34.1"
   },

schemas/json/developer/globals.json

@@ -0,0 +1,9 @@
+{
+    "properties": {
+        "testing_data_folder": {
+            "type": "string",
+            "format": "directory",
+            "description": "Common folder where GIN testing data (e.g. ephy_testing_data, ophys_testing_data) can be found"
+        }
+    }
+}

src/renderer/src/stories/Accordion.js

@@ -234,7 +234,8 @@ export class Accordion extends LitElement {
     };
 
     toOpen = (state = this.open) => {
-        if (!this.toggleable) return true; // Force open if not toggleable
+        if (!this.toggleable)
+            return true; // Force open if not toggleable
         else if (this.disabled) return false; // Force closed if disabled
         return state;
     };

src/renderer/src/stories/FileSystemSelector.js

@@ -167,9 +167,10 @@ export class FilesystemSelector extends LitElement {
             // const path = file.filePath ?? file.filePaths?.[0];
             this.#handleFiles(results.filePath ?? results.filePaths, type);
         } else {
-            let handles = await (type === "directory"
-                ? window.showDirectoryPicker()
-                : window.showOpenFilePicker({ multiple: this.multiple })
+            let handles = await (
+                type === "directory"
+                    ? window.showDirectoryPicker()
+                    : window.showOpenFilePicker({ multiple: this.multiple })
             ).catch(() => []); // Call using the same options
 
             const result = Array.isArray(handles) ? handles.map(({ name }) => name) : handles.name;

src/renderer/src/stories/SimpleTable.js

@@ -613,7 +613,8 @@ export class SimpleTable extends LitElement {
         let target = this.data;
 
         if (!isResolved) {
-            if (!this.keyColumn) this.data[rowName] = {}; // Add new row to array
+            if (!this.keyColumn)
+                this.data[rowName] = {}; // Add new row to array
             else {
                 rowName = row;
                 if (!this.#unresolved[rowName]) this.#unresolved[rowName] = {}; // Ensure row exists

src/renderer/src/stories/Table.js

@@ -444,7 +444,8 @@ export class Table extends LitElement {
                 let target = this.data;
 
                 if (!isResolved) {
-                    if (!this.keyColumn) this.data[rowName] = {}; // Add new row to array
+                    if (!this.keyColumn)
+                        this.data[rowName] = {}; // Add new row to array
                     else {
                         rowName = row;
                         if (!unresolved[rowName]) unresolved[rowName] = {}; // Ensure row exists

src/renderer/src/stories/pages/Page.js

@@ -66,7 +66,8 @@ export class Page extends LitElement {
                 transition === 1 && // Only ensure save for standard forward progression
                 !this.info.states.saved)
         ) {
-            if (transition === 1) await this.save(); // Save before a single forward transition
+            if (transition === 1)
+                await this.save(); // Save before a single forward transition
             else {
                 await Swal.fire({
                     title: "You have unsaved data on this page.",
@@ -144,7 +145,8 @@ export class Page extends LitElement {
         if (!Array.isArray(toRun)) toRun = this.mapSessions();
 
         // Filter the sessions to run
-        if (typeof original === "number") toRun = randomizeElements(toRun, original); // Grab a random set of sessions
+        if (typeof original === "number")
+            toRun = randomizeElements(toRun, original); // Grab a random set of sessions
         else if (typeof original === "string") toRun = toRun.filter(({ subject }) => subject === original);
         else if (typeof original === "function") toRun = toRun.filter(original);
 

src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js

@@ -190,12 +190,14 @@ export class GuidedPathExpansionPage extends Page {
             if (!keepExistingData) {
                 for (let sub in globalResults) {
                     const subRef = results[sub];
-                    if (!subRef) delete globalResults[sub]; // Delete removed subjects
+                    if (!subRef)
+                        delete globalResults[sub]; // Delete removed subjects
                     else {
                         for (let ses in globalResults[sub]) {
                             const sesRef = subRef[ses];
 
-                            if (!sesRef) delete globalResults[sub][ses]; // Delete removed sessions
+                            if (!sesRef)
+                                delete globalResults[sub][ses]; // Delete removed sessions
                             else {
                                 const globalSesRef = globalResults[sub][ses];
 

src/renderer/src/stories/pages/settings/SettingsPage.js

@@ -4,14 +4,97 @@ import { Page } from "../Page.js";
 import { onThrow } from "../../../errors";
 import dandiGlobalSchema from "../../../../../../schemas/json/dandi/global.json";
 import projectGlobalSchema from "../../../../../../schemas/json/project/globals.json" assert { type: "json" };
+import developerGlobalSchema from "../../../../../../schemas/json/developer/globals.json" assert { type: "json" };
 
 import { validateDANDIApiKey } from "../../../validation/dandi";
 
+import { Button } from "../../Button.js";
+import { global, save } from "../../../progress/index.js";
+import { merge, setUndefinedIfNotDeclared } from "../utils.js";
+
+import { notyf } from "../../../dependencies/globals.js";
+import { SERVER_FILE_PATH, fs, path, port } from "../../../electron/index.js";
+
+import saveSVG from "../../assets/save.svg?raw";
+
+import { header } from "../../forms/utils";
+
+import testingSuiteYaml from "../../../../../../guide_testing_suite.yml";
+
+const propertiesToTransform = ["folder_path", "file_path"];
+
+function saveNewPipelineFromYaml(name, sourceData, rootFolder) {
+    const subjectId = "mouse1";
+    const sessions = ["session1"];
+
+    const resolvedSourceData = structuredClone(sourceData);
+    Object.values(resolvedSourceData).forEach((info) => {
+        propertiesToTransform.forEach((property) => {
+            if (info[property]) info[property] = path.join(rootFolder, info[property]);
+        });
+    });
+
+    save({
+        info: {
+            globalState: {
+                project: {
+                    name: header(name),
+                    initialized: true,
+                },
+
+                // provide data for all supported interfaces
+                interfaces: Object.keys(resolvedSourceData).reduce((acc, key) => {
+                    acc[key] = `${key}`;
+                    return acc;
+                }, {}),
+
+                structure: {
+                    keep_existing_data: true,
+                    state: false,
+                },
+
+                results: {
+                    [subjectId]: sessions.reduce((acc, sessionId) => {
+                        acc[subjectId] = {
+                            metadata: {
+                                Subject: {
+                                    subject_id: subjectId,
+                                },
+                                NWBFile: {
+                                    session_id: sessionId,
+                                },
+                            },
+                            source_data: resolvedSourceData,
+                        };
+                        return acc;
+                    }, {}),
+                },
+
+                subjects: {
+                    [subjectId]: {
+                        sessions: sessions,
+                        sex: "M",
+                        species: "Mus musculus",
+                        age: "P30D",
+                    },
+                },
+            },
+        },
+    });
+}
+
 const schema = merge(
     projectGlobalSchema,
     {
         properties: {
-            DANDI: dandiGlobalSchema,
+            DANDI: {
+                title: "DANDI Settings",
+                ...dandiGlobalSchema,
+            },
+            developer: {
+                title: "Developer Settings",
+                ...developerGlobalSchema,
+            },
         },
         required: ["DANDI"],
     },
@@ -20,15 +103,6 @@ const schema = merge(
     }
 );
 
-import { Button } from "../../Button.js";
-import { global } from "../../../progress/index.js";
-import { merge, setUndefinedIfNotDeclared } from "../utils.js";
-
-import { notyf } from "../../../dependencies/globals.js";
-import { SERVER_FILE_PATH, port } from "../../../electron/index.js";
-
-import saveSVG from "../../assets/save.svg?raw";
-
 export class SettingsPage extends Page {
     header = {
         title: "App Settings",
@@ -82,9 +156,40 @@ export class SettingsPage extends Page {
             onThrow,
         });
 
+        const generatePipelineButton = new Button({
+            label: "Generate Test Pipelines",
+            onClick: async () => {
+                const { testing_data_folder } = this.form.results.developer ?? {};
+
+                if (!testing_data_folder)
+                    return this.#openNotyf(
+                        `Please specify a testing data folder in the Developer section before attempting to generate pipelines.`,
+                        "error"
+                    );
+
+                const { pipelines = {} } = testingSuiteYaml;
+
+                const pipelineNames = Object.keys(pipelines);
+                const nPipelines = pipelineNames.length;
+                pipelineNames
+                    .reverse()
+                    .forEach((name) => saveNewPipelineFromYaml(name, pipelines[name], testing_data_folder));
+
+                this.#openNotyf(`Generated ${nPipelines} test pipelines`, "success");
+            },
+        });
+
+        setTimeout(() => {
+            const testFolderInput = this.form.getInput(["developer", "testing_data_folder"]);
+            console.log(testFolderInput);
+            testFolderInput.after(generatePipelineButton);
+        }, 100);
+
         return html`
             <p><b>Server Port:</b> ${port}</p>
             <p><b>Server File Location:</b> ${SERVER_FILE_PATH}</p>
+            <hr />
+            <br />
             ${this.form}
         `;
     }

src/renderer/src/stories/pages/utils.js

@@ -53,7 +53,8 @@ export function merge(toMerge = {}, target = {}, mergeOptions = {}) {
         } else if (isObject(value)) {
             if (isObject(targetValue)) target[k] = merge(value, targetValue, mergeOptions);
             else {
-                if (mergeOptions.clone) target[k] = merge(value, {}, mergeOptions); // Replace primitive values
+                if (mergeOptions.clone)
+                    target[k] = merge(value, {}, mergeOptions); // Replace primitive values
                 else target[k] = value; // Replace object values
             }
         } else target[k] = value; // Replace primitive values