Merge branch 'main' into improve-subject-interaction-with-results

NeurodataWithoutBorders · Mar 13, 2024 · ae598c4 · ae598c4
2 parents 632522e + 1340258
commit ae598c4
Show file tree

Hide file tree

Showing 29 changed files with 1,268 additions and 594 deletions.
diff --git a/.codespellrc b/.codespellrc
@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list= afterall
+ignore-words-list= afterall,preform
 skip = .git,*.svg,package-lock.json,node_modules,*lotties*,nwb-guide.spec,prepare_pyinstaller_spec.py
diff --git a/.gitignore b/.gitignore
@@ -36,3 +36,6 @@ src/build
 .env
 .env.local
 .env.production
+
+# Spyder
+.spyproject/
diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py
diff --git a/schemas/base-metadata.schema.ts b/schemas/base-metadata.schema.ts
@@ -4,6 +4,8 @@ import { header, replaceRefsWithValue } from '../src/renderer/src/stories/forms/
 
 import baseMetadataSchema from './json/base_metadata_schema.json' assert { type: "json" }
 
+const uvMathFormat = `&micro;V`; //`<math xmlns="http://www.w3.org/1998/Math/MathML"><mo>&micro;</mo><mi>V</mi></math>`
+
 function getSpeciesNameComponents(arr: any[]) {
     const split = arr[arr.length - 1].split(' - ')
     return {
@@ -13,6 +15,7 @@ function getSpeciesNameComponents(arr: any[]) {
 }
 
 
+
 function getSpeciesInfo(species: any[][] = []) {
 
 
@@ -34,6 +37,10 @@ function getSpeciesInfo(species: any[][] = []) {
 
 }
 
+const propsToInclude = {
+    ecephys: ["Device", "ElectrodeGroup", "Electrodes", "ElectrodeColumns", "definitions"]
+}
+
 export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, global = false) => {
 
 
@@ -89,17 +96,40 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa
     // Override description of keywords
     nwbProps.keywords.description = 'Terms to describe your dataset (e.g. Neural circuits, V1, etc.)' // Add description to keywords
 
-
+    const ecephys = copy.properties.Ecephys
     const ophys = copy.properties.Ophys
 
+    if (ecephys) {
+
+        // Change rendering order for electrode table columns
+        const electrodesProp = ecephys.properties["Electrodes"]
+        for (let name in electrodesProp.properties) {
+            const interfaceProps = electrodesProp.properties[name].properties
+            const electrodeItems = interfaceProps["Electrodes"].items.properties
+            const uvProperties = ["gain_to_uV", "offset_to_uV"]
+
+            uvProperties.forEach(prop => {
+                electrodeItems[prop] = {}
+                electrodeItems[prop].title = prop.replace('uV', uvMathFormat)
+                console.log(electrodeItems[prop])
+            })
+            interfaceProps["Electrodes"].items.order = ["channel_name", "group_name", "shank_electrode_number", ...uvProperties];
+            interfaceProps["ElectrodeColumns"].items.order = ["name", "description", "data_type"];
+
+        }
+
+    }
+
     if (ophys) {
 
         ophys.required = Object.keys(ophys.properties)
 
         const getProp = (name: string) => ophys.properties[name]
 
-        if (getProp("TwoPhotonSeries")) {
-            const tpsItemSchema = getProp("TwoPhotonSeries").items
+        const tpsItemSchema = getProp("TwoPhotonSeries")?.items
+
+        if (tpsItemSchema) {
+
             tpsItemSchema.order = [
                 "name",
                 "description",
@@ -111,8 +141,9 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa
         }
 
 
-        if (getProp("ImagingPlane")) {
-            const imagingPlaneItems = getProp("ImagingPlane").items
+        const imagingPlaneItems = getProp("ImagingPlane")?.items
+
+        if (imagingPlaneItems) {
             imagingPlaneItems.order = [
                 "name",
                 "description",

diff --git a/src/renderer/src/pages.js b/src/renderer/src/pages.js
@@ -10,7 +10,6 @@ import { GuidedSubjectsPage } from "./stories/pages/guided-mode/setup/GuidedSubj
 import { GuidedSourceDataPage } from "./stories/pages/guided-mode/data/GuidedSourceData";
 import { GuidedMetadataPage } from "./stories/pages/guided-mode/data/GuidedMetadata";
 import { GuidedUploadPage } from "./stories/pages/guided-mode/options/GuidedUpload";
-// import { GuidedConversionOptionsPage } from "./stories/pages/guided-mode/options/GuidedConversionOptions";
 import { GuidedResultsPage } from "./stories/pages/guided-mode/results/GuidedResults";
 import { Dashboard } from "./stories/Dashboard";
 import { GuidedStubPreviewPage } from "./stories/pages/guided-mode/options/GuidedStubPreview";
@@ -28,6 +27,7 @@ import { UploadsPage } from "./stories/pages/uploads/UploadsPage";
 import { SettingsPage } from "./stories/pages/settings/SettingsPage";
 import { InspectPage } from "./stories/pages/inspect/InspectPage";
 import { PreviewPage } from "./stories/pages/preview/PreviewPage";
+import { GuidedPreform } from "./stories/pages/guided-mode/setup/Preform";
 
 let dashboard = document.querySelector("nwb-dashboard");
 if (!dashboard) dashboard = new Dashboard();
@@ -94,6 +94,12 @@ const pages = {
                 section: sections[0],
             }),
 
+            workflow: new GuidedPreform({
+                title: "Pipeline Workflow",
+                label: "Pipeline workflow",
+                section: sections[0],
+            }),
+
             structure: new GuidedStructurePage({
                 title: "Provide Data Formats",
                 label: "Data formats",

diff --git a/src/renderer/src/progress/operations.js b/src/renderer/src/progress/operations.js
@@ -12,8 +12,6 @@ export const remove = (name) => {
     } else localStorage.removeItem(progressFilePathToDelete);
 
     if (fs) {
-        console.log(previewSaveFolderPath, conversionSaveFolderPath, name);
-
         // delete default preview location
         fs.rmSync(joinPath(previewSaveFolderPath, name), { recursive: true, force: true });
 

diff --git a/src/renderer/src/stories/BasicTable.js b/src/renderer/src/stories/BasicTable.js
@@ -1,14 +1,15 @@
-import { LitElement, css, html } from "lit";
+import { LitElement, css, html, unsafeCSS } from "lit";
 import { styleMap } from "lit/directives/style-map.js";
 import { header } from "./forms/utils";
 import { checkStatus } from "../validation";
-import { errorHue, warningHue } from "./globals";
+import { emojiFontFamily, errorHue, warningHue } from "./globals";
 
 import * as promises from "../promises";
 
 import "./Button";
 import { sortTable } from "./Table";
 import tippy from "tippy.js";
+import { getIgnore } from "./JSONSchemaForm";
 
 export class BasicTable extends LitElement {
     static get styles() {
@@ -65,6 +66,12 @@ export class BasicTable extends LitElement {
                 user-select: none;
             }
 
+            .relative .info {
+                margin: 0px 5px;
+                font-size: 80%;
+                font-family: ${unsafeCSS(emojiFontFamily)};
+            }
+
             th span {
                 display: inline-block;
             }
@@ -115,6 +122,7 @@ export class BasicTable extends LitElement {
         validateOnChange,
         onStatusChange,
         onLoaded,
+        onUpdate,
     } = {}) {
         super();
         this.name = name ?? "data_table";
@@ -127,6 +135,7 @@ export class BasicTable extends LitElement {
         this.ignore = ignore ?? {};
 
         if (validateOnChange) this.validateOnChange = validateOnChange;
+        if (onUpdate) this.onUpdate = onUpdate;
         if (onStatusChange) this.onStatusChange = onStatusChange;
         if (onLoaded) this.onLoaded = onLoaded;
     }
@@ -158,9 +167,29 @@ export class BasicTable extends LitElement {
         return html`<div class="relative"><span>${header(str)}</span></div>`;
     };
 
-    #renderHeader = (str, { description }) => {
-        if (description) return html`<th title="${description}">${this.#renderHeaderContent(str)}</th>`;
-        return html`<th>${this.#renderHeaderContent(str)}</th>`;
+    #renderHeader = (prop, { description, title = prop } = {}) => {
+        const th = document.createElement("th");
+
+        const required = this.#itemSchema.required ? this.#itemSchema.required.includes(prop) : false;
+        const container = document.createElement("div");
+        container.classList.add("relative");
+        const span = document.createElement("span");
+        span.innerHTML = header(title);
+        if (required) span.setAttribute("required", "");
+        container.appendChild(span);
+
+        // Add Description Tooltip
+        if (description) {
+            const span = document.createElement("span");
+            span.classList.add("info");
+            span.innerText = "ℹ️";
+            container.append(span);
+            tippy(span, { content: `${description[0].toUpperCase() + description.slice(1)}`, allowHTML: true });
+        }
+
+        th.appendChild(container);
+
+        return th;
     };
 
     #getRowData(row, cols = this.colHeaders) {
@@ -169,13 +198,12 @@ export class BasicTable extends LitElement {
             let value;
             if (col === this.keyColumn) {
                 if (hasRow) value = row;
-                else return "";
+                else return;
             } else
                 value =
                     (hasRow ? this.data[row][col] : undefined) ??
                     // this.globals[col] ??
-                    this.#itemSchema.properties[col].default ??
-                    "";
+                    this.#itemSchema.properties[col]?.default;
             return value;
         });
     }
@@ -210,43 +238,52 @@ export class BasicTable extends LitElement {
     onStatusChange = () => {};
     onLoaded = () => {};
 
-    #validateCell = (value, col, parent) => {
+    #getType = (value, { type, data_type } = {}) => {
+        let inferred = typeof value;
+        if (Array.isArray(value)) inferred = "array";
+        if (value == undefined) inferred = "null";
+
+        const original = type || data_type;
+        let resolved = original;
+
+        // Handle based on JSON Schema types
+        if (type) {
+            if (resolved === "integer") resolved = "number"; // Map to javascript type
+        } else if (data_type) {
+            if (resolved.includes("array")) resolved = "array";
+            if (resolved.includes("int") || resolved.includes("float")) resolved = "number";
+            if (resolved.startsWith("bool")) resolved = "boolean";
+            if (resolved.startsWith("str")) resolved = "string";
+        }
+
+        return {
+            type: resolved,
+            original,
+            inferred,
+        };
+    };
+
+    #validateCell = (value, col, row, parent) => {
         if (!value && !this.validateEmptyCells) return true; // Empty cells are valid
         if (!this.validateOnChange) return true;
 
         let result;
 
         const propInfo = this.#itemProps[col] ?? {};
-        let thisTypeOf = typeof value;
-        let ogType;
-        let type = (ogType = propInfo.type || propInfo.data_type);
 
-        // Handle based on JSON Schema types
-        if ("type" in propInfo) {
-            // Map to javascript type
-            if (type === "integer") type = "number";
-
-            // Convert to json schema type
-            if (Array.isArray(value)) thisTypeOf = "array";
-            if (value == undefined) thisTypeOf = "null";
-        } else if ("data_type" in propInfo) {
-            if (type.includes("array")) type = "array";
-            if (type.includes("int") || type.includes("float")) type = "number";
-            if (type.startsWith("bool")) type = "boolean";
-            if (type.startsWith("str")) type = "string";
-        }
+        let { type, original, inferred } = this.#getType(value, propInfo);
 
         // Check if required
-        if (!value && "required" in this.#itemSchema.required.includes(col))
+        if (!value && "required" in this.#itemSchema && this.#itemSchema.required.includes(col))
             result = [{ message: `${col} is a required property`, type: "error" }];
-        // If not required, check matching types for values that are defined
-        else if (value !== "" && thisTypeOf !== type)
-            result = [{ message: `${col} is expected to be of type ${ogType}, not ${thisTypeOf}`, type: "error" }];
+        // If not required, check matching types (if provided) for values that are defined
+        else if (value !== "" && type && inferred !== type)
+            result = [{ message: `${col} is expected to be of type ${original}, not ${inferred}`, type: "error" }];
         // Otherwise validate using the specified onChange function
-        else result = this.validateOnChange(col, parent, value, this.#itemProps[col]);
+        else result = this.validateOnChange([row, col], parent, value, this.#itemProps[col]);
 
         // Will run synchronously if not a promise result
-        return promises.resolve(result, () => {
+        return promises.resolve(result, (result) => {
             let info = {
                 title: undefined,
                 warning: undefined,
@@ -277,7 +314,7 @@ export class BasicTable extends LitElement {
 
         const results = this.#data.map((v, i) => {
             return v.map((vv, j) => {
-                const info = this.#validateCell(vv, this.colHeaders[j], { ...this.data[rows[i]] }); // Could be a promise or a basic response
+                const info = this.#validateCell(vv, this.colHeaders[j], i, { ...this.data[rows[i]] }); // Could be a promise or a basic response
                 return promises.resolve(info, (info) => {
                     if (info === true) return;
                     const td = this.shadowRoot.getElementById(`i${i}_j${j}`);
@@ -336,7 +373,7 @@ export class BasicTable extends LitElement {
         let data = text.split("\n").map((row) =>
             row.split("\t").map((v) => {
                 try {
-                    return JSON.parse(v);
+                    return eval(v);
                 } catch {
                     return v.trim();
                 }
@@ -356,19 +393,24 @@ export class BasicTable extends LitElement {
         Object.keys(data).forEach((row) => {
             const cols = structuredData[row];
             const latest = (this.data[this.keyColumn ? cols[this.keyColumn] : row] = {});
-            Object.entries(cols).forEach(([key, value]) => (key in this.#itemProps ? (latest[key] = value) : "")); // Only include data from schema
+            Object.entries(cols).forEach(([key, value]) => {
+                if (key in this.#itemProps) {
+                    const { type } = this.#getType(value, this.#itemProps[key]);
+                    if (type === "string") value = `${value}`; // Convert to string if necessary
+                    latest[key] = value;
+                }
+            }); // Only include data from schema
         });
 
-        this.onUpdate(null, null, value); // Update the whole table
+        if (this.onUpdate) this.onUpdate([], data); // Update the whole table
     }
 
     // Render Code
     render() {
         this.#updateRendered();
 
+        this.schema = this.schema; // Always update the schema
         const entries = this.#itemProps;
-        for (let key in this.ignore) delete entries[key];
-        for (let key in this.ignore["*"] ?? {}) delete entries[key];
 
         // Add existing additional properties to the entries variable if necessary
         if (this.#itemSchema.additionalProperties) {
@@ -384,6 +426,10 @@ export class BasicTable extends LitElement {
             }, entries);
         }
 
+        // Ignore any additions in the ignore configuration
+        for (let key in this.ignore) delete entries[key];
+        for (let key in this.ignore["*"] ?? {}) delete entries[key];
+
         // Sort Columns by Key Column and Requirement
         const keys =
             (this.#keys =
@@ -418,7 +464,9 @@ export class BasicTable extends LitElement {
                         ${data.map(
                             (row, i) =>
                                 html`<tr>
-                                    ${row.map((col, j) => html`<td id="i${i}_j${j}"><div>${col}</div></td>`)}
+                                    ${row.map(
+                                        (col, j) => html`<td id="i${i}_j${j}"><div>${JSON.stringify(col)}</div></td>`
+                                    )}
                                 </tr>`
                         )}
                     </tbody>