From b8c2691018b94426fae079a58ddce4f762242a77 Mon Sep 17 00:00:00 2001 From: Garrett Date: Fri, 15 Sep 2023 09:45:30 -0700 Subject: [PATCH 1/8] Test multi-upload for all conditions (fails on symlinked nested folders) --- pyflask/apis/neuroconv.py | 33 +++++++++++++++---- pyflask/manageNeuroconv/__init__.py | 1 + pyflask/manageNeuroconv/manage_neuroconv.py | 9 +++++ schemas/json/dandi/standalone.json | 11 ++++--- .../src/stories/FileSystemSelector.js | 4 +++ .../src/stories/pages/uploads/UploadsPage.js | 13 +++++--- 6 files changed, 56 insertions(+), 15 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 0db136328..0f3020720 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -15,7 +15,10 @@ generate_dataset, inspect_nwb_file, inspect_nwb_folder, - inspect_multiple_filesystem_objects + inspect_multiple_filesystem_objects, + upload_to_dandi, + upload_folder_to_dandi, + upload_multiple_filesystem_objects_to_dandi ) from errorHandlers import notBadRequestException @@ -112,13 +115,11 @@ def post(self): neuroconv_api.abort(500, str(e)) -@neuroconv_api.route("/upload") +@neuroconv_api.route("/upload/project") class Upload(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - from manageNeuroconv import upload_to_dandi - return upload_to_dandi(**neuroconv_api.payload) except Exception as e: @@ -131,8 +132,6 @@ class Upload(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - from manageNeuroconv import upload_folder_to_dandi - return upload_folder_to_dandi(**neuroconv_api.payload) except Exception as e: @@ -140,6 +139,28 @@ def post(self): neuroconv_api.abort(500, str(e)) +@neuroconv_api.route("/upload") +class Upload(Resource): + @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def post(self): + from os.path import isdir + + try: + paths = neuroconv_api.payload["filesystem_paths"] + + if (len(paths) == 1 and isdir(paths[0])): + kwargs = { **neuroconv_api.payload } + del kwargs["filesystem_paths"] + kwargs["nwb_folder_path"] = paths[0] + return upload_folder_to_dandi(**kwargs) + + else: + return upload_multiple_filesystem_objects_to_dandi(**neuroconv_api.payload) + + except Exception as e: + if notBadRequestException(e): + neuroconv_api.abort(500, str(e)) + @neuroconv_api.route("/inspect_file") class InspectNWBFile(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index 8adea2add..db610e516 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -7,6 +7,7 @@ validate_metadata, upload_to_dandi, upload_folder_to_dandi, + upload_multiple_filesystem_objects_to_dandi, listen_to_neuroconv_events, generate_dataset, inspect_nwb_file, diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b3da52419..2569b6b02 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -330,6 +330,15 @@ def update_conversion_progress(**kwargs): return dict(file=str(resolved_output_path)) +def upload_multiple_filesystem_objects_to_dandi(**kwargs): + tmp_folder_path = aggregate_in_temp_directory(kwargs['filesystem_paths'], 'upload') + innerKwargs = { **kwargs } + del innerKwargs['filesystem_paths'] + innerKwargs["nwb_folder_path"] = tmp_folder_path + result = upload_folder_to_dandi(**innerKwargs) + rmtree(tmp_folder_path) + return result + def upload_folder_to_dandi( dandiset_id: str, api_key: str, diff --git a/schemas/json/dandi/standalone.json b/schemas/json/dandi/standalone.json index e56062017..dfa6369d2 100644 --- a/schemas/json/dandi/standalone.json +++ b/schemas/json/dandi/standalone.json @@ -1,9 +1,12 @@ { "properties": { - "nwb_folder_path": { - "type": "string", - "format": "directory" + "filesystem_paths": { + "type": "array", + "items":{ + "type": "string", + "format": ["file", "directory"] + } } }, - "required": ["nwb_folder_path"] + "required": ["filesystem_paths"] } diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index fdf71b649..a6702ad8a 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -39,6 +39,10 @@ const componentCSS = css` gap: 5px; } + #button-div > nwb-button { + margin-bottom: 10px; + } + button { background: WhiteSmoke; border: 1px solid #c3c3c3; diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index 2e4691212..f464563c8 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -2,6 +2,8 @@ import { html } from "lit"; import { JSONSchemaForm } from "../../JSONSchemaForm.js"; import { Page } from "../Page.js"; import { onThrow } from "../../../errors"; + +const folderPathKey = "filesystem_paths"; import dandiUploadSchema from "../../../../../../schemas/json/dandi/upload.json"; import dandiStandaloneSchema from "../../../../../../schemas/json/dandi/standalone.json"; const dandiSchema = merge(dandiStandaloneSchema, merge(dandiUploadSchema, {}), { arrays: true }); @@ -18,7 +20,7 @@ import { DandiResults } from "../../DandiResults.js"; export const isStaging = (id) => parseInt(id) >= 100000; -export async function uploadToDandi(info, type = "project" in info ? "" : "folder") { +export async function uploadToDandi(info, type = "project" in info ? "project" : "") { const api_key = global.data.DANDI?.api_key; if (!api_key) { await Swal.fire({ @@ -31,11 +33,13 @@ export async function uploadToDandi(info, type = "project" in info ? "" : "folde return this.to("settings"); } + const { dandiset_id } = info + const result = await run( type ? `upload/${type}` : "upload", { ...info, - staging: isStaging(info.dandiset_id), // Automatically detect staging IDs + staging: isStaging(dandiset_id), // Automatically detect staging IDs api_key, }, { title: "Uploading to DANDI" } @@ -50,7 +54,7 @@ export async function uploadToDandi(info, type = "project" in info ? "" : "folde if (result) notyf.open({ type: "success", - message: `${info.project ?? info.nwb_folder_path} successfully uploaded to Dandiset ${info.dandiset_id}`, + message: `${info.project ?? `${info[folderPathKey].length} filesystem entries`} successfully uploaded to Dandiset ${dandiset_id}`, }); return result; @@ -85,13 +89,12 @@ export class UploadsPage extends Page { }, }); - const folderPathKey = "nwb_folder_path"; // NOTE: API Keys and Dandiset IDs persist across selected project this.form = new JSONSchemaForm({ results: globalState, schema: dandiSchema, sort: ([k1]) => { - if (k1 === "nwb_folder_path") return -1; + if (k1 === folderPathKey) return -1; }, onUpdate: ([id]) => { if (id === folderPathKey) { From 5a738b74080b5c5da0a27f5ff785d7b7e3537950 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 21 Sep 2023 17:46:57 -0700 Subject: [PATCH 2/8] Produce an information box with all the source files found --- src/renderer/src/stories/CodeBlock.js | 38 ++++ src/renderer/src/stories/InfoBox.js | 60 +++--- src/renderer/src/stories/JSONSchemaForm.js | 25 ++- src/renderer/src/stories/OptionalSection.js | 24 +-- .../guided-mode/data/GuidedPathExpansion.js | 172 ++++++++++++++++-- .../preview/inspector/InspectorList.js | 2 +- 6 files changed, 260 insertions(+), 61 deletions(-) create mode 100644 src/renderer/src/stories/CodeBlock.js diff --git a/src/renderer/src/stories/CodeBlock.js b/src/renderer/src/stories/CodeBlock.js new file mode 100644 index 000000000..db26762c6 --- /dev/null +++ b/src/renderer/src/stories/CodeBlock.js @@ -0,0 +1,38 @@ +import { LitElement, css, html } from "lit"; + +export class CodeBlock extends LitElement { + + static get styles() { + return css` + + :host { + display: block; + font-size: 85%; + background: #f2f1f1; + border-radius: 10px; + border: 1px solid gray; + overflow: hidden; + } + + pre { + overflow: auto; + padding: 5px 10px; + box-sizing: border-box; + user-select: text; + margin: 0; + } + ` + } + + constructor({ text = '' }){ + super() + this.text = text + } + + render(){ + return html`
${this.text}
` + } +} + +customElements.get("code-block") || + customElements.define("code-block", CodeBlock); diff --git a/src/renderer/src/stories/InfoBox.js b/src/renderer/src/stories/InfoBox.js index 42130b898..3ee6120da 100644 --- a/src/renderer/src/stories/InfoBox.js +++ b/src/renderer/src/stories/InfoBox.js @@ -73,45 +73,49 @@ export class InfoBox extends LitElement { margin-bottom: 0px; animation: demo-box-fade-in 0.2s cubic-bezier(0, 0.2, 0.2, 0.96); } + `; } - constructor({ header = "Info", content, type = "info" } = {}) { + constructor({ header = "Info", content, type = "info", open = false } = {}) { super(); this.header = header; this.content = content; this.type = type; + this.open = open } updated() { - const infoDropdowns = this.shadowRoot.querySelectorAll(".guided--info-dropdown"); - for (const infoDropdown of Array.from(infoDropdowns)) { - const infoTextElement = infoDropdown.querySelector("#header"); - - // Auto-add icons if they're not there - if (this.type === "info") infoTextElement.insertAdjacentHTML("beforebegin", `ℹ️`); - if (this.type === "warning") - infoTextElement.insertAdjacentHTML("beforebegin", ` ⚠️`); - - infoDropdown.onclick = () => { - const infoContainer = infoDropdown.nextElementSibling; - const infoContainerChevron = infoDropdown.querySelector("nwb-chevron"); - - const infoContainerIsopen = infoContainer.classList.contains("container-open"); - - if (infoContainerIsopen) { - infoContainerChevron.direction = "right"; - infoContainer.classList.remove("container-open"); - } else { - infoContainerChevron.direction = "bottom"; - infoContainer.classList.add("container-open"); - } - }; - } + const infoDropdown = this.shadowRoot.querySelector(".guided--info-dropdown"); + const infoTextElement = infoDropdown.querySelector("#header"); + + // Auto-add icons if they're not there + if (this.type === "info") infoTextElement.insertAdjacentHTML("beforebegin", `ℹ️`); + if (this.type === "warning") infoTextElement.insertAdjacentHTML("beforebegin", ` ⚠️`); + + const infoContainer = infoDropdown.nextElementSibling; + infoDropdown.onclick = () => this.onToggle(!infoContainer.classList.contains("container-open")) + + this.onToggle() } - render() { - return html` + onToggle(open = this.open) { + const infoDropdown = this.shadowRoot.querySelector(".guided--info-dropdown"); + + const infoContainer = infoDropdown.nextElementSibling; + const infoContainerChevron = infoDropdown.querySelector("nwb-chevron"); + + if (open) { + infoContainerChevron.direction = "bottom"; + infoContainer.classList.add("container-open"); + } else { + infoContainerChevron.direction = "right"; + infoContainer.classList.remove("container-open"); + } +} + +render() { + return html`
${this.header} ${new Chevron({ direction: "right" })} @@ -120,7 +124,7 @@ export class InfoBox extends LitElement { ${this.content}
`; - } +} } customElements.get("nwbguide-info-box") || customElements.define("nwbguide-info-box", InfoBox); diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index e1902a2a3..7328559ea 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -96,15 +96,15 @@ hr { margin: 1em 0 1.5em 0; } -pre { - white-space: pre-wrap; /* Since CSS 2.1 */ - white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ - white-space: -pre-wrap; /* Opera 4-6 */ - white-space: -o-pre-wrap; /* Opera 7 */ - word-wrap: break-word; /* Internet Explorer 5.5+ */ - font-family: unset; - color: DimGray; -} + pre { + white-space: pre-wrap; /* Since CSS 2.1 */ + white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + word-wrap: break-word; /* Internet Explorer 5.5+ */ + font-family: unset; + color: DimGray; + } .required label:after { content: " *"; @@ -434,6 +434,7 @@ export class JSONSchemaForm extends LitElement { ${interactiveInput}
+
`; }; @@ -587,6 +588,10 @@ export class JSONSchemaForm extends LitElement { ? valid?.filter((info) => info.type === "error" || (isRequired && info.missing)) : []; + const info = Array.isArray(valid) + ? valid?.filter((info) => info.type === "info" ) + : []; + const hasLinks = this.#getLink(externalPath); if (hasLinks) { if (checkLinks) { @@ -612,6 +617,7 @@ export class JSONSchemaForm extends LitElement { // Clear old errors and warnings this.#clearMessages(fullPath, "errors"); this.#clearMessages(fullPath, "warnings"); + this.#clearMessages(fullPath, "info"); const isFunction = typeof valid === "function"; const isValid = @@ -629,6 +635,7 @@ export class JSONSchemaForm extends LitElement { // Show aggregated errors and warnings (if any) warnings.forEach((info) => this.#addMessage(fullPath, info, "warnings")); + info.forEach((info) => this.#addMessage(fullPath, info, "info")); if (isValid && errors.length === 0) { element.classList.remove("invalid"); diff --git a/src/renderer/src/stories/OptionalSection.js b/src/renderer/src/stories/OptionalSection.js index aa46e45d2..be2ccf524 100644 --- a/src/renderer/src/stories/OptionalSection.js +++ b/src/renderer/src/stories/OptionalSection.js @@ -5,14 +5,18 @@ import { Button } from "./Button"; export class OptionalSection extends LitElement { static get styles() { return css` + :host { - text-align: center; + display: block; } h2 { margin: 0; } + .optional-section__toggle { + padding-bottom: 20px; } + .optional-section__content { text-align: left; } @@ -97,17 +101,15 @@ export class OptionalSection extends LitElement { render() { return html` -
-
- ${this.header ? html`

${this.header}

` : ""} -
${this.description}
-
${this.yes} ${this.no}
-
- - +
+ ${this.header ? html`

${this.header}

` : ""} +
${this.description}
+
${this.yes} ${this.no}
+
+ + `; } } diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js index 3b1705405..14358b6c4 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js @@ -11,6 +11,107 @@ import { onThrow } from "../../../../errors"; import pathExpansionSchema from "../../../../../../../schemas/json/path-expansion.schema.json" assert { type: "json" }; import { InfoBox } from "../../../InfoBox.js"; import { merge } from "../../utils.js"; +import { CodeBlock } from "../../../CodeBlock.js"; +import { List } from "../../../List"; +import { fs } from "../../../../electron/index.js"; +import { joinPath } from "../../../../globals.js"; + + +const exampleFileStructure = `mylab/ + ¦ Subjects/ + ¦ +-- NR_0017/ + ¦ ¦ +-- 2022-03-22/ + ¦ ¦ ¦ +-- 001/ + ¦ ¦ ¦ ¦ +-- raw_video_data/ + ¦ ¦ ¦ ¦ ¦ +-- _leftCamera.raw.6252a2f0-c10f-4e49-b085-75749ba29c35.mp4 + ¦ ¦ ¦ ¦ ¦ +-- ... + ¦ ¦ ¦ ¦ +-- ... + ¦ +-- NR_0019/ + ¦ ¦ +-- 2022-04-29/ + ¦ ¦ ¦ +-- 001/ + ¦ ¦ ¦ ¦ +-- raw_video_data/ + ¦ ¦ ¦ ¦ ¦ +-- _leftCamera.raw.9041b63e-02e2-480e-aaa7-4f6b776a647f.mp4 + ¦ ¦ ¦ ¦ ¦ +-- ... + ¦ ¦ ¦ ¦ +-- ... + ¦ ...` + + +const exampleFormatPath = "Subjects/{subject_id}/{session_start_time:%Y-%m-%d}/{session_id}/raw_video_data/leftCamera.raw.{}.mp4" + +const exampleMetadata = { + "source_data": { + "IBL_video": { + "file_path": "/mylab/Subjects/NR_0017/2022-03-22/001/raw_video_data/leftCamera.raw.6252a2f0-c10f-4e49-b085-75749ba29c35.mp4" + } + }, + "metadata": { + "NWBFile": { + "session_id": "001", + "session_start_time": "2022-03-22" + }, + "Subject": { + "subject_id": "NR_0017" + } + } +} + +const pathExpansionInfoBox = new InfoBox({ + header: "How do I use a Python format string for path expansion?", + content: html` +
+

Consider a dataset of that includes video recordings from three cameras, stored in the following directory structure.

+ ${new CodeBlock({ text: exampleFileStructure })} + +

Using mylab as the base directory, the correct format string to extract the subject ID, session start time, and session number would be:

+ ${new CodeBlock({ text: exampleFormatPath})} + +
+ +

The above example applies all of the supported f-string variables, which are used to extract information into the resulting metadata:

+ ${new List({ + items: [ + { + value: 'subject_id' + }, + { + value: 'session_id' + }, + { + value: 'session_start_time' + } + ], + editable: false + })} + +

Additional variables (or blank braces) can be specified to indicate wildcard patterns.

+ +

Consequently, the metadata extracted from the first file found using this approach would be the following:

+ ${new CodeBlock({ text: JSON.stringify(exampleMetadata, null, 2)})} + +
+ + For complete documentation of the path expansion feature of NeuroConv, visit the + path expansion documentation page. + +
+ `, +}); + +pathExpansionInfoBox.style.margin = "10px 0px"; + +function getFiles(dir) { + const dirents = fs.readdirSync(dir, { withFileTypes: true }); + let entries = [] + for (const dirent of dirents) { + const res = joinPath(dir, dirent.name); + if (dirent.isDirectory()) entries.push(...getFiles(res)); + else entries.push(res) + } + + return entries +} export class GuidedPathExpansionPage extends Page { constructor(...args) { @@ -140,7 +241,7 @@ export class GuidedPathExpansionPage extends Page { optional = new OptionalSection({ header: "Would you like to locate data programmatically?", - description: html`

Automatically detect source data for multiple subjects and sessions.

`, + description: pathExpansionInfoBox, onChange: () => (this.unsavedUpdates = true), // altContent: this.altForm, }); @@ -152,6 +253,7 @@ export class GuidedPathExpansionPage extends Page { const state = structureState.state; if (state !== undefined) this.optional.state = state; + else pathExpansionInfoBox.open = true // Open the info box if no option has been selected // Require properties for all sources const generatedSchema = { type: "object", properties: {} }; @@ -164,21 +266,67 @@ export class GuidedPathExpansionPage extends Page { const form = (this.form = new JSONSchemaForm({ ...structureState, onThrow, - onUpdate: () => (this.unsavedUpdates = true), + validateOnChange: async (name, parent, parentPath) => { + const value = parent[name] + if (fs) { + // if (name === 'base_directory') { + // for (const f of getFiles(value)) { + // console.log(f); + // } + // const res = getFiles(value); + // } + // else + if (name === 'format_string_path') { + + const base_directory = [...parentPath, 'base_directory'].reduce((acc, key) => acc[key], this.form.resolved) + if (!base_directory) return true // Do not calculate if base is not found + + const entry = { base_directory }; + + if (value.split(".").length > 1) entry.file_path = value; + else entry.folder_path = value; + + const interfaceName = parentPath.slice(-1)[0] + + const results = await run(`locate`, {[interfaceName]: entry}, { swal: false }).catch((e) => { + this.notify(e.message, "error"); + throw e; + }); + + const resolved = [] + + for (let sub in results) { + for (let ses in results[sub]) { + const source_data = results[sub][ses].source_data[interfaceName] + const path = source_data.file_path ?? source_data.folder_path + resolved.push(path) + } + } + + console.log('Metadata Results for', interfaceName, results) + + return [{ + message: html` +

Source Files Found

+ Inspect the Developer Console to preview the metadata results + ${new List({ + items: resolved.map(path => { return { value: path} }), + emptyMessage: 'N/A', + editable: false + })}`, + type: 'info' + }] + + // console.log('Updated format string', value, resolved) + } + } + } })); - const pathExpansionInfoBox = new InfoBox({ - header: "How do I use a Python format string for path expansion?", - content: html`For complete documentation of the path expansion feature of neuroconv, visit the - Path Expansion documentation - page.`, - }); + this.optional.innerHTML = ""; - pathExpansionInfoBox.style.margin = "10px 0px"; + this.optional.style.paddingTop = '10px' - this.optional.innerHTML = ""; this.optional.append(pathExpansionInfoBox, form); form.style.width = "100%"; diff --git a/src/renderer/src/stories/preview/inspector/InspectorList.js b/src/renderer/src/stories/preview/inspector/InspectorList.js index b4fedb670..3d5a0d7b0 100644 --- a/src/renderer/src/stories/preview/inspector/InspectorList.js +++ b/src/renderer/src/stories/preview/inspector/InspectorList.js @@ -67,7 +67,7 @@ export class InspectorListItem extends LitElement { return css` :host { display: block; - background: gainsboro; + background: WhiteSmoke; border: 1px solid gray; border-radius: 10px; padding: 5px 10px; From 6371ecf7e54f465a1fa47a24aa79be7d0af627f9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 Sep 2023 00:51:46 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/CodeBlock.js | 17 +- src/renderer/src/stories/InfoBox.js | 15 +- src/renderer/src/stories/JSONSchemaForm.js | 4 +- src/renderer/src/stories/OptionalSection.js | 6 +- .../guided-mode/data/GuidedPathExpansion.js | 171 ++++++++++-------- 5 files changed, 114 insertions(+), 99 deletions(-) diff --git a/src/renderer/src/stories/CodeBlock.js b/src/renderer/src/stories/CodeBlock.js index db26762c6..6c1f8c73d 100644 --- a/src/renderer/src/stories/CodeBlock.js +++ b/src/renderer/src/stories/CodeBlock.js @@ -1,10 +1,8 @@ import { LitElement, css, html } from "lit"; export class CodeBlock extends LitElement { - static get styles() { return css` - :host { display: block; font-size: 85%; @@ -21,18 +19,17 @@ export class CodeBlock extends LitElement { user-select: text; margin: 0; } - ` + `; } - constructor({ text = '' }){ - super() - this.text = text + constructor({ text = "" }) { + super(); + this.text = text; } - render(){ - return html`
${this.text}
` + render() { + return html`
${this.text}
`; } } -customElements.get("code-block") || - customElements.define("code-block", CodeBlock); +customElements.get("code-block") || customElements.define("code-block", CodeBlock); diff --git a/src/renderer/src/stories/InfoBox.js b/src/renderer/src/stories/InfoBox.js index 3ee6120da..c11b4c54b 100644 --- a/src/renderer/src/stories/InfoBox.js +++ b/src/renderer/src/stories/InfoBox.js @@ -73,7 +73,6 @@ export class InfoBox extends LitElement { margin-bottom: 0px; animation: demo-box-fade-in 0.2s cubic-bezier(0, 0.2, 0.2, 0.96); } - `; } @@ -82,7 +81,7 @@ export class InfoBox extends LitElement { this.header = header; this.content = content; this.type = type; - this.open = open + this.open = open; } updated() { @@ -94,9 +93,9 @@ export class InfoBox extends LitElement { if (this.type === "warning") infoTextElement.insertAdjacentHTML("beforebegin", ` ⚠️`); const infoContainer = infoDropdown.nextElementSibling; - infoDropdown.onclick = () => this.onToggle(!infoContainer.classList.contains("container-open")) + infoDropdown.onclick = () => this.onToggle(!infoContainer.classList.contains("container-open")); - this.onToggle() + this.onToggle(); } onToggle(open = this.open) { @@ -112,10 +111,10 @@ export class InfoBox extends LitElement { infoContainerChevron.direction = "right"; infoContainer.classList.remove("container-open"); } -} + } -render() { - return html` + render() { + return html`
${this.header} ${new Chevron({ direction: "right" })} @@ -124,7 +123,7 @@ render() { ${this.content}
`; -} + } } customElements.get("nwbguide-info-box") || customElements.define("nwbguide-info-box", InfoBox); diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index 7328559ea..1501b6ed8 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -588,9 +588,7 @@ export class JSONSchemaForm extends LitElement { ? valid?.filter((info) => info.type === "error" || (isRequired && info.missing)) : []; - const info = Array.isArray(valid) - ? valid?.filter((info) => info.type === "info" ) - : []; + const info = Array.isArray(valid) ? valid?.filter((info) => info.type === "info") : []; const hasLinks = this.#getLink(externalPath); if (hasLinks) { diff --git a/src/renderer/src/stories/OptionalSection.js b/src/renderer/src/stories/OptionalSection.js index be2ccf524..534ee35f3 100644 --- a/src/renderer/src/stories/OptionalSection.js +++ b/src/renderer/src/stories/OptionalSection.js @@ -5,9 +5,8 @@ import { Button } from "./Button"; export class OptionalSection extends LitElement { static get styles() { return css` - :host { - display: block; + display: block; } h2 { @@ -15,7 +14,8 @@ export class OptionalSection extends LitElement { } .optional-section__toggle { - padding-bottom: 20px; } + padding-bottom: 20px; + } .optional-section__content { text-align: left; diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js index 14358b6c4..47b82510a 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedPathExpansion.js @@ -16,7 +16,6 @@ import { List } from "../../../List"; import { fs } from "../../../../electron/index.js"; import { joinPath } from "../../../../globals.js"; - const exampleFileStructure = `mylab/ ¦ Subjects/ ¦ +-- NR_0017/ @@ -33,84 +32,99 @@ const exampleFileStructure = `mylab/ ¦ ¦ ¦ ¦ ¦ +-- _leftCamera.raw.9041b63e-02e2-480e-aaa7-4f6b776a647f.mp4 ¦ ¦ ¦ ¦ ¦ +-- ... ¦ ¦ ¦ ¦ +-- ... - ¦ ...` - + ¦ ...`; -const exampleFormatPath = "Subjects/{subject_id}/{session_start_time:%Y-%m-%d}/{session_id}/raw_video_data/leftCamera.raw.{}.mp4" +const exampleFormatPath = + "Subjects/{subject_id}/{session_start_time:%Y-%m-%d}/{session_id}/raw_video_data/leftCamera.raw.{}.mp4"; const exampleMetadata = { - "source_data": { - "IBL_video": { - "file_path": "/mylab/Subjects/NR_0017/2022-03-22/001/raw_video_data/leftCamera.raw.6252a2f0-c10f-4e49-b085-75749ba29c35.mp4" - } + source_data: { + IBL_video: { + file_path: + "/mylab/Subjects/NR_0017/2022-03-22/001/raw_video_data/leftCamera.raw.6252a2f0-c10f-4e49-b085-75749ba29c35.mp4", + }, }, - "metadata": { - "NWBFile": { - "session_id": "001", - "session_start_time": "2022-03-22" + metadata: { + NWBFile: { + session_id: "001", + session_start_time: "2022-03-22", }, - "Subject": { - "subject_id": "NR_0017" - } - } -} + Subject: { + subject_id: "NR_0017", + }, + }, +}; const pathExpansionInfoBox = new InfoBox({ header: "How do I use a Python format string for path expansion?", content: html`
-

Consider a dataset of that includes video recordings from three cameras, stored in the following directory structure.

+

+ Consider a dataset of that includes video recordings from three cameras, stored in the following + directory structure. +

${new CodeBlock({ text: exampleFileStructure })} -

Using mylab as the base directory, the correct format string to extract the subject ID, session start time, and session number would be:

- ${new CodeBlock({ text: exampleFormatPath})} +

+ Using mylab as the base directory, the correct format string to extract the subject ID, + session start time, and session number would be: +

+ ${new CodeBlock({ text: exampleFormatPath })} -
+
-

The above example applies all of the supported f-string variables, which are used to extract information into the resulting metadata:

+

+ The above example applies all of the supported f-string variables, which are used to extract information + into the resulting metadata: +

${new List({ items: [ - { - value: 'subject_id' + { + value: "subject_id", }, - { - value: 'session_id' + { + value: "session_id", + }, + { + value: "session_start_time", }, - { - value: 'session_start_time' - } ], - editable: false + editable: false, })}

Additional variables (or blank braces) can be specified to indicate wildcard patterns.

-

Consequently, the metadata extracted from the first file found using this approach would be the following:

- ${new CodeBlock({ text: JSON.stringify(exampleMetadata, null, 2)})} +

+ Consequently, the metadata extracted from the first file found using this approach would be the + following: +

+ ${new CodeBlock({ text: JSON.stringify(exampleMetadata, null, 2) })} + +
-
- - For complete documentation of the path expansion feature of NeuroConv, visit the + For complete documentation of the path expansion feature of NeuroConv, visit the path expansion documentation page. + > + page.
- `, + `, }); pathExpansionInfoBox.style.margin = "10px 0px"; function getFiles(dir) { - const dirents = fs.readdirSync(dir, { withFileTypes: true }); - let entries = [] - for (const dirent of dirents) { - const res = joinPath(dir, dirent.name); - if (dirent.isDirectory()) entries.push(...getFiles(res)); - else entries.push(res) - } - - return entries + const dirents = fs.readdirSync(dir, { withFileTypes: true }); + let entries = []; + for (const dirent of dirents) { + const res = joinPath(dir, dirent.name); + if (dirent.isDirectory()) entries.push(...getFiles(res)); + else entries.push(res); + } + + return entries; } export class GuidedPathExpansionPage extends Page { @@ -253,7 +267,7 @@ export class GuidedPathExpansionPage extends Page { const state = structureState.state; if (state !== undefined) this.optional.state = state; - else pathExpansionInfoBox.open = true // Open the info box if no option has been selected + else pathExpansionInfoBox.open = true; // Open the info box if no option has been selected // Require properties for all sources const generatedSchema = { type: "object", properties: {} }; @@ -267,7 +281,7 @@ export class GuidedPathExpansionPage extends Page { ...structureState, onThrow, validateOnChange: async (name, parent, parentPath) => { - const value = parent[name] + const value = parent[name]; if (fs) { // if (name === 'base_directory') { // for (const f of getFiles(value)) { @@ -275,57 +289,64 @@ export class GuidedPathExpansionPage extends Page { // } // const res = getFiles(value); // } - // else - if (name === 'format_string_path') { - - const base_directory = [...parentPath, 'base_directory'].reduce((acc, key) => acc[key], this.form.resolved) - if (!base_directory) return true // Do not calculate if base is not found + // else + if (name === "format_string_path") { + const base_directory = [...parentPath, "base_directory"].reduce( + (acc, key) => acc[key], + this.form.resolved + ); + if (!base_directory) return true; // Do not calculate if base is not found const entry = { base_directory }; if (value.split(".").length > 1) entry.file_path = value; else entry.folder_path = value; - const interfaceName = parentPath.slice(-1)[0] + const interfaceName = parentPath.slice(-1)[0]; - const results = await run(`locate`, {[interfaceName]: entry}, { swal: false }).catch((e) => { + const results = await run(`locate`, { [interfaceName]: entry }, { swal: false }).catch((e) => { this.notify(e.message, "error"); throw e; }); - const resolved = [] + const resolved = []; for (let sub in results) { for (let ses in results[sub]) { - const source_data = results[sub][ses].source_data[interfaceName] - const path = source_data.file_path ?? source_data.folder_path - resolved.push(path) + const source_data = results[sub][ses].source_data[interfaceName]; + const path = source_data.file_path ?? source_data.folder_path; + resolved.push(path); } } - console.log('Metadata Results for', interfaceName, results) - - return [{ - message: html` -

Source Files Found

- Inspect the Developer Console to preview the metadata results - ${new List({ - items: resolved.map(path => { return { value: path} }), - emptyMessage: 'N/A', - editable: false - })}`, - type: 'info' - }] - + console.log("Metadata Results for", interfaceName, results); + + return [ + { + message: html`

Source Files Found

+ Inspect the Developer Console to preview the metadata results + ${new List({ + items: resolved.map((path) => { + return { value: path }; + }), + emptyMessage: "N/A", + editable: false, + })}`, + type: "info", + }, + ]; + // console.log('Updated format string', value, resolved) } } - } + }, })); this.optional.innerHTML = ""; - this.optional.style.paddingTop = '10px' + this.optional.style.paddingTop = "10px"; this.optional.append(pathExpansionInfoBox, form); From 2fd1660c3506efb58733b384f906febc0d6530ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 21:55:25 +0000 Subject: [PATCH 4/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/neuroconv.py | 9 +++++---- pyflask/manageNeuroconv/manage_neuroconv.py | 15 ++++++++------- schemas/json/dandi/standalone.json | 2 +- .../src/stories/pages/uploads/UploadsPage.js | 7 ++++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 3935cb803..03fa85220 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -18,7 +18,7 @@ inspect_multiple_filesystem_objects, upload_to_dandi, upload_folder_to_dandi, - upload_multiple_filesystem_objects_to_dandi + upload_multiple_filesystem_objects_to_dandi, ) from errorHandlers import notBadRequestException @@ -148,8 +148,8 @@ def post(self): try: paths = neuroconv_api.payload["filesystem_paths"] - if (len(paths) == 1 and isdir(paths[0])): - kwargs = { **neuroconv_api.payload } + if len(paths) == 1 and isdir(paths[0]): + kwargs = {**neuroconv_api.payload} del kwargs["filesystem_paths"] kwargs["nwb_folder_path"] = paths[0] return upload_folder_to_dandi(**kwargs) @@ -160,7 +160,8 @@ def post(self): except Exception as e: if notBadRequestException(e): neuroconv_api.abort(500, str(e)) - + + @neuroconv_api.route("/inspect_file") class InspectNWBFile(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 83b79a253..cf56206e8 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -410,13 +410,14 @@ def update_conversion_progress(**kwargs): def upload_multiple_filesystem_objects_to_dandi(**kwargs): - tmp_folder_path = aggregate_in_temp_directory(kwargs['filesystem_paths'], 'upload') - innerKwargs = { **kwargs } - del innerKwargs['filesystem_paths'] - innerKwargs["nwb_folder_path"] = tmp_folder_path - result = upload_folder_to_dandi(**innerKwargs) - rmtree(tmp_folder_path) - return result + tmp_folder_path = aggregate_in_temp_directory(kwargs["filesystem_paths"], "upload") + innerKwargs = {**kwargs} + del innerKwargs["filesystem_paths"] + innerKwargs["nwb_folder_path"] = tmp_folder_path + result = upload_folder_to_dandi(**innerKwargs) + rmtree(tmp_folder_path) + return result + def upload_folder_to_dandi( dandiset_id: str, diff --git a/schemas/json/dandi/standalone.json b/schemas/json/dandi/standalone.json index dfa6369d2..fa210901f 100644 --- a/schemas/json/dandi/standalone.json +++ b/schemas/json/dandi/standalone.json @@ -8,5 +8,5 @@ } } }, - "required": ["filesystem_paths"] + "required": ["filesystem_paths"] } diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index 9f861bddd..fc759aa58 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -21,8 +21,7 @@ import { DandiResults } from "../../DandiResults.js"; export const isStaging = (id) => parseInt(id) >= 100000; export async function uploadToDandi(info, type = "project" in info ? "" : "folder") { - - const { dandiset_id } = info + const { dandiset_id } = info; const staging = isStaging(dandiset_id); // Automatically detect staging IDs @@ -59,7 +58,9 @@ export async function uploadToDandi(info, type = "project" in info ? "" : "folde if (result) notyf.open({ type: "success", - message: `${info.project ?? `${info[folderPathKey].length} filesystem entries`} successfully uploaded to Dandiset ${dandiset_id}`, + message: `${ + info.project ?? `${info[folderPathKey].length} filesystem entries` + } successfully uploaded to Dandiset ${dandiset_id}`, }); return result; From 3371b7f4e40003e9fcd60194f48fb8ee31a76f1a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:07:33 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/progress/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index b30fb5eb7..14bc2a2f1 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -42,7 +42,7 @@ function decode(message) { function drill(o, callback) { if (o && typeof o === "object") { - const copy = Array.isArray(o) ? [...o] : { ...o } ; + const copy = Array.isArray(o) ? [...o] : { ...o }; for (let k in copy) copy[k] = drill(copy[k], callback); return copy; } else return callback(o); From 6363a38a27d3e5def2aead442784463fffa8e9b7 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 28 Sep 2023 15:08:48 -0700 Subject: [PATCH 6/8] Fix merge and update function name --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- src/renderer/src/stories/pages/uploads/UploadsPage.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index cf56206e8..c562f5543 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -410,7 +410,7 @@ def update_conversion_progress(**kwargs): def upload_multiple_filesystem_objects_to_dandi(**kwargs): - tmp_folder_path = aggregate_in_temp_directory(kwargs["filesystem_paths"], "upload") + tmp_folder_path = aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") innerKwargs = {**kwargs} del innerKwargs["filesystem_paths"] innerKwargs["nwb_folder_path"] = tmp_folder_path diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index fc759aa58..d50259348 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -20,7 +20,7 @@ import { DandiResults } from "../../DandiResults.js"; export const isStaging = (id) => parseInt(id) >= 100000; -export async function uploadToDandi(info, type = "project" in info ? "" : "folder") { +export async function uploadToDandi(info, type = "project" in info ? "project" : "") { const { dandiset_id } = info; const staging = isStaging(dandiset_id); // Automatically detect staging IDs From 4fb1ba3ccdd2dcf0268a66d07c6ab4a061e91461 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:15:23 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/main/main.ts | 2 +- src/renderer/src/progress/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/main.ts b/src/main/main.ts index 0f78f25c1..f15f363fb 100755 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -184,7 +184,7 @@ const exitPyProc = async () => { const killAllPreviousProcesses = async () => { const fetch = globalThis.fetch - + if (fetch){ console.log("Killing all previous processes"); diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index b30fb5eb7..14bc2a2f1 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -42,7 +42,7 @@ function decode(message) { function drill(o, callback) { if (o && typeof o === "object") { - const copy = Array.isArray(o) ? [...o] : { ...o } ; + const copy = Array.isArray(o) ? [...o] : { ...o }; for (let k in copy) copy[k] = drill(copy[k], callback); return copy; } else return callback(o); From 7ae92a9da2d8b84cb05e0062c2579cf2dae62224 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 2 Oct 2023 07:05:11 -0400 Subject: [PATCH 8/8] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b83666582..e2f0131a3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "nwb-guide", "productName": "NWB GUIDE", - "version": "0.0.6", + "version": "0.0.7", "description": "", "main": "./build/main/main.js", "engine": {