From b8c2691018b94426fae079a58ddce4f762242a77 Mon Sep 17 00:00:00 2001 From: Garrett Date: Fri, 15 Sep 2023 09:45:30 -0700 Subject: [PATCH 1/4] Test multi-upload for all conditions (fails on symlinked nested folders) --- pyflask/apis/neuroconv.py | 33 +++++++++++++++---- pyflask/manageNeuroconv/__init__.py | 1 + pyflask/manageNeuroconv/manage_neuroconv.py | 9 +++++ schemas/json/dandi/standalone.json | 11 ++++--- .../src/stories/FileSystemSelector.js | 4 +++ .../src/stories/pages/uploads/UploadsPage.js | 13 +++++--- 6 files changed, 56 insertions(+), 15 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 0db136328..0f3020720 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -15,7 +15,10 @@ generate_dataset, inspect_nwb_file, inspect_nwb_folder, - inspect_multiple_filesystem_objects + inspect_multiple_filesystem_objects, + upload_to_dandi, + upload_folder_to_dandi, + upload_multiple_filesystem_objects_to_dandi ) from errorHandlers import notBadRequestException @@ -112,13 +115,11 @@ def post(self): neuroconv_api.abort(500, str(e)) -@neuroconv_api.route("/upload") +@neuroconv_api.route("/upload/project") class Upload(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - from manageNeuroconv import upload_to_dandi - return upload_to_dandi(**neuroconv_api.payload) except Exception as e: @@ -131,8 +132,6 @@ class Upload(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - from manageNeuroconv import upload_folder_to_dandi - return upload_folder_to_dandi(**neuroconv_api.payload) except Exception as e: @@ -140,6 +139,28 @@ def post(self): neuroconv_api.abort(500, str(e)) +@neuroconv_api.route("/upload") +class Upload(Resource): + @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def post(self): + from os.path import isdir + + try: + paths = neuroconv_api.payload["filesystem_paths"] + + if (len(paths) == 1 and isdir(paths[0])): + kwargs = { **neuroconv_api.payload } + del kwargs["filesystem_paths"] + kwargs["nwb_folder_path"] = paths[0] + return upload_folder_to_dandi(**kwargs) + + else: + return upload_multiple_filesystem_objects_to_dandi(**neuroconv_api.payload) + + except Exception as e: + if notBadRequestException(e): + neuroconv_api.abort(500, str(e)) + @neuroconv_api.route("/inspect_file") class InspectNWBFile(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index 8adea2add..db610e516 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -7,6 +7,7 @@ validate_metadata, upload_to_dandi, upload_folder_to_dandi, + upload_multiple_filesystem_objects_to_dandi, listen_to_neuroconv_events, generate_dataset, inspect_nwb_file, diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index b3da52419..2569b6b02 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -330,6 +330,15 @@ def update_conversion_progress(**kwargs): return dict(file=str(resolved_output_path)) +def upload_multiple_filesystem_objects_to_dandi(**kwargs): + tmp_folder_path = aggregate_in_temp_directory(kwargs['filesystem_paths'], 'upload') + innerKwargs = { **kwargs } + del innerKwargs['filesystem_paths'] + innerKwargs["nwb_folder_path"] = tmp_folder_path + result = upload_folder_to_dandi(**innerKwargs) + rmtree(tmp_folder_path) + return result + def upload_folder_to_dandi( dandiset_id: str, api_key: str, diff --git a/schemas/json/dandi/standalone.json b/schemas/json/dandi/standalone.json index e56062017..dfa6369d2 100644 --- a/schemas/json/dandi/standalone.json +++ b/schemas/json/dandi/standalone.json @@ -1,9 +1,12 @@ { "properties": { - "nwb_folder_path": { - "type": "string", - "format": "directory" + "filesystem_paths": { + "type": "array", + "items":{ + "type": "string", + "format": ["file", "directory"] + } } }, - "required": ["nwb_folder_path"] + "required": ["filesystem_paths"] } diff --git a/src/renderer/src/stories/FileSystemSelector.js b/src/renderer/src/stories/FileSystemSelector.js index fdf71b649..a6702ad8a 100644 --- a/src/renderer/src/stories/FileSystemSelector.js +++ b/src/renderer/src/stories/FileSystemSelector.js @@ -39,6 +39,10 @@ const componentCSS = css` gap: 5px; } + #button-div > nwb-button { + margin-bottom: 10px; + } + button { background: WhiteSmoke; border: 1px solid #c3c3c3; diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index 2e4691212..f464563c8 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -2,6 +2,8 @@ import { html } from "lit"; import { JSONSchemaForm } from "../../JSONSchemaForm.js"; import { Page } from "../Page.js"; import { onThrow } from "../../../errors"; + +const folderPathKey = "filesystem_paths"; import dandiUploadSchema from "../../../../../../schemas/json/dandi/upload.json"; import dandiStandaloneSchema from "../../../../../../schemas/json/dandi/standalone.json"; const dandiSchema = merge(dandiStandaloneSchema, merge(dandiUploadSchema, {}), { arrays: true }); @@ -18,7 +20,7 @@ import { DandiResults } from "../../DandiResults.js"; export const isStaging = (id) => parseInt(id) >= 100000; -export async function uploadToDandi(info, type = "project" in info ? "" : "folder") { +export async function uploadToDandi(info, type = "project" in info ? "project" : "") { const api_key = global.data.DANDI?.api_key; if (!api_key) { await Swal.fire({ @@ -31,11 +33,13 @@ export async function uploadToDandi(info, type = "project" in info ? "" : "folde return this.to("settings"); } + const { dandiset_id } = info + const result = await run( type ? `upload/${type}` : "upload", { ...info, - staging: isStaging(info.dandiset_id), // Automatically detect staging IDs + staging: isStaging(dandiset_id), // Automatically detect staging IDs api_key, }, { title: "Uploading to DANDI" } @@ -50,7 +54,7 @@ export async function uploadToDandi(info, type = "project" in info ? "" : "folde if (result) notyf.open({ type: "success", - message: `${info.project ?? info.nwb_folder_path} successfully uploaded to Dandiset ${info.dandiset_id}`, + message: `${info.project ?? `${info[folderPathKey].length} filesystem entries`} successfully uploaded to Dandiset ${dandiset_id}`, }); return result; @@ -85,13 +89,12 @@ export class UploadsPage extends Page { }, }); - const folderPathKey = "nwb_folder_path"; // NOTE: API Keys and Dandiset IDs persist across selected project this.form = new JSONSchemaForm({ results: globalState, schema: dandiSchema, sort: ([k1]) => { - if (k1 === "nwb_folder_path") return -1; + if (k1 === folderPathKey) return -1; }, onUpdate: ([id]) => { if (id === folderPathKey) { From 2fd1660c3506efb58733b384f906febc0d6530ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 21:55:25 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/neuroconv.py | 9 +++++---- pyflask/manageNeuroconv/manage_neuroconv.py | 15 ++++++++------- schemas/json/dandi/standalone.json | 2 +- .../src/stories/pages/uploads/UploadsPage.js | 7 ++++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 3935cb803..03fa85220 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -18,7 +18,7 @@ inspect_multiple_filesystem_objects, upload_to_dandi, upload_folder_to_dandi, - upload_multiple_filesystem_objects_to_dandi + upload_multiple_filesystem_objects_to_dandi, ) from errorHandlers import notBadRequestException @@ -148,8 +148,8 @@ def post(self): try: paths = neuroconv_api.payload["filesystem_paths"] - if (len(paths) == 1 and isdir(paths[0])): - kwargs = { **neuroconv_api.payload } + if len(paths) == 1 and isdir(paths[0]): + kwargs = {**neuroconv_api.payload} del kwargs["filesystem_paths"] kwargs["nwb_folder_path"] = paths[0] return upload_folder_to_dandi(**kwargs) @@ -160,7 +160,8 @@ def post(self): except Exception as e: if notBadRequestException(e): neuroconv_api.abort(500, str(e)) - + + @neuroconv_api.route("/inspect_file") class InspectNWBFile(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 83b79a253..cf56206e8 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -410,13 +410,14 @@ def update_conversion_progress(**kwargs): def upload_multiple_filesystem_objects_to_dandi(**kwargs): - tmp_folder_path = aggregate_in_temp_directory(kwargs['filesystem_paths'], 'upload') - innerKwargs = { **kwargs } - del innerKwargs['filesystem_paths'] - innerKwargs["nwb_folder_path"] = tmp_folder_path - result = upload_folder_to_dandi(**innerKwargs) - rmtree(tmp_folder_path) - return result + tmp_folder_path = aggregate_in_temp_directory(kwargs["filesystem_paths"], "upload") + innerKwargs = {**kwargs} + del innerKwargs["filesystem_paths"] + innerKwargs["nwb_folder_path"] = tmp_folder_path + result = upload_folder_to_dandi(**innerKwargs) + rmtree(tmp_folder_path) + return result + def upload_folder_to_dandi( dandiset_id: str, diff --git a/schemas/json/dandi/standalone.json b/schemas/json/dandi/standalone.json index dfa6369d2..fa210901f 100644 --- a/schemas/json/dandi/standalone.json +++ b/schemas/json/dandi/standalone.json @@ -8,5 +8,5 @@ } } }, - "required": ["filesystem_paths"] + "required": ["filesystem_paths"] } diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index 9f861bddd..fc759aa58 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -21,8 +21,7 @@ import { DandiResults } from "../../DandiResults.js"; export const isStaging = (id) => parseInt(id) >= 100000; export async function uploadToDandi(info, type = "project" in info ? "" : "folder") { - - const { dandiset_id } = info + const { dandiset_id } = info; const staging = isStaging(dandiset_id); // Automatically detect staging IDs @@ -59,7 +58,9 @@ export async function uploadToDandi(info, type = "project" in info ? "" : "folde if (result) notyf.open({ type: "success", - message: `${info.project ?? `${info[folderPathKey].length} filesystem entries`} successfully uploaded to Dandiset ${dandiset_id}`, + message: `${ + info.project ?? `${info[folderPathKey].length} filesystem entries` + } successfully uploaded to Dandiset ${dandiset_id}`, }); return result; From 3371b7f4e40003e9fcd60194f48fb8ee31a76f1a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 22:07:33 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/progress/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/src/progress/index.js b/src/renderer/src/progress/index.js index b30fb5eb7..14bc2a2f1 100644 --- a/src/renderer/src/progress/index.js +++ b/src/renderer/src/progress/index.js @@ -42,7 +42,7 @@ function decode(message) { function drill(o, callback) { if (o && typeof o === "object") { - const copy = Array.isArray(o) ? [...o] : { ...o } ; + const copy = Array.isArray(o) ? [...o] : { ...o }; for (let k in copy) copy[k] = drill(copy[k], callback); return copy; } else return callback(o); From 6363a38a27d3e5def2aead442784463fffa8e9b7 Mon Sep 17 00:00:00 2001 From: Garrett Date: Thu, 28 Sep 2023 15:08:48 -0700 Subject: [PATCH 4/4] Fix merge and update function name --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- src/renderer/src/stories/pages/uploads/UploadsPage.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index cf56206e8..c562f5543 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -410,7 +410,7 @@ def update_conversion_progress(**kwargs): def upload_multiple_filesystem_objects_to_dandi(**kwargs): - tmp_folder_path = aggregate_in_temp_directory(kwargs["filesystem_paths"], "upload") + tmp_folder_path = aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") innerKwargs = {**kwargs} del innerKwargs["filesystem_paths"] innerKwargs["nwb_folder_path"] = tmp_folder_path diff --git a/src/renderer/src/stories/pages/uploads/UploadsPage.js b/src/renderer/src/stories/pages/uploads/UploadsPage.js index fc759aa58..d50259348 100644 --- a/src/renderer/src/stories/pages/uploads/UploadsPage.js +++ b/src/renderer/src/stories/pages/uploads/UploadsPage.js @@ -20,7 +20,7 @@ import { DandiResults } from "../../DandiResults.js"; export const isStaging = (id) => parseInt(id) >= 100000; -export async function uploadToDandi(info, type = "project" in info ? "" : "folder") { +export async function uploadToDandi(info, type = "project" in info ? "project" : "") { const { dandiset_id } = info; const staging = isStaging(dandiset_id); // Automatically detect staging IDs