Skip to content

Commit

Permalink
Merge pull request #371 from NeurodataWithoutBorders/inspector-files-…
Browse files Browse the repository at this point in the history
…or-folders

Inspect Multiple Files and Folders
  • Loading branch information
CodyCBakerPhD authored Oct 2, 2023
2 parents 7853b81 + 07a7dfb commit affd9c4
Show file tree
Hide file tree
Showing 8 changed files with 171 additions and 63 deletions.
66 changes: 29 additions & 37 deletions pyflask/apis/neuroconv.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@
validate_metadata,
listen_to_neuroconv_events,
generate_dataset,
inspect_nwb_file,
inspect_nwb_folder,
inspect_multiple_filesystem_objects,
)

from errorHandlers import notBadRequestException

neuroconv_api = Namespace("neuroconv", description="Neuroconv neuroconv_api for the NWB GUIDE.")
Expand Down Expand Up @@ -141,25 +145,7 @@ class InspectNWBFile(Resource):
@neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
def post(self):
try:
import json
from nwbinspector import inspect_nwbfile
from nwbinspector.nwbinspector import InspectorOutputJSONEncoder

return json.loads(
json.dumps(
list(
inspect_nwbfile(
ignore=[
"check_description",
"check_data_orientation",
], # TODO: remove when metadata control is exposed
**neuroconv_api.payload,
)
),
cls=InspectorOutputJSONEncoder,
)
)

return inspect_nwb_file(neuroconv_api.payload)
except Exception as e:
if notBadRequestException(e):
neuroconv_api.abort(500, str(e))
Expand All @@ -170,24 +156,30 @@ class InspectNWBFolder(Resource):
@neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
def post(self):
try:
import json
from nwbinspector import inspect_all
from nwbinspector.nwbinspector import InspectorOutputJSONEncoder

messages = list(
inspect_all(
n_jobs=-2, # uses number of CPU - 1
ignore=[
"check_description",
"check_data_orientation",
], # TODO: remove when metadata control is exposed
**neuroconv_api.payload,
)
)

# messages = organize_messages(messages, levels=["importance", "message"])

return json.loads(json.dumps(messages, cls=InspectorOutputJSONEncoder))
return inspect_nwb_folder(neuroconv_api.payload)

except Exception as e:
if notBadRequestException(e):
neuroconv_api.abort(500, str(e))


@neuroconv_api.route("/inspect")
class InspectNWBFolder(Resource):
@neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
def post(self):
from os.path import isfile

try:
paths = neuroconv_api.payload["paths"]

if len(paths) == 1:
if isfile(paths[0]):
return inspect_nwb_file({"path": paths[0]})
else:
return inspect_nwb_folder({"path": paths[0]})

else:
return inspect_multiple_filesystem_objects(paths)

except Exception as e:
if notBadRequestException(e):
Expand Down
3 changes: 3 additions & 0 deletions pyflask/manageNeuroconv/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
upload_folder_to_dandi,
listen_to_neuroconv_events,
generate_dataset,
inspect_nwb_file,
inspect_nwb_folder,
inspect_multiple_filesystem_objects,
)


Expand Down
8 changes: 7 additions & 1 deletion pyflask/manageNeuroconv/info/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,7 @@
from .urls import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH
from .urls import (
resource_path,
GUIDE_ROOT_FOLDER,
STUB_SAVE_FOLDER_PATH,
CONVERSION_SAVE_FOLDER_PATH,
TUTORIAL_SAVE_FOLDER_PATH,
)
1 change: 1 addition & 0 deletions pyflask/manageNeuroconv/info/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ def resource_path(relative_path):
) # NOTE: Must have pyflask for running the GUIDE as a whole, but errors for just the server
f = path_config.open()
data = json.load(f)
GUIDE_ROOT_FOLDER = Path(Path.home(), data["root"])
STUB_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["preview"])
CONVERSION_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["conversions"])
TUTORIAL_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["tutorial"])
Expand Down
88 changes: 87 additions & 1 deletion pyflask/manageNeuroconv/manage_neuroconv.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from pathlib import Path

from sse import MessageAnnouncer
from .info import STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH
from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH

announcer = MessageAnnouncer()

Expand Down Expand Up @@ -507,3 +507,89 @@ def generate_dataset(test_data_directory_path: str):
phy_output_dir.symlink_to(phy_base_directory, True)

return {"output_directory": str(output_directory)}


def inspect_nwb_file(payload):
from nwbinspector import inspect_nwbfile
from nwbinspector.nwbinspector import InspectorOutputJSONEncoder

return json.loads(
json.dumps(
list(
inspect_nwbfile(
ignore=[
"check_description",
"check_data_orientation",
], # TODO: remove when metadata control is exposed
**payload,
)
),
cls=InspectorOutputJSONEncoder,
)
)


def inspect_nwb_file(payload):
from nwbinspector import inspect_nwbfile
from nwbinspector.nwbinspector import InspectorOutputJSONEncoder

return json.loads(
json.dumps(
list(
inspect_nwbfile(
ignore=[
"check_description",
"check_data_orientation",
], # TODO: remove when metadata control is exposed
**payload,
)
),
cls=InspectorOutputJSONEncoder,
)
)


def inspect_nwb_folder(payload):
from nwbinspector import inspect_all
from nwbinspector.nwbinspector import InspectorOutputJSONEncoder

messages = list(
inspect_all(
n_jobs=-2, # uses number of CPU - 1
ignore=[
"check_description",
"check_data_orientation",
], # TODO: remove when metadata control is exposed
**payload,
)
)

# messages = organize_messages(messages, levels=["importance", "message"])

return json.loads(json.dumps(messages, cls=InspectorOutputJSONEncoder))


def aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None):
if folder_path is None:
folder_path = GUIDE_ROOT_FOLDER / ".temp" / reason / f"temp_{datetime.now().strftime('%Y%m%d-%H%M%S')}"

folder_path.mkdir(parents=True)

for path in paths:
path = Path(path)
new_path = folder_path / path.name
if path.is_dir():
aggregate_symlinks_in_new_directory(
list(map(lambda name: os.path.join(path, name), os.listdir(path))), None, new_path
)
else:
new_path.symlink_to(path, path.is_dir())

return folder_path


def inspect_multiple_filesystem_objects(paths):
tmp_folder_path = aggregate_symlinks_in_new_directory(paths, "inspect")
result = inspect_nwb_folder({"path": tmp_folder_path})
rmtree(tmp_folder_path)
return result
23 changes: 16 additions & 7 deletions src/renderer/src/stories/FileSystemSelector.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ function getObjectTypeReferenceString(type, multiple, { nested, native } = {}) {
.join(" / ")}`;

const isDir = type === "directory";
return multiple && (!isDir || (isDir && !native))
return multiple && (!isDir || (isDir && !native) || dialog)
? type === "directory"
? "directories"
: "files"
Expand Down Expand Up @@ -122,7 +122,10 @@ export class FilesystemSelector extends LitElement {
};

#onCancel = () => {
this.#onThrow(`No ${this.type} selected`, "The request was cancelled by the user");
this.#onThrow(
`No ${getObjectTypeReferenceString(this.type, this.multiple, { native: true })} selected`,
"The request was cancelled by the user"
);
};

#checkType = (value) => {
Expand All @@ -133,12 +136,18 @@ export class FilesystemSelector extends LitElement {

#handleFiles = async (pathOrPaths, type) => {
if (!pathOrPaths)
this.#onThrow("No paths detected", `Unable to parse ${this.type} path${this.multiple ? "s" : ""}`);
this.#onThrow(
"No paths detected",
`Unable to parse ${getObjectTypeReferenceString(this.type, false, { native: true })} path${
this.multiple ? "s" : ""
}`
);

if (Array.isArray(pathOrPaths)) pathOrPaths.forEach(this.#checkType);
else if (!type) this.#checkType(pathOrPaths);

let resolvedValue = pathOrPaths;

if (Array.isArray(resolvedValue) && !this.multiple) {
if (resolvedValue.length > 1)
this.#onThrow(
Expand All @@ -158,9 +167,9 @@ export class FilesystemSelector extends LitElement {

async selectFormat(type = this.type) {
if (dialog) {
const file = await this.#useElectronDialog(type);
const path = file.filePath ?? file.filePaths?.[0];
this.#handleFiles(path, type);
const results = await this.#useElectronDialog(type);
// const path = file.filePath ?? file.filePaths?.[0];
this.#handleFiles(results.filePath ?? results.filePaths, type);
} else {
let handles = await (type === "directory"
? window.showDirectoryPicker()
Expand Down Expand Up @@ -236,7 +245,7 @@ export class FilesystemSelector extends LitElement {
native: true,
})}`}</span
>${this.multiple &&
(this.type === "directory" || (isMultipleTypes && this.type.includes("directory")))
(this.type === "directory" || (isMultipleTypes && this.type.includes("directory") && !dialog))
? html`<br /><small
>Multiple directory support only available using drag-and-drop.</small
>`
Expand Down
9 changes: 6 additions & 3 deletions src/renderer/src/stories/JSONSchemaInput.js
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ export class JSONSchemaInput extends LitElement {
`;
}

#onThrow = (...args) => (this.onThrow ? this.onThrow(...args) : this.form?.onThrow(...args));

#render() {
const { validateOnChange, info, path: fullPath } = this;

Expand All @@ -158,7 +160,8 @@ export class JSONSchemaInput extends LitElement {
const isArray = info.type === "array"; // Handle string (and related) formats / types

const hasItemsRef = "items" in info && "$ref" in info.items;
if (!("items" in info) || (!("type" in info.items) && !hasItemsRef)) info.items = { type: "string" };
if (!("items" in info)) info.items = {};
if (!("type" in info.items) && !hasItemsRef) info.items.type = "string";

// Handle file and directory formats
const createFilesystemSelector = (format) => {
Expand All @@ -167,7 +170,7 @@ export class JSONSchemaInput extends LitElement {
value: this.value,
onSelect: (filePath) => this.#updateData(fullPath, filePath),
onChange: (filePath) => validateOnChange && this.#triggerValidation(name, el, path),
onThrow: (...args) => this.form?.onThrow(...args),
onThrow: (...args) => this.#onThrow(...args),
dialogOptions: this.form?.dialogOptions,
dialogType: this.form?.dialogType,
multiple: isArray,
Expand Down Expand Up @@ -211,7 +214,7 @@ export class JSONSchemaInput extends LitElement {
this.form.checkAllLoaded();
}
},
onThrow: (...args) => this.form?.onThrow(...args),
onThrow: (...args) => this.#onThrow(...args),
};

return (this.form.tables[name] =
Expand Down
36 changes: 22 additions & 14 deletions src/renderer/src/stories/pages/inspect/InspectPage.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { Button } from "../../Button.js";
import { run } from "../guided-mode/options/utils.js";
import { JSONSchemaInput } from "../../JSONSchemaInput.js";
import { Modal } from "../../Modal";
import { truncateFilePaths } from "../../preview/NWBFilePreview.js";
import { getSharedPath, truncateFilePaths } from "../../preview/NWBFilePreview.js";
import { InspectorList } from "../../preview/inspector/InspectorList.js";

export class InspectPage extends Page {
Expand All @@ -16,24 +16,29 @@ export class InspectPage extends Page {

showReport = async (value) => {
if (!value) {
const message = "Please provide a folder to inspect.";
const message = "Please provide filesystem entries to inspect.";
onThrow(message);
throw new Error(message);
}

const items = truncateFilePaths(
await run("inspect_folder", { path: value }, { title: "Inspecting your files" }).catch((e) => {
this.notify(e.message, "error");
throw e;
}),
value
);
const result = await run(
"inspect",
{ paths: value },
{ title: "Inspecting selected filesystem entries." }
).catch((e) => {
this.notify(e.message, "error");
throw e;
});

if (!result.length) return this.notify("No messages received from the NWB Inspector");

const items = truncateFilePaths(result, getSharedPath(result.map((o) => o.file_path)));

const list = new InspectorList({ items });
list.style.padding = "25px";

const modal = new Modal({
header: value,
header: value.length === 1 ? value : `Selected Filesystem Entries`,
});
modal.append(list);
document.body.append(modal);
Expand All @@ -42,17 +47,20 @@ export class InspectPage extends Page {
};

input = new JSONSchemaInput({
path: ["folder_path"],
path: ["filesystem_paths"],
info: {
type: "string",
format: "directory",
type: "array",
items: {
format: ["file", "directory"],
multiple: true,
},
},
onThrow,
});

render() {
const button = new Button({
label: "Inspect Files",
label: "Start Inspection",
onClick: async () => this.showReport(this.input.value),
});

Expand Down

0 comments on commit affd9c4

Please sign in to comment.