From 52119c4619cde2167684651e5c7548da6f62a485 Mon Sep 17 00:00:00 2001 From: Garrett Date: Tue, 31 Oct 2023 09:46:58 -0700 Subject: [PATCH 1/5] Fallback to n_jobs=1 if NWB Inspector fails --- pyflask/manageNeuroconv/manage_neuroconv.py | 24 ++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 89a89544f..3dc1cd03c 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -593,18 +593,22 @@ def inspect_nwb_folder(payload): from nwbinspector import inspect_all, load_config from nwbinspector.nwbinspector import InspectorOutputJSONEncoder - messages = list( - inspect_all( - n_jobs=-2, # uses number of CPU - 1 - ignore=[ - "check_description", - "check_data_orientation", - ], # TODO: remove when metadata control is exposed - config=load_config(filepath_or_keyword="dandi"), - **payload, - ) + kwargs = dict( + n_jobs=-2, # uses number of CPU - 1 + ignore=[ + "check_description", + "check_data_orientation", + ], # TODO: remove when metadata control is exposed + config=load_config(filepath_or_keyword="dandi"), + **payload, ) + try: + messages = list(inspect_all(**kwargs)) + except: + del kwargs['n_jobs'] + messages = list(inspect_all(**kwargs)) + return json.loads(json.dumps(obj=messages, cls=InspectorOutputJSONEncoder)) From 76f98d58f86899be75950a122c7627d691ff2f6e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:48:27 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 3dc1cd03c..199c2b2e5 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -606,7 +606,7 @@ def inspect_nwb_folder(payload): try: messages = list(inspect_all(**kwargs)) except: - del kwargs['n_jobs'] + del kwargs["n_jobs"] messages = list(inspect_all(**kwargs)) return json.loads(json.dumps(obj=messages, cls=InspectorOutputJSONEncoder)) From 981fdbcc0963381e3ff96886f92aab304d6008c6 Mon Sep 17 00:00:00 2001 From: Garrett Date: Tue, 31 Oct 2023 10:38:32 -0700 Subject: [PATCH 3/5] Update manage_neuroconv.py --- pyflask/manageNeuroconv/manage_neuroconv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 3dc1cd03c..f7bd20a36 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -592,6 +592,7 @@ def inspect_nwb_file(payload): def inspect_nwb_folder(payload): from nwbinspector import inspect_all, load_config from nwbinspector.nwbinspector import InspectorOutputJSONEncoder + from pickle import PicklingError kwargs = dict( n_jobs=-2, # uses number of CPU - 1 @@ -605,9 +606,11 @@ def inspect_nwb_folder(payload): try: messages = list(inspect_all(**kwargs)) - except: + except PicklingError as e: del kwargs['n_jobs'] messages = list(inspect_all(**kwargs)) + except Exception as e: + raise e return json.loads(json.dumps(obj=messages, cls=InspectorOutputJSONEncoder)) From 4c64e020ca5ff4447d703aab2d8d5e3d84ce8b58 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Oct 2023 17:40:16 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index f7bd20a36..db798cc96 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -607,7 +607,7 @@ def inspect_nwb_folder(payload): try: messages = list(inspect_all(**kwargs)) except PicklingError as e: - del kwargs['n_jobs'] + del kwargs["n_jobs"] messages = list(inspect_all(**kwargs)) except Exception as e: raise e From 007874a00ce23a6435c3d2679de6deeacdd933a9 Mon Sep 17 00:00:00 2001 From: Garrett Date: Tue, 31 Oct 2023 11:06:14 -0700 Subject: [PATCH 5/5] Check against known error message --- pyflask/manageNeuroconv/manage_neuroconv.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index db798cc96..ff07cb8b6 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -607,8 +607,11 @@ def inspect_nwb_folder(payload): try: messages = list(inspect_all(**kwargs)) except PicklingError as e: - del kwargs["n_jobs"] - messages = list(inspect_all(**kwargs)) + if "attribute lookup auto_parse_some_output on nwbinspector.register_checks failed" in str(e): + del kwargs["n_jobs"] + messages = list(inspect_all(**kwargs)) + else: + raise e except Exception as e: raise e