From 84e9d47afba251e8e085d3ebf7511996917a68ed Mon Sep 17 00:00:00 2001 From: valosekj Date: Mon, 6 Nov 2023 15:47:17 -0500 Subject: [PATCH 1/4] Check if task is in suffix_dict.keys(), if not, skip it --- manual_correction.py | 4 ++++ utils.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/manual_correction.py b/manual_correction.py index 4f71819..da107a3 100644 --- a/manual_correction.py +++ b/manual_correction.py @@ -756,6 +756,10 @@ def main(): # Perform manual corrections for task, files in dict_yml.items(): if task.startswith('FILES'): + # Check if task is in suffix_dict.keys(), if not, skip it + if task not in suffix_dict.keys(): + logging.warning("WARNING: {} is not a valid task. Skipping it.".format(task)) + continue # Get the list of segmentation files to add to derivatives, excluding the manually corrected files in -config. # TODO: probably extend also for other tasks (such as FILES_GMSEG) if args.add_seg_only and task == 'FILES_SEG': diff --git a/utils.py b/utils.py index 0f1552b..7083683 100644 --- a/utils.py +++ b/utils.py @@ -211,6 +211,10 @@ def check_files_exist(dict_yml, path_img, path_label, suffix_dict): missing_files_labels = [] for task, files in dict_yml.items(): if task.startswith('FILES') and files: + # Check if task is in suffix_dict.keys(), if not, skip it + if task not in suffix_dict.keys(): + logging.warning("WARNING: {} is not a valid task. Skipping it.".format(task)) + continue # Do no check if key is empty or if regex is used if files is not None and '*' not in files[0]: for file in files: From 78f5cfc5ab6531fb4b9387ab6d5bb77b9be52cbd Mon Sep 17 00:00:00 2001 From: Mathieu Guay-Paquet Date: Mon, 6 Nov 2023 19:16:20 -0500 Subject: [PATCH 2/4] Keep track of which tasks have possibly-bad suffixes Otherwise, the last task from the loop is always used, regardless of whether it actually has missing_files_labels (and regardless of whether it's actually in suffix_dict.keys()). --- utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 7083683..04124db 100644 --- a/utils.py +++ b/utils.py @@ -209,6 +209,7 @@ def check_files_exist(dict_yml, path_img, path_label, suffix_dict): """ missing_files = [] missing_files_labels = [] + missing_suffixes = set() for task, files in dict_yml.items(): if task.startswith('FILES') and files: # Check if task is in suffix_dict.keys(), if not, skip it @@ -227,6 +228,7 @@ def check_files_exist(dict_yml, path_img, path_label, suffix_dict): fname_label = add_suffix(os.path.join(path_label, subject, ses, contrast, filename), suffix_dict[task]) if not os.path.exists(fname_label): missing_files_labels.append(fname_label) + missing_suffixes.add(suffix_dict[task]) if missing_files: logging.warning("The following files are missing: \n{}".format(missing_files)) logging.warning("\nPlease check that the files listed in the yaml file and the input path are correct.\n") @@ -234,7 +236,7 @@ def check_files_exist(dict_yml, path_img, path_label, suffix_dict): logging.warning("If you are creating label(s) from scratch, ignore the following message.") logging.warning("\nThe following label files are missing: \n{}".format(missing_files_labels)) logging.warning("\nPlease check that the used suffix '{}' is correct. " - "If not, you can provide custom suffix using '-suffix-files-' flags.\n".format(suffix_dict[task])) + "If not, you can provide custom suffix using '-suffix-files-' flags.\n".format(sorted(missing_suffixes))) def check_output_folder(path_bids): @@ -327,4 +329,4 @@ def track_corrections(files_dict, config_path, file_path, task): return files_dict - \ No newline at end of file + From eb7c18456070bef259b73d0706fdf454b5d322fa Mon Sep 17 00:00:00 2001 From: Mathieu Guay-Paquet Date: Mon, 6 Nov 2023 19:44:55 -0500 Subject: [PATCH 3/4] Fix test --- tests/test_utils.py | 2 +- utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index dd60564..af26390 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -167,7 +167,7 @@ def test_check_files_exist_missing_file(tmp_path, caplog): assert any('Please check that the files listed in the yaml file and the input path are correct' in rec.message for rec in caplog.records) assert any('BIDS/sub-001/ses-01/anat/sub-001_ses-01_T1w_seg.nii.gz' in rec.message for rec in caplog.records) assert any('BIDS/sub-002/ses-01/anat/sub-001_ses-01_T2star_gmseg.nii.gz' in rec.message for rec in caplog.records) - assert any("Please check that the used suffix '_gmseg' is correct" in rec.message for rec in caplog.records) + assert any("Please check that the used suffix ['_gmseg', '_seg'] is correct" in rec.message for rec in caplog.records) def test_track_corrections(tmp_path): diff --git a/utils.py b/utils.py index 04124db..66dd9e7 100644 --- a/utils.py +++ b/utils.py @@ -235,7 +235,7 @@ def check_files_exist(dict_yml, path_img, path_label, suffix_dict): if missing_files_labels: logging.warning("If you are creating label(s) from scratch, ignore the following message.") logging.warning("\nThe following label files are missing: \n{}".format(missing_files_labels)) - logging.warning("\nPlease check that the used suffix '{}' is correct. " + logging.warning("\nPlease check that the used suffix {} is correct. " "If not, you can provide custom suffix using '-suffix-files-' flags.\n".format(sorted(missing_suffixes))) From 93a38b2a3645313729fd8c7d8a988b18f4a1b9ff Mon Sep 17 00:00:00 2001 From: valosekj Date: Tue, 7 Nov 2023 07:56:36 -0500 Subject: [PATCH 4/4] Add comment why `if task not in suffix_dict.keys()` check is done after `if task.startswith('FILES')'` check --- manual_correction.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manual_correction.py b/manual_correction.py index da107a3..d41aeac 100644 --- a/manual_correction.py +++ b/manual_correction.py @@ -757,6 +757,9 @@ def main(): for task, files in dict_yml.items(): if task.startswith('FILES'): # Check if task is in suffix_dict.keys(), if not, skip it + # Note that this check is done after the task.startswith('FILES') check because the manual-correction + # script should ignore keys that start with CORR (CORR keys are used to track the manual correction + # progress) if task not in suffix_dict.keys(): logging.warning("WARNING: {} is not a valid task. Skipping it.".format(task)) continue