Implement flagging system at each pipeline stage with --inclusion_ids…

… argument.

rabies/analysis_pkg/main_wf.py

@@ -25,8 +25,9 @@ def init_main_analysis_wf(preprocess_opts, cr_opts, analysis_opts):
             the case.
             """)
 
-    # update split_name according to the --scan_list option
-    split_name_list = get_iterable_scan_list(analysis_opts.scan_list, split_name_list)
+    # filter inclusion/exclusion lists
+    from rabies.utils import filter_scan_inclusion
+    split_name_list = filter_scan_inclusion(analysis_opts.inclusion_ids, split_name_list)
 
     # setting up iterables from the BOLD scan splits
     main_split = pe.Node(niu.IdentityInterface(fields=['split_name']),

rabies/confound_correction_pkg/main_wf.py

@@ -23,6 +23,10 @@ def init_main_confound_correction_wf(preprocess_opts, cr_opts):
     else:
         split_dict, split_name, target_list = read_preproc_workflow(preproc_output, nativespace=cr_opts.nativespace_analysis)
 
+    # filter inclusion/exclusion lists
+    from rabies.utils import filter_scan_inclusion
+    split_name = filter_scan_inclusion(cr_opts.inclusion_ids, split_name)
+
     # setting up iterables from the BOLD scan splits
     main_split = pe.Node(niu.IdentityInterface(fields=['split_name']),
                          name="main_split")

rabies/parser.py

@@ -80,6 +80,22 @@ def get_parser():
         description=
             "Options for parallel execution and memory management."
         )
+    g_execution.add_argument(
+        '--inclusion_ids', type=str,
+        nargs="*",  # 0 or more values expected => creates a list
+        default=['all'],
+        help=
+            "Define a list of BOLD scan to include, i.e. run the pipeline on a subset of the data. \n"
+            "To do so, provide the full path to the corresponding BOLD file in the input BIDS folder. The list \n"
+            "of scan can be specified manually as a list of file name '--scan_list scan1.nii.gz \n"
+            "scan2.nii.gz ...' or the files can be imbedded into a .txt file with one filename per row.\n"
+            "By default, 'all' the scans found in the input BIDS directory or from the previous \n"
+            "processing step. This can be provided at any processing stage.\n"
+            "***NOTE: do not enter this parameter right before the processing stage (preprocess, etc...), this will cause \n"
+            "parsing errors. Instead, provide another parameter after --inclusion_ids (e.g. --verbose or -p). \n"
+            "(default: %(default)s)\n"
+            "\n"
+        )
     g_execution.add_argument(
         "-p", "--plugin", default='Linear',
         choices=['Linear', 'MultiProc', 'SGE', 'SGEGraph',

rabies/preprocess_pkg/main_wf.py

@@ -135,7 +135,7 @@ def init_main_wf(data_dir_path, output_folder, opts, name='main_wf'):
     bids.config.set_option('extension_initial_dot', True)
     layout = bids.layout.BIDSLayout(data_dir_path, validate=False)
     split_name, scan_info, run_iter, scan_list, bold_scan_list = prep_bids_iter(
-        layout, opts.bold_only)
+        layout, opts.bold_only, inclusion_list=opts.inclusion_ids)
 
     # setting up all iterables
     main_split = pe.Node(niu.IdentityInterface(fields=['split_name', 'scan_info']),

rabies/preprocess_pkg/utils.py

@@ -8,7 +8,7 @@
 )
 from rabies.utils import run_command
 
-def prep_bids_iter(layout, bold_only=False):
+def prep_bids_iter(layout, bold_only=False, inclusion_list=['all']):
     '''
     This function takes as input a BIDSLayout, and generates iteration lists
     for managing the workflow's iterables depending on whether --bold_only is
@@ -38,6 +38,18 @@ def prep_bids_iter(layout, bold_only=False):
         raise ValueError(
             "No functional file with the suffix 'bold' were found among the BIDS directory.")
 
+    # filter inclusion/exclusion lists
+    from rabies.utils import filter_scan_inclusion
+    boldname_list=[pathlib.Path(bold.filename).name.rsplit(".nii")[0] for bold in bold_bids]
+    updated_split_name = filter_scan_inclusion(inclusion_list, boldname_list)
+
+    filtered_bold_bids=[]
+    for name in updated_split_name:
+        for bold in bold_bids:
+            if name in bold.filename:
+                filtered_bold_bids.append(bold)
+    bold_bids = filtered_bold_bids
+
     bold_dict = {}
     for bold in bold_bids:
         sub = bold.get_entities()['subject']

rabies/utils.py

@@ -389,6 +389,38 @@ def flatten_list(l):
         return l
 
 
+def filter_scan_inclusion(inclusion_list, split_name):
+    # the function will update the list of scan IDs in split_name to correspond to inclusion/exclusion list
+
+    # inclusion_list: the input provided by the user
+    # split_name: a list of all scan IDs that were found
+
+    import numpy as np
+    import pandas as pd
+    if os.path.isfile(os.path.abspath(inclusion_list[0])):
+        updated_split_name=[]
+        if '.nii' in pathlib.Path(inclusion_list[0]).name:
+            for scan in inclusion_list:
+                updated_split_name.append(find_split(scan, split_name))
+        else:
+            # read the file as a .txt
+            inclusion_list = np.array(pd.read_csv(os.path.abspath(inclusion_list[0]), header=None)).flatten()
+            for scan in inclusion_list:
+                updated_split_name.append(find_split(scan, split_name))
+    elif inclusion_list[0]=='all':
+        updated_split_name = split_name
+    else:
+        raise ValueError(f"The --inclusion_ids {inclusion_list} input had improper format. It must the full path to a .txt or .nii files, or 'all' to keep all scans.")
+    return updated_split_name
+
+
+def find_split(scan, split_name):
+    for split in split_name:
+        if split in scan:
+            return split
+    raise ValueError(f"No previous file name is matching {scan}")
+
+
 ######################
 #FUNCTIONS TO READ WORKFLOW GRAPH
 ######################