From 4ee7b0fc87240b639ba1481a20216837f36b6556 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Tue, 6 Aug 2019 11:16:31 -0400 Subject: [PATCH 01/29] acquisition.info.Date is for DICOMS, use acquisition.info.DateTime #28 --- fw_heudiconv/cli/export.py | 5 ++++- fw_heudiconv/query.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fw_heudiconv/cli/export.py b/fw_heudiconv/cli/export.py index 89af40f..a2c8e6b 100644 --- a/fw_heudiconv/cli/export.py +++ b/fw_heudiconv/cli/export.py @@ -306,7 +306,10 @@ def main(): args = parser.parse_args() project_label = ' '.join(args.project) - assert os.path.exists(args.path), "Path does not exist!" + if not os.path.exists(args.path): + logger.info("Creating target directory...") + os.makedirs(args.path) + #assert os.path.exists(args.path), "Path does not exist!" downloads = gather_bids(client=fw, project_label=project_label, session_labels=args.session, diff --git a/fw_heudiconv/query.py b/fw_heudiconv/query.py index bb07536..0e37fdc 100644 --- a/fw_heudiconv/query.py +++ b/fw_heudiconv/query.py @@ -81,7 +81,7 @@ def acquisition_to_heudiconv(client, acq, context): info.get("AccessionNumber"), info.get("PatientAge"), info.get("PatientSex"), - info.get("AcquisitionDate"), + info.get("AcquisitionDateTime"), info.get("SeriesInstanceUID") )) # We could possible add a context field which would contain flywheel From 06a74291ae2ac5f5116dcf886b6bad6aa946cd8c Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Fri, 30 Aug 2019 16:47:40 -0400 Subject: [PATCH 02/29] added a patch flag for unique vs. non unique tabulation. also addresses #36 --- fw_heudiconv/cli/tabulate.py | 13 ++++++++++--- setup.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/fw_heudiconv/cli/tabulate.py b/fw_heudiconv/cli/tabulate.py index ba204c8..5268af3 100644 --- a/fw_heudiconv/cli/tabulate.py +++ b/fw_heudiconv/cli/tabulate.py @@ -46,6 +46,7 @@ def tabulate_bids(client, project_label, path=".", subject_labels=None, df = pd.DataFrame.from_dict(seq_info_dicts) if unique: df = df.drop_duplicates(subset=['TR', 'TE', 'protocol_name', 'is_motion_corrected', 'is_derived']) + df = df.drop(columns=['total_files_till_now', 'dcm_dir_name']) if dry_run: print(df) else: @@ -96,10 +97,16 @@ def get_parser(): default=False ) parser.add_argument( - "--unique", - help="Strip down to unique sequence combinations", - default=True + '--unique', + dest='unique', + action='store_true' ) + parser.add_argument( + '--no-unique', + dest='unique', + action='store_false' + ) + parser.set_defaults(unique=True) return parser diff --git a/setup.py b/setup.py index 884aa92..d3c0ba3 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setuptools.setup( name="fw_heudiconv", - version="0.0.18", + version="0.0.19", author="Tinashe M. Tapera, Matt Cieslak, Harsha Kethineni", author_email="tinashemtapera@gmail.com", description="Use heudiconv heuristics for BIDS curation on flywheel", From 61fe6b340f0e99b1641699093507edfe6a8e1e91 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Tue, 10 Sep 2019 18:05:33 -0400 Subject: [PATCH 03/29] adds user option to name bids output directory --- fw_heudiconv/cli/export.py | 40 ++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/fw_heudiconv/cli/export.py b/fw_heudiconv/cli/export.py index a2c8e6b..a24a889 100644 --- a/fw_heudiconv/cli/export.py +++ b/fw_heudiconv/cli/export.py @@ -7,6 +7,7 @@ import shutil import re import csv +import pandas as pd from pathlib import Path from ..query import print_directory_tree @@ -78,13 +79,15 @@ def gather_bids(client, project_label, subject_labels=None, session_labels=None) 'name': container.filename, 'path': path, 'type': type of file, - 'data': container} + 'data': container.id + } ''' logger.info("Gathering bids data:") to_download = { 'dataset_description': [], 'project': [], + 'subject': [], 'session': [], 'acquisition': [] } @@ -106,7 +109,8 @@ def gather_bids(client, project_label, subject_labels=None, session_labels=None) d = { 'name': pf.name, 'type': 'attachment', - 'data': project_obj.id + 'data': project_obj.id, + 'BIDS': get_nested(pf, 'info', 'BIDS') } to_download['project'].append(d) @@ -150,13 +154,13 @@ def gather_bids(client, project_label, subject_labels=None, session_labels=None) return to_download -def download_bids(client, to_download, root_path, folders_to_download = ['anat', 'dwi', 'func', 'fmap'], dry_run=True): +def download_bids(client, to_download, root_path, folders_to_download = ['anat', 'dwi', 'func', 'fmap'], dry_run=True, name='bids_dataset'): if dry_run: logger.info("Preparing output directory tree...") else: logger.info("Downloading files...") - root_path = "/".join([root_path, "bids_dataset"]) + root_path = "/".join([root_path, name]) Path(root_path).mkdir() # handle dataset description if to_download['dataset_description']: @@ -181,13 +185,26 @@ def download_bids(client, to_download, root_path, folders_to_download = ['anat', bidsignore.writelines(ignored_modalities) # deal with project level files - # NOT YET IMPLEMENTED + # Project's subject data for fi in to_download['project']: - pass + + project_path = get_nested(fi, 'BIDS', 'Path') + folder = get_nested(fi, 'BIDS', 'Folder') + ignore = get_nested(fi, 'BIDS', 'ignore') + + if project_path \ + and folder in folders_to_download \ + and not ignore \ + and any(fi['name'] == 'participants.tsv' or fi['name'] == 'participants.json'): + + proj = client.get(fi['data']) #download_path = get_metadata(fi, ['BIDS', 'Path']) #if download_path: # print('/'.join([root_path, download_path, fi['name']])) + proj.download_file(fi['name'], file_path) + download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True) + # deal with session level files # NOT YET IMPLEMENTED for fi in to_download['session']: @@ -292,6 +309,12 @@ def get_parser(): action='store_true', default=False ) + parser.add_argument( + "--directory_name", + help="Name of the directory", + default="bids_directory", + type=str + ) return parser @@ -313,9 +336,10 @@ def main(): downloads = gather_bids(client=fw, project_label=project_label, session_labels=args.session, - subject_labels=args.subject) + subject_labels=args.subject + ) - download_bids(client=fw, to_download=downloads, root_path=args.path, folders_to_download=args.folders, dry_run=args.dry_run) + download_bids(client=fw, to_download=downloads, root_path=args.path, folders_to_download=args.folders, dry_run=args.dry_run, name=args.directory_name) if __name__ == '__main__': From f58aa5f67b9a7fa8964f5b623cba69af5ae16d2b Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 11 Sep 2019 11:35:09 -0400 Subject: [PATCH 04/29] Major change: Fixed an inconsistency between sub-{subject} and ses-{session}. **Heuristics now require the full ses-{session} for all paths and templates** Cosmetics: Pretty print for IntendedFor mapping --- fw_heudiconv/cli/curate.py | 8 ++++---- fw_heudiconv/convert.py | 23 +++++++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/fw_heudiconv/cli/curate.py b/fw_heudiconv/cli/curate.py index e84887f..61a237c 100644 --- a/fw_heudiconv/cli/curate.py +++ b/fw_heudiconv/cli/curate.py @@ -4,6 +4,7 @@ import argparse import warnings import flywheel +import pprint from collections import defaultdict from ..convert import apply_heuristic, confirm_intentions, confirm_bids_namespace from ..query import get_seq_info @@ -94,7 +95,7 @@ def convert_to_bids(client, project_label, heuristic_path, subject_labels=None, if hasattr(heuristic, "IntendedFor"): logger.info("Processing IntendedFor fields based on heuristic file") intention_map.update(heuristic.IntendedFor) - logger.debug("Intention map: %s", intention_map) + logger.debug("Intention map: %s", pprint.pformat([(k[0], v) for k, v in dict(intention_map).items()])) metadata_extras = defaultdict(list) if hasattr(heuristic, "MetadataExtras"): @@ -123,9 +124,8 @@ def convert_to_bids(client, project_label, heuristic_path, subject_labels=None, apply_heuristic(client, key, value, dry_run, intention_map[key], metadata_extras[key], subject_rename, session_rename, seqitem+1) - if not dry_run: - for ses in sessions: - confirm_intentions(client, ses) + for ses in sessions: + confirm_intentions(client, ses, dry_run) def get_parser(): diff --git a/fw_heudiconv/convert.py b/fw_heudiconv/convert.py index 2991daf..3947931 100644 --- a/fw_heudiconv/convert.py +++ b/fw_heudiconv/convert.py @@ -6,6 +6,7 @@ import operator import re import traceback +import pprint from .cli.export import get_nested logger = logging.getLogger('fwHeuDiConv-curator') @@ -49,11 +50,11 @@ def apply_heuristic(client, heur, acquisition_id, dry_run=False, intended_for=[] files = [f for f in acquisition_object.files if f.type in ftypes] bids_keys = ['sub', 'ses', 'folder', 'name'] - ses_fmt = sess_label if sess_label.startswith("ses-") else "ses-" + sess_label + #ses_fmt = sess_label if sess_label.startswith("ses-") else "ses-" + sess_label files.sort(key=operator.itemgetter("name")) for fnum, f in enumerate(files): - bids_vals = template.format(subject=subj_label, session=ses_fmt, item=fnum+1, seqitem=item_num).split("/") + bids_vals = template.format(subject=subj_label, session=sess_label, item=fnum+1, seqitem=item_num).split("/") bids_dict = dict(zip(bids_keys, bids_vals)) suffix = suffixes[f.type] @@ -79,9 +80,9 @@ def apply_heuristic(client, heur, acquisition_id, dry_run=False, intended_for=[] acquisition_object.update_file_info(f.name, {'BIDS': new_bids}) if intended_for and (f.name.endswith(".nii.gz") or f.name.endswith(".nii")): - intendeds = [intend.format(subject=subj_label, session=ses_fmt) + intendeds = [intend.format(subject=subj_label, session=sess_label) for intend in intended_for] - logger.debug("%s IntendedFor: %s", new_bids['Filename'], intendeds) + logger.debug("%s IntendedFor: %s", pprint.pformat(new_bids['Filename']), pprint.pformat(intendeds)) if not dry_run: acquisition_object.update_file_info(f.name, {'IntendedFor': intendeds}) @@ -214,7 +215,7 @@ def infer_params_from_filename(bdict): bdict.update(to_fill) -def confirm_intentions(client, session): +def confirm_intentions(client, session, dry_run=False): try: acqs = [client.get(s.id) for s in session.acquisitions()] @@ -228,8 +229,11 @@ def confirm_intentions(client, session): pass else: full_filenames.append("/".join(x)) - ses_labs = [re.search(r"ses-[a-zA-z0-9]+(?=_)", x).group() for x in full_filenames if x is not None] - l2 = list(zip(ses_labs, full_filenames)) + + ses_labs = [re.search(r"ses-[a-zA-z0-9]+(?=_)", x).group() for x in full_filenames if x is not None] #some subjects don't have BIDS, this is useless + sub_labs = [re.search(r"sub-[a-zA-z0-9]+(?=_)", x).group() for x in full_filenames if x is not None] + l2 = list(zip(sub_labs, ses_labs, full_filenames)) + paths = [] for x in l2: if not(None in x): @@ -238,14 +242,17 @@ def confirm_intentions(client, session): for a in acqs: for x in a.files: if x.type == 'nifti': + intendeds = get_nested(x.to_dict(), 'info', 'IntendedFor') + if intendeds: if not all([i in paths for i in intendeds]): logger.debug("Ensuring all intentions apply for acquisition %s: %s", a.label, x.name) exists = [i for i in intendeds if i in paths] missing = [i for i in intendeds if i not in paths] logger.debug("Missing paths: %s" % missing) - a.update_file_info(x.name, {'IntendedFor': exists}) + if not dry_run: + a.update_file_info(x.name, {'IntendedFor': exists}) except Exception as e: logger.warning("Trouble updating intentions for this session %s", session.label) From d73a22b312eaa170fc4391e361bb86507426e1cb Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 11 Sep 2019 14:34:07 -0400 Subject: [PATCH 05/29] sorting seqinfos --- fw_heudiconv/query.py | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fw_heudiconv/query.py b/fw_heudiconv/query.py index 0e37fdc..fad274f 100644 --- a/fw_heudiconv/query.py +++ b/fw_heudiconv/query.py @@ -103,7 +103,9 @@ def session_to_seq_info(client, session, context): """ seq_info = collections.OrderedDict() context['total'] = 0 - for acquisition in session.acquisitions(): + acquisitions = session.acquisitions() + sorted_acquisitions = sorted(acquisitions, key=lambda x: x.timestamp) + for acquisition in sorted_acquisitions: acquisition = client.get(acquisition.id) context['acquisition'] = acquisition diff --git a/setup.py b/setup.py index d3c0ba3..eb62810 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setuptools.setup( name="fw_heudiconv", - version="0.0.19", + version="0.0.20", author="Tinashe M. Tapera, Matt Cieslak, Harsha Kethineni", author_email="tinashemtapera@gmail.com", description="Use heudiconv heuristics for BIDS curation on flywheel", From 74377597044f4b712ac38cf0e66b4985e17e7083 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 11 Sep 2019 14:34:07 -0400 Subject: [PATCH 06/29] sorting seqinfos --- fw_heudiconv/query.py | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fw_heudiconv/query.py b/fw_heudiconv/query.py index 0e37fdc..fad274f 100644 --- a/fw_heudiconv/query.py +++ b/fw_heudiconv/query.py @@ -103,7 +103,9 @@ def session_to_seq_info(client, session, context): """ seq_info = collections.OrderedDict() context['total'] = 0 - for acquisition in session.acquisitions(): + acquisitions = session.acquisitions() + sorted_acquisitions = sorted(acquisitions, key=lambda x: x.timestamp) + for acquisition in sorted_acquisitions: acquisition = client.get(acquisition.id) context['acquisition'] = acquisition diff --git a/setup.py b/setup.py index d3c0ba3..eb62810 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setuptools.setup( name="fw_heudiconv", - version="0.0.19", + version="0.0.20", author="Tinashe M. Tapera, Matt Cieslak, Harsha Kethineni", author_email="tinashemtapera@gmail.com", description="Use heudiconv heuristics for BIDS curation on flywheel", From 895d53d2d7287a6cca9c8ce7a38f9304aa45f018 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Fri, 13 Sep 2019 17:53:28 -0400 Subject: [PATCH 07/29] added fw-heudiconv-clean --- fw_heudiconv/cli/clean.py | 156 +++++++++++++++++++++++++++++++++++++ fw_heudiconv/cli/curate.py | 2 +- fw_heudiconv/cli/export.py | 4 +- setup.py | 5 +- 4 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 fw_heudiconv/cli/clean.py diff --git a/fw_heudiconv/cli/clean.py b/fw_heudiconv/cli/clean.py new file mode 100644 index 0000000..5ff0301 --- /dev/null +++ b/fw_heudiconv/cli/clean.py @@ -0,0 +1,156 @@ +import argparse +import flywheel +import logging +import warnings +import sys + +with warnings.catch_warnings(): + warnings.simplefilter('ignore') + from fw_heudiconv.cli.export import get_nested + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger('fw-heudiconv-cleaner') + + +def clear_bids(client, project_label, session_labels, subject_labels, dry_run=False, file_types = ['.nii', '.bval', '.bvec']): + + logger.info("\t\t=======: fw-heudiconv starting up :=======\n") + logger.info("Querying Flywheel server...") + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + project_obj = client.projects.find_first('label="{}"'.format(project_label)) + + if project_obj is None: + logger.error("Project not found! Maybe check spelling...?") + return 1 + + logger.debug('\tFound project: \n\t\t%s (%s)', project_obj['label'], project_obj.id) + sessions = client.get_project_sessions(project_obj.id) + + # filters + if subject_labels: + sessions = [s for s in sessions if s.subject['label'] in subject_labels] + if session_labels: + sessions = [s for s in sessions if s.label in session_labels] + + if not sessions: + logger.error("No sessions found!") + return 1 + + logger.info('\tFound subjects:\n\t\t%s', + "\n\t\t".join(set(['%s (%s)' % (ses.subject.label, ses.subject.id) for ses in sessions]))) + + logger.info('\tFound sessions:\n\t\t%s', + "\n\t\t".join(['%s (%s)' % (ses['label'], ses.id) for ses in sessions])) + + file_list = [] + for ses in sessions: + + acquisitions = ses.acquisitions() + + for acq in acquisitions: + + files = [f.to_dict() for f in acq.files if any([x in f.name for x in file_types])] + + files = [f for f in files if get_nested(f, 'info', 'BIDS') != 'NA' and get_nested(f, 'info', 'BIDS') is not None and get_nested(f, 'info', 'BIDS', 'Filename') != ''] + + if files: + file_list.append({acq.id: files}) + + fnames = [] + for x in file_list: + for k, v in x.items(): + for u in v: + fnames.append(get_nested(u, 'info', 'BIDS', 'Filename')) + + if file_list: + logger.debug("This will remove BIDS data from %d files:\n\t%s" % (len(file_list), "\n\t".join([x for x in fnames]))) + + + if not dry_run: + logger.info('\t\t=======: Removing BIDS data :=======\n') + + for acq_files in file_list: + + for k, v in acq_files.items(): + acq = client.get(k) + + for fi in v: + + BIDS = get_nested(fi, 'info', 'BIDS') + new_bids = {k:'' for k,v in BIDS.items()} + acq.update_file_info(fi['name'], {'BIDS': new_bids}) + + else: + logger.info("Disable `dry_run` mode to apply these changes and remove the BIDS information.") + + else: + logger.info("No BIDS data to remove! (That was easy...)") + + return 0 + + +def get_parser(): + + parser = argparse.ArgumentParser( + description="Go nuclear: clear BIDS data from Flywheel") + parser.add_argument( + "--project", + help="The project in flywheel", + nargs="+", + required=True + ) + parser.add_argument( + "--subject", + help="The subject label(s)", + nargs="+", + default=None + ) + parser.add_argument( + "--session", + help="The session label(s)", + nargs="+", + default=None + ) + parser.add_argument( + "--verbose", + help="Print ongoing messages of progress", + action='store_true', + default=False + ) + parser.add_argument( + "--dry_run", + help="Don't apply changes", + action='store_true', + default=False + ) + + return parser + + +def main(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fw = flywheel.Client() + assert fw, "Your Flywheel CLI credentials aren't set!" + parser = get_parser() + args = parser.parse_args() + + # Print a lot if requested + if args.verbose: + logger.setLevel(logging.DEBUG) + + project_label = ' '.join(args.project) + status = clear_bids(client=fw, + project_label=project_label, + session_labels=args.session, + subject_labels=args.subject, + dry_run=args.dry_run) + + logger.info("\t\t=======: Done :=======") + sys.exit(status) + +if __name__ == '__main__': + main() diff --git a/fw_heudiconv/cli/curate.py b/fw_heudiconv/cli/curate.py index 61a237c..271506c 100644 --- a/fw_heudiconv/cli/curate.py +++ b/fw_heudiconv/cli/curate.py @@ -13,7 +13,7 @@ import logging logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('fwHeuDiConv-curator') +logger = logging.getLogger('fw-heudiconv-curator') def pretty_string_seqinfo(seqinfo): tr = seqinfo.TR if seqinfo.TR is not None else -1.0 diff --git a/fw_heudiconv/cli/export.py b/fw_heudiconv/cli/export.py index a24a889..2e06d75 100644 --- a/fw_heudiconv/cli/export.py +++ b/fw_heudiconv/cli/export.py @@ -202,8 +202,8 @@ def download_bids(client, to_download, root_path, folders_to_download = ['anat', #if download_path: # print('/'.join([root_path, download_path, fi['name']])) - proj.download_file(fi['name'], file_path) - download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True) + #proj.download_file(fi['name'], file_path) + #download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True) # deal with session level files # NOT YET IMPLEMENTED diff --git a/setup.py b/setup.py index d3c0ba3..835cae0 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setuptools.setup( name="fw_heudiconv", - version="0.0.19", + version="0.1.0", author="Tinashe M. Tapera, Matt Cieslak, Harsha Kethineni", author_email="tinashemtapera@gmail.com", description="Use heudiconv heuristics for BIDS curation on flywheel", @@ -29,7 +29,8 @@ 'console_scripts': [ 'fw-heudiconv-curate=fw_heudiconv.cli.curate:main', 'fw-heudiconv-export=fw_heudiconv.cli.export:main', - 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main' + 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main', + 'fw-heudiconv-clear=fw_heudiconv.cli.clear:main' ], } ) From 26cd5d6530ca3e67487d3f8ec1aa3c4ce09e1ace Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Fri, 13 Sep 2019 17:53:28 -0400 Subject: [PATCH 08/29] ammending --- fw_heudiconv/cli/clean.py | 156 +++++++++++++++++++++++++++++++++++++ fw_heudiconv/cli/curate.py | 2 +- fw_heudiconv/cli/export.py | 4 +- setup.py | 5 +- 4 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 fw_heudiconv/cli/clean.py diff --git a/fw_heudiconv/cli/clean.py b/fw_heudiconv/cli/clean.py new file mode 100644 index 0000000..5ff0301 --- /dev/null +++ b/fw_heudiconv/cli/clean.py @@ -0,0 +1,156 @@ +import argparse +import flywheel +import logging +import warnings +import sys + +with warnings.catch_warnings(): + warnings.simplefilter('ignore') + from fw_heudiconv.cli.export import get_nested + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger('fw-heudiconv-cleaner') + + +def clear_bids(client, project_label, session_labels, subject_labels, dry_run=False, file_types = ['.nii', '.bval', '.bvec']): + + logger.info("\t\t=======: fw-heudiconv starting up :=======\n") + logger.info("Querying Flywheel server...") + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + project_obj = client.projects.find_first('label="{}"'.format(project_label)) + + if project_obj is None: + logger.error("Project not found! Maybe check spelling...?") + return 1 + + logger.debug('\tFound project: \n\t\t%s (%s)', project_obj['label'], project_obj.id) + sessions = client.get_project_sessions(project_obj.id) + + # filters + if subject_labels: + sessions = [s for s in sessions if s.subject['label'] in subject_labels] + if session_labels: + sessions = [s for s in sessions if s.label in session_labels] + + if not sessions: + logger.error("No sessions found!") + return 1 + + logger.info('\tFound subjects:\n\t\t%s', + "\n\t\t".join(set(['%s (%s)' % (ses.subject.label, ses.subject.id) for ses in sessions]))) + + logger.info('\tFound sessions:\n\t\t%s', + "\n\t\t".join(['%s (%s)' % (ses['label'], ses.id) for ses in sessions])) + + file_list = [] + for ses in sessions: + + acquisitions = ses.acquisitions() + + for acq in acquisitions: + + files = [f.to_dict() for f in acq.files if any([x in f.name for x in file_types])] + + files = [f for f in files if get_nested(f, 'info', 'BIDS') != 'NA' and get_nested(f, 'info', 'BIDS') is not None and get_nested(f, 'info', 'BIDS', 'Filename') != ''] + + if files: + file_list.append({acq.id: files}) + + fnames = [] + for x in file_list: + for k, v in x.items(): + for u in v: + fnames.append(get_nested(u, 'info', 'BIDS', 'Filename')) + + if file_list: + logger.debug("This will remove BIDS data from %d files:\n\t%s" % (len(file_list), "\n\t".join([x for x in fnames]))) + + + if not dry_run: + logger.info('\t\t=======: Removing BIDS data :=======\n') + + for acq_files in file_list: + + for k, v in acq_files.items(): + acq = client.get(k) + + for fi in v: + + BIDS = get_nested(fi, 'info', 'BIDS') + new_bids = {k:'' for k,v in BIDS.items()} + acq.update_file_info(fi['name'], {'BIDS': new_bids}) + + else: + logger.info("Disable `dry_run` mode to apply these changes and remove the BIDS information.") + + else: + logger.info("No BIDS data to remove! (That was easy...)") + + return 0 + + +def get_parser(): + + parser = argparse.ArgumentParser( + description="Go nuclear: clear BIDS data from Flywheel") + parser.add_argument( + "--project", + help="The project in flywheel", + nargs="+", + required=True + ) + parser.add_argument( + "--subject", + help="The subject label(s)", + nargs="+", + default=None + ) + parser.add_argument( + "--session", + help="The session label(s)", + nargs="+", + default=None + ) + parser.add_argument( + "--verbose", + help="Print ongoing messages of progress", + action='store_true', + default=False + ) + parser.add_argument( + "--dry_run", + help="Don't apply changes", + action='store_true', + default=False + ) + + return parser + + +def main(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fw = flywheel.Client() + assert fw, "Your Flywheel CLI credentials aren't set!" + parser = get_parser() + args = parser.parse_args() + + # Print a lot if requested + if args.verbose: + logger.setLevel(logging.DEBUG) + + project_label = ' '.join(args.project) + status = clear_bids(client=fw, + project_label=project_label, + session_labels=args.session, + subject_labels=args.subject, + dry_run=args.dry_run) + + logger.info("\t\t=======: Done :=======") + sys.exit(status) + +if __name__ == '__main__': + main() diff --git a/fw_heudiconv/cli/curate.py b/fw_heudiconv/cli/curate.py index 61a237c..271506c 100644 --- a/fw_heudiconv/cli/curate.py +++ b/fw_heudiconv/cli/curate.py @@ -13,7 +13,7 @@ import logging logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('fwHeuDiConv-curator') +logger = logging.getLogger('fw-heudiconv-curator') def pretty_string_seqinfo(seqinfo): tr = seqinfo.TR if seqinfo.TR is not None else -1.0 diff --git a/fw_heudiconv/cli/export.py b/fw_heudiconv/cli/export.py index a24a889..2e06d75 100644 --- a/fw_heudiconv/cli/export.py +++ b/fw_heudiconv/cli/export.py @@ -202,8 +202,8 @@ def download_bids(client, to_download, root_path, folders_to_download = ['anat', #if download_path: # print('/'.join([root_path, download_path, fi['name']])) - proj.download_file(fi['name'], file_path) - download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True) + #proj.download_file(fi['name'], file_path) + #download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True) # deal with session level files # NOT YET IMPLEMENTED diff --git a/setup.py b/setup.py index d3c0ba3..d9b5dd3 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setuptools.setup( name="fw_heudiconv", - version="0.0.19", + version="0.1.0", author="Tinashe M. Tapera, Matt Cieslak, Harsha Kethineni", author_email="tinashemtapera@gmail.com", description="Use heudiconv heuristics for BIDS curation on flywheel", @@ -29,7 +29,8 @@ 'console_scripts': [ 'fw-heudiconv-curate=fw_heudiconv.cli.curate:main', 'fw-heudiconv-export=fw_heudiconv.cli.export:main', - 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main' + 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main', + 'fw-heudiconv-clear=fw_heudiconv.cli.clean:main' ], } ) From e8567b5f73231174979580c66eef15470bbed74d Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Tue, 17 Sep 2019 10:56:07 -0400 Subject: [PATCH 09/29] #42 add call to setup.py --- setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.py b/setup.py index 88556b2..d9b5dd3 100644 --- a/setup.py +++ b/setup.py @@ -30,11 +30,7 @@ 'fw-heudiconv-curate=fw_heudiconv.cli.curate:main', 'fw-heudiconv-export=fw_heudiconv.cli.export:main', 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main', -<<<<<<< HEAD 'fw-heudiconv-clear=fw_heudiconv.cli.clean:main' -======= - 'fw-heudiconv-clear=fw_heudiconv.cli.clear:main' ->>>>>>> 895d53d2d7287a6cca9c8ce7a38f9304aa45f018 ], } ) From 9495098697f2a793eef9d0ff79a89f3edb868c2e Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Thu, 19 Sep 2019 17:15:25 -0400 Subject: [PATCH 10/29] fix problem with None names turning up in acquisitions wihtout niftis --- fw_heudiconv/cli/clean.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fw_heudiconv/cli/clean.py b/fw_heudiconv/cli/clean.py index 5ff0301..5a561f6 100644 --- a/fw_heudiconv/cli/clean.py +++ b/fw_heudiconv/cli/clean.py @@ -13,7 +13,7 @@ logger = logging.getLogger('fw-heudiconv-cleaner') -def clear_bids(client, project_label, session_labels, subject_labels, dry_run=False, file_types = ['.nii', '.bval', '.bvec']): +def clear_bids(client, project_label, session_labels=None, subject_labels=None, dry_run=False, file_types = ['.nii', '.bval', '.bvec']): logger.info("\t\t=======: fw-heudiconv starting up :=======\n") logger.info("Querying Flywheel server...") @@ -63,7 +63,9 @@ def clear_bids(client, project_label, session_labels, subject_labels, dry_run=Fa for x in file_list: for k, v in x.items(): for u in v: - fnames.append(get_nested(u, 'info', 'BIDS', 'Filename')) + name = get_nested(u, 'info', 'BIDS', 'Filename') + if name is not None: + fnames.append(name) if file_list: logger.debug("This will remove BIDS data from %d files:\n\t%s" % (len(file_list), "\n\t".join([x for x in fnames]))) From 234b5a0b6eeeb5afa96ef5207c2ba2b88958e97e Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Fri, 20 Sep 2019 16:19:37 -0400 Subject: [PATCH 11/29] fix a bug that returns the subject "code" instead of "label" in queries --- fw_heudiconv/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fw_heudiconv/query.py b/fw_heudiconv/query.py index 0e37fdc..10b9059 100644 --- a/fw_heudiconv/query.py +++ b/fw_heudiconv/query.py @@ -72,7 +72,7 @@ def acquisition_to_heudiconv(client, acq, context): info.get("ProtocolName", ""), "MOCO" in info.get("ImageType", []), "DERIVED" in info.get("ImageType", []), - info.get("PatientID", context['subject'].code), + info.get("PatientID", context['subject'].label), info.get("StudyDescription"), info.get("ReferringPhysicianName", ""), info.get("SeriesDescription", ""), From d65ff5c82ba4ffea73790cb1fc8201ccaab7ba5f Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Fri, 20 Sep 2019 16:53:35 -0400 Subject: [PATCH 12/29] a real fix to the previous commit --- fw_heudiconv/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fw_heudiconv/query.py b/fw_heudiconv/query.py index 10b9059..589f80c 100644 --- a/fw_heudiconv/query.py +++ b/fw_heudiconv/query.py @@ -72,7 +72,7 @@ def acquisition_to_heudiconv(client, acq, context): info.get("ProtocolName", ""), "MOCO" in info.get("ImageType", []), "DERIVED" in info.get("ImageType", []), - info.get("PatientID", context['subject'].label), + context['subject'].label, info.get("StudyDescription"), info.get("ReferringPhysicianName", ""), info.get("SeriesDescription", ""), From 114e4766b2ee49f42d8457b30f8405d4136e743d Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Tue, 24 Sep 2019 10:33:55 -0400 Subject: [PATCH 13/29] renamed "clean" to "clear" --- fw_heudiconv/cli/{clean.py => clear.py} | 0 fw_heudiconv/convert.py | 3 ++- setup.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename fw_heudiconv/cli/{clean.py => clear.py} (100%) diff --git a/fw_heudiconv/cli/clean.py b/fw_heudiconv/cli/clear.py similarity index 100% rename from fw_heudiconv/cli/clean.py rename to fw_heudiconv/cli/clear.py diff --git a/fw_heudiconv/convert.py b/fw_heudiconv/convert.py index 3947931..d59bea7 100644 --- a/fw_heudiconv/convert.py +++ b/fw_heudiconv/convert.py @@ -50,7 +50,8 @@ def apply_heuristic(client, heur, acquisition_id, dry_run=False, intended_for=[] files = [f for f in acquisition_object.files if f.type in ftypes] bids_keys = ['sub', 'ses', 'folder', 'name'] - #ses_fmt = sess_label if sess_label.startswith("ses-") else "ses-" + sess_label + subj_label = subj_label if subj_label.startswith("sub-") else "sub-" + subj_label + sess_label = sess_label if sess_label.startswith("ses-") else "ses-" + sess_label files.sort(key=operator.itemgetter("name")) for fnum, f in enumerate(files): diff --git a/setup.py b/setup.py index d9b5dd3..835cae0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ 'fw-heudiconv-curate=fw_heudiconv.cli.curate:main', 'fw-heudiconv-export=fw_heudiconv.cli.export:main', 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main', - 'fw-heudiconv-clear=fw_heudiconv.cli.clean:main' + 'fw-heudiconv-clear=fw_heudiconv.cli.clear:main' ], } ) From b1afc0e6e0470c40022ad5351b2023e554bc5686 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 25 Sep 2019 13:51:21 -0400 Subject: [PATCH 14/29] init validate feature for #44 --- fw_heudiconv/cli/validate.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 fw_heudiconv/cli/validate.py diff --git a/fw_heudiconv/cli/validate.py b/fw_heudiconv/cli/validate.py new file mode 100644 index 0000000..ebfd92a --- /dev/null +++ b/fw_heudiconv/cli/validate.py @@ -0,0 +1,7 @@ +import flywheel + +def main(): + pass + +if __name__ == "__main__": + main() From e0b57dd434210c5f7e491415559214095e2b7188 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Wed, 25 Sep 2019 18:59:57 -0400 Subject: [PATCH 15/29] working version of validate tool --- fw_heudiconv/cli/export.py | 45 +++++++----- fw_heudiconv/cli/validate.py | 129 ++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 18 deletions(-) diff --git a/fw_heudiconv/cli/export.py b/fw_heudiconv/cli/export.py index 2e06d75..8381efb 100644 --- a/fw_heudiconv/cli/export.py +++ b/fw_heudiconv/cli/export.py @@ -177,7 +177,7 @@ def download_bids(client, to_download, root_path, folders_to_download = ['anat', if not any(x['name'] == '.bidsignore' for x in to_download['project']): # write bids ignore path = "/".join([root_path, ".bidsignore"]) - ignored_modalities = ['asl/\n', 'qsm/\n'] + ignored_modalities = ['asl/\n', 'qsm/\n', '*.bval', '*.bvec'] if dry_run: Path(path).touch() else: @@ -263,14 +263,12 @@ def download_bids(client, to_download, root_path, folders_to_download = ['anat', logger.info("Done!") print_directory_tree(root_path) - if dry_run: - shutil.rmtree(root_path) def get_parser(): parser = argparse.ArgumentParser( - description="Export BIDS compliant data") + description="Export BIDS-curated data from Flywheel") parser.add_argument( "--project", help="The project in flywheel", @@ -279,39 +277,44 @@ def get_parser(): ) parser.add_argument( "--path", - help="The target directory to download", - required=True, - default="." + help="The target directory to download [DEPRECATED. PLEASE USE INSTEAD]", + default=None ) parser.add_argument( "--subject", - help="The subject to curate", + help="The subject(s) to export", nargs="+", default=None, type=str ) parser.add_argument( "--session", - help="The session to curate", + help="The session(s) to export", nargs="+", default=None, type=str ) parser.add_argument( "--folders", - help="The BIDS folders to download", + help="The BIDS folders to export", nargs="+", default=['anat', 'dwi', 'fmap', 'func'] ) parser.add_argument( "--dry_run", - help="Don't apply changes", + help="Don't apply changes (only print the directory tree to the console)", action='store_true', default=False ) + parser.add_argument( + "--destination", + help="Path to destination directory", + default=".", + type=str + ) parser.add_argument( "--directory_name", - help="Name of the directory", + help="Name of destination directory", default="bids_directory", type=str ) @@ -329,18 +332,26 @@ def main(): args = parser.parse_args() project_label = ' '.join(args.project) - if not os.path.exists(args.path): - logger.info("Creating target directory...") - os.makedirs(args.path) - #assert os.path.exists(args.path), "Path does not exist!" + + if args.path: + destination = args.path + else: + destination = args.destination + + if not os.path.exists(destination): + logger.info("Creating destination directory...") + os.makedirs(args.destination) + downloads = gather_bids(client=fw, project_label=project_label, session_labels=args.session, subject_labels=args.subject ) - download_bids(client=fw, to_download=downloads, root_path=args.path, folders_to_download=args.folders, dry_run=args.dry_run, name=args.directory_name) + download_bids(client=fw, to_download=downloads, root_path=destination, folders_to_download=args.folders, dry_run=args.dry_run, name=args.directory_name) + if args.dry_run: + shutil.rmtree(Path(args.destination, args.directory_name)) if __name__ == '__main__': main() diff --git a/fw_heudiconv/cli/validate.py b/fw_heudiconv/cli/validate.py index ebfd92a..8680e02 100644 --- a/fw_heudiconv/cli/validate.py +++ b/fw_heudiconv/cli/validate.py @@ -1,7 +1,134 @@ import flywheel +import warnings +import os +import sys +import shutil +import subprocess as sub +import logging +import argparse +from pathlib import Path + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger('fw-heudiconv-validator') + + +def validate_local(path, verbose): + + command = ['bids-validator', path] + if verbose: + command.extend(['--verbose']) + p = sub.Popen(command, stdout=sub.PIPE, stdin=sub.PIPE, stderr=sub.PIPE, universal_newlines=True) + output, error = p.communicate() + + logger.info(output) + if p.returncode != 0: + logger.info(error) + return p.returncode + + +def fw_heudiconv_export(proj, subjects=None, sessions=None, destination="tmp", name="bids_directory"): + + command = ['fw-heudiconv-export', '--project', ' '.join(proj), '--destination', destination, '--directory_name', name] + + if subjects: + command.extend(['--subject'] + subjects) + if sessions: + command.extend(['--session'] + sessions) + + p = sub.Popen(command, stdout=sub.PIPE, stdin=sub.PIPE, stderr=sub.PIPE, universal_newlines=True) + output, error = p.communicate() + + logger.info(output) + if p.returncode != 0: + logger.info(error) + return p.returncode + + +def get_parser(): + + parser = argparse.ArgumentParser( + description="Validate BIDS-curated data on Flywheel. A simple wrapper around the original BIDS Validator https://github.com/bids-standard/bids-validator") + + location = parser.add_mutually_exclusive_group(required=True) + location.add_argument( + '--local', + help="Validate a local directory of BIDS data", + action='store_true') + location.add_argument( + '--flywheel', + help="Validate a BIDS project on Flywheel", + action='store_true') + + parser.add_argument( + "--directory", + help="Path to existing BIDS data directory OR temp space used for validation", + default="bids_directory", + required=False, + type=str + ) + parser.add_argument( + "--project", + help="The project on Flywheel", + nargs="+" + ) + parser.add_argument( + "--subject", + help="The subject(s) on Flywheel to validate", + nargs="+", + default=None, + type=str + ) + parser.add_argument( + "--session", + help="The session(s) on Flywheel to validate", + nargs="+", + default=None, + type=str + ) + parser.add_argument( + "--verbose", + help="Pass on flag to bids-validator", + default=False, + action='store_true' + ) + + return parser + + def main(): - pass + + parser = get_parser() + + args = parser.parse_args() + + exit = 1 + + if args.local: + if not os.path.exists(args.directory): + logger.error("Couldn't find the BIDS dataset!") + sys.exit(exit) + else: + logger.info("Validating local BIDS dataset {}".format(args.directory)) + exit = validate_local(args.directory, args.verbose) + + else: + if not os.path.exists(args.directory): + logger.info("Creating download directory...") + os.makedirs(args.directory) + + if not args.project: + logger.error("No project on Flywheel specified!") + sys.exit(exit) + + success = fw_heudiconv_export(proj=args.project, subjects=args.subject, sessions=args.session, destination=args.directory, name='bids_directory') + + if success == 0: + path = Path(args.directory + '/bids_directory') + validate_local(path, args.verbose) + shutil.rmtree(path) + + sys.exit(exit) if __name__ == "__main__": main() From 450a652164e31fc1420ed62439ae65e2537b0f00 Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Thu, 26 Sep 2019 11:46:31 -0400 Subject: [PATCH 16/29] Works! Ready to merge into development branch --- fw_heudiconv/cli/export.py | 2 +- fw_heudiconv/cli/validate.py | 16 +++++++++++----- setup.py | 3 ++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/fw_heudiconv/cli/export.py b/fw_heudiconv/cli/export.py index 8381efb..6580062 100644 --- a/fw_heudiconv/cli/export.py +++ b/fw_heudiconv/cli/export.py @@ -177,7 +177,7 @@ def download_bids(client, to_download, root_path, folders_to_download = ['anat', if not any(x['name'] == '.bidsignore' for x in to_download['project']): # write bids ignore path = "/".join([root_path, ".bidsignore"]) - ignored_modalities = ['asl/\n', 'qsm/\n', '*.bval', '*.bvec'] + ignored_modalities = ['asl/\n', 'qsm/\n', '*.bval\n', '*.bvec\n'] if dry_run: Path(path).touch() else: diff --git a/fw_heudiconv/cli/validate.py b/fw_heudiconv/cli/validate.py index 8680e02..d5a800a 100644 --- a/fw_heudiconv/cli/validate.py +++ b/fw_heudiconv/cli/validate.py @@ -14,6 +14,7 @@ def validate_local(path, verbose): + logger.info("Launching bids-validator...") command = ['bids-validator', path] if verbose: command.extend(['--verbose']) @@ -28,6 +29,7 @@ def validate_local(path, verbose): def fw_heudiconv_export(proj, subjects=None, sessions=None, destination="tmp", name="bids_directory"): + logger.info("Launching fw-heudiconv-export...") command = ['fw-heudiconv-export', '--project', ' '.join(proj), '--destination', destination, '--directory_name', name] if subjects: @@ -113,21 +115,25 @@ def main(): exit = validate_local(args.directory, args.verbose) else: - if not os.path.exists(args.directory): - logger.info("Creating download directory...") - os.makedirs(args.directory) - if not args.project: logger.error("No project on Flywheel specified!") sys.exit(exit) + if not os.path.exists(args.directory): + logger.info("Creating download directory...") + os.makedirs(args.directory) + success = fw_heudiconv_export(proj=args.project, subjects=args.subject, sessions=args.session, destination=args.directory, name='bids_directory') if success == 0: path = Path(args.directory + '/bids_directory') - validate_local(path, args.verbose) + exit = validate_local(path, args.verbose) shutil.rmtree(path) + else: + + logger.error("There was a problem downloading the data to a temp space for validation!") + logger.info("Done!") sys.exit(exit) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 835cae0..cfffbb0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,8 @@ 'fw-heudiconv-curate=fw_heudiconv.cli.curate:main', 'fw-heudiconv-export=fw_heudiconv.cli.export:main', 'fw-heudiconv-tabulate=fw_heudiconv.cli.tabulate:main', - 'fw-heudiconv-clear=fw_heudiconv.cli.clear:main' + 'fw-heudiconv-clear=fw_heudiconv.cli.clear:main', + 'fw-heudiconv-validate=fw_heudiconv.cli.validate:main' ], } ) From d09d9b809a041712bf49ba031cca74962196ffed Mon Sep 17 00:00:00 2001 From: TinasheMTapera Date: Fri, 27 Sep 2019 17:34:02 -0400 Subject: [PATCH 17/29] added function for fetching bids labels from session objects --- fw_heudiconv/cli/curate.py | 2 +- fw_heudiconv/cli/meta.py | 187 +++++++++++++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 fw_heudiconv/cli/meta.py diff --git a/fw_heudiconv/cli/curate.py b/fw_heudiconv/cli/curate.py index 271506c..c3a9016 100644 --- a/fw_heudiconv/cli/curate.py +++ b/fw_heudiconv/cli/curate.py @@ -130,7 +130,7 @@ def convert_to_bids(client, project_label, heuristic_path, subject_labels=None, def get_parser(): parser = argparse.ArgumentParser( - description="Use a heudiconv heuristic to curate bids on flywheel") + description="Use a heudiconv heuristic to curate data into BIDS on flywheel") parser.add_argument( "--project", help="The project in flywheel", diff --git a/fw_heudiconv/cli/meta.py b/fw_heudiconv/cli/meta.py new file mode 100644 index 0000000..efd67ad --- /dev/null +++ b/fw_heudiconv/cli/meta.py @@ -0,0 +1,187 @@ +import os +import sys +import importlib +import argparse +import warnings +import flywheel +import pprint +import logging +import re +from ..convert import get_nested + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger('fw-heudiconv-curator') + + +def get_BIDS_label_from_session(ses_object, regex='sub'): + + logger.debug("Processing session {}...".format(ses_object.label)) + acquisitions = ses_object.acquisitions() + pattern = re.compile(r"(?<={}-)[a-zA-z0-9]+(?=_)".format(regex)) + + bids_labels = [] + for x in acquisitions: + files = [f for f in x.files if "nifti" in f.type] + for y in files: + bids_labels.append(get_nested(y.to_dict(), 'info', 'BIDS', 'Filename')) + if bids_labels: + for b in range(len(bids_labels)): + if bids_labels[b] is not None: + label = pattern.search(bids_labels[b]) + if label: + bids_labels[b] = label.group() + else: + bids_labels[b] = None + + final_label = set(filter(None, bids_labels)) + if len(final_label) == 1: + return final_label.pop() + else: + return None + + + +def initialise_dataset(client, project_label, subject_labels=None, session_labels=None, dry_run=True): + + if dry_run: + logger.setLevel(logging.DEBUG) + logger.info("Querying Flywheel server...") + project_obj = client.projects.find_first('label="{}"'.format(project_label)) + assert project_obj, "Project not found! Maybe check spelling...?" + logger.debug('Found project: %s (%s)', project_obj['label'], project_obj.id) + sessions = client.get_project_sessions(project_obj.id) + # filters + if subject_labels: + sessions = [s for s in sessions if s.subject['label'] in subject_labels] + if session_labels: + sessions = [s for s in sessions if s.label in session_labels] + + return sessions + + +def get_parser(): + + parser = argparse.ArgumentParser( + description="Curate BIDS metadata on Flywheel\n\nSee the BIDS spec for details: https://bids-specification.readthedocs.io/en/stable/03-modality-agnostic-files.html", + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--project", + help="The project in flywheel", + required=True + ) + parser.add_argument( + "--subject", + help="The subject label(s)", + nargs="+", + default=None + ) + parser.add_argument( + "--session", + help="The session label(s)", + nargs="+", + default=None + ) + parser.add_argument( + "--verbose", + help="Print ongoing messages of progress", + action='store_true', + default=False + ) + parser.add_argument( + "--dry-run", + help="Don't apply changes", + action='store_true', + default=False + ) + + # participants metadata + participants_meta = parser.add_mutually_exclusive_group() + participants_meta.add_argument( + "--autogen-participants-meta", + help="Automatically generate participants.tsv metadata", + action='store_true', + default=False + ) + participants_meta.add_argument( + "--upload-participants-meta", + help="Path to a participants.tsv metadata file to upload", + action='store' + ) + + # sessions metadata + sessions_meta = parser.add_mutually_exclusive_group() + sessions_meta.add_argument( + "--autogen-sessions-meta", + help="Automatically generate sub-