From 50574086eb8e40b424778250089f994076e24c7c Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 6 Nov 2017 12:54:38 -0800 Subject: [PATCH 01/56] update bids validator --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 11c70d6..1012065 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ apt-get install -y nodejs && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN npm install -g bids-validator@0.19.2 +RUN npm install -g bids-validator@0.23.11 # Download FreeSurfer RUN apt-get -y update \ From 4670419e02c45757e77bd4143abab8a8793ffa85 Mon Sep 17 00:00:00 2001 From: Joke Durnez Date: Wed, 29 Nov 2017 11:07:02 -0800 Subject: [PATCH 02/56] update bids validator --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1012065..0441233 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ apt-get install -y nodejs && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN npm install -g bids-validator@0.23.11 +RUN npm install -g bids-validator@0.24.00 # Download FreeSurfer RUN apt-get -y update \ From 43f1541519d0b71993a5b7e66000013616eeb07f Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 29 Nov 2017 14:28:51 -0800 Subject: [PATCH 03/56] implement diffusion preprocessing --- run.py | 65 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/run.py b/run.py index 0de1026..2e95c9e 100644 --- a/run.py +++ b/run.py @@ -412,28 +412,43 @@ def run_diffusion_processsing(**args): dwis = layout.get(subject=subject_label, type='dwi', extensions=["nii.gz", "nii"]) - # print(dwis) - # acqs = set(layout.get(target='acquisition', return_type='id', - # subject=subject_label, type='dwi', - # extensions=["nii.gz", "nii"])) - # print(acqs) - # posData = [] - # negData = [] - # for acq in acqs: - # pos = "EMPTY" - # neg = "EMPTY" - # dwis = layout.get(subject=subject_label, - # type='dwi', acquisition=acq, - # extensions=["nii.gz", "nii"]) - # assert len(dwis) <= 2 - # for dwi in dwis: - # dwi = dwi.filename - # if "-" in layout.get_metadata(dwi)["PhaseEncodingDirection"]: - # neg = dwi - # else: - # pos = dwi - # posData.append(pos) - # negData.append(neg) - # - # print(negData) - # print(posData) + pos = []; neg = [] + PEdir = None; echospacing = None + + for idx,dwi in enumerate(dwis): + metadata = layout.get_metadata(dwi.filename) + # get phaseencodingdirection + phaseenc = metadata['PhaseEncodingDirection'] + print("hase: %s - ind: %i"%(phaseenc,idx)) + acq = 1 if phaseenc[0]=='i' else 2 + if not PEdir: + PEdir = acq + if PEdir != acq: + raise RuntimeError("Not all dwi images have the same encoding direction (both LR and AP). Not implemented.") + # get pos/neg + if len(phaseenc)>1: + neg.append(dwi.filename) + else: + pos.append(dwi.filename) + # get echospacing + if not echospacing: + echospacing = metadata['EffectiveEchoSpacing'] + if echospacing != metadata['EffectiveEchoSpacing']: + raise RuntimeError("Not all dwi images have the same echo spacing. Not implemented.") + + posdata = "@".join(pos) + negdata = "@".join(neg) + + dif_stages_dict = OrderedDict([("DiffusionPreprocessing", partial(run_diffusion_processsing, + path=os.path.join(args.bids_dir,"sub-%s"%subject_label), + subject="sub-%s"%subject_label, + posData=posdata, + negData=negdata, + echospacing=echospacing, + n_cpus=args.n_cpus, + PEdir=PEdir)) + ]) + + for stage, stage_func in dif_stages_dict.iteritems(): + if stage in args.stages: + stage_func() From c1da28f7bb9e27b93730643ed3de2413176aca21 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 29 Nov 2017 14:40:14 -0800 Subject: [PATCH 04/56] debug outdir --- run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 2e95c9e..9a9de1a 100644 --- a/run.py +++ b/run.py @@ -168,7 +168,7 @@ def run_diffusion_processsing(**args): __version__ = open('/version').read() -parser = argparse.ArgumentParser(description='HCP Pipeliens BIDS App (T1w, T2w, fMRI)') +parser = argparse.ArgumentParser(description='HCP Pipelines BIDS App (T1w, T2w, fMRI, DWI)') parser.add_argument('bids_dir', help='The directory with the input dataset ' 'formatted according to the BIDS standard.') parser.add_argument('output_dir', help='The directory where the output files ' @@ -197,7 +197,7 @@ def run_diffusion_processsing(**args): parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', required=True) parser.add_argument('-v', '--version', action='version', - version='HCP Pielines BIDS App version {}'.format(__version__)) + version='HCP Pipelines BIDS App version {}'.format(__version__)) args = parser.parse_args() @@ -440,7 +440,7 @@ def run_diffusion_processsing(**args): negdata = "@".join(neg) dif_stages_dict = OrderedDict([("DiffusionPreprocessing", partial(run_diffusion_processsing, - path=os.path.join(args.bids_dir,"sub-%s"%subject_label), + path=args.output_dir, subject="sub-%s"%subject_label, posData=posdata, negData=negdata, From fe69ca581daeafcfbef6f629d2f7650d625606b4 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 29 Nov 2017 15:13:57 -0800 Subject: [PATCH 05/56] echo spacing units --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 0d5d4df..8c9216e 100644 --- a/run.py +++ b/run.py @@ -444,7 +444,7 @@ def run_diffusion_processsing(**args): subject="sub-%s"%subject_label, posData=posdata, negData=negdata, - echospacing=echospacing, + echospacing=echospacing * 1000., n_cpus=args.n_cpus, PEdir=PEdir)) ]) From 41bd5f560a58a0adac1d9b940267e47ba5168052 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 6 Dec 2017 19:22:53 -0800 Subject: [PATCH 06/56] add data_is_shelled for dwi --- run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/run.py b/run.py index 8c9216e..e8ec163 100644 --- a/run.py +++ b/run.py @@ -162,6 +162,7 @@ def run_diffusion_processsing(**args): '--echospacing="{echospacing}" '+ \ '--PEdir={PEdir} ' + \ '--gdcoeffs="NONE" ' + \ + '--extra-eddy-arg="--data_is_shelled" ' + \ '--printcom=""' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) From cc0eabcd5ea6aefb50fb0cff9bf1458b8b774f99 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 6 Dec 2017 19:25:20 -0800 Subject: [PATCH 07/56] units of echo spacing --- run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index e8ec163..5b56c38 100644 --- a/run.py +++ b/run.py @@ -433,8 +433,8 @@ def run_diffusion_processsing(**args): pos.append(dwi.filename) # get echospacing if not echospacing: - echospacing = metadata['EffectiveEchoSpacing'] - if echospacing != metadata['EffectiveEchoSpacing']: + echospacing = metadata['EffectiveEchoSpacing']*1000. + if echospacing != metadata['EffectiveEchoSpacing']*1000.: raise RuntimeError("Not all dwi images have the same echo spacing. Not implemented.") posdata = "@".join(pos) From e205d7d6d9d82feb9b72e453cf5cdef469f38eca Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 6 Dec 2017 19:27:20 -0800 Subject: [PATCH 08/56] update pybids --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0441233..defbebb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,7 @@ ENV MSMBin=${HCPPIPEDIR}/MSMBinaries RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN pip install pybids==0.0.1 +RUN pip install pybids==0.3.0 ENV PYTHONPATH="" COPY run.py /run.py From 278ecfbd9ae061f8ec570810f6cdde2fae6f2fa4 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 6 Dec 2017 19:27:43 -0800 Subject: [PATCH 09/56] add eddy --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index defbebb..2bc6e52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,5 +111,9 @@ ENV PYTHONPATH="" COPY run.py /run.py RUN chmod +x /run.py +RUN wget -qO- https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/eddy-patch-fsl-5.0.9/centos6/eddy_cuda7.5 > $FSLDIR/bin/eddy_cuda +RUN wget -qO- https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/eddy-patch-fsl-5.0.9/centos6/eddy_openmp > $FSLDIR/bin/eddy_openmp +RUN chmod 775 $FSLDIR/bin/eddy_* + COPY version /version ENTRYPOINT ["/run.py"] From 4e596fb01f062dd9fe5a0d16fd61e853b8890ef7 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 7 Dec 2017 11:29:24 -0800 Subject: [PATCH 10/56] return list of fieldmaps, debug echospacing --- run.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 5b56c38..2ab2b50 100644 --- a/run.py +++ b/run.py @@ -163,6 +163,7 @@ def run_diffusion_processsing(**args): '--PEdir={PEdir} ' + \ '--gdcoeffs="NONE" ' + \ '--extra-eddy-arg="--data_is_shelled" ' + \ + '--no-gpu ' + \ '--printcom=""' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) @@ -204,7 +205,7 @@ def run_diffusion_processsing(**args): -run("bids-validator " + args.bids_dir) +#run("bids-validator " + args.bids_dir) layout = BIDSLayout(args.bids_dir) subjects_to_analyze = [] @@ -237,7 +238,7 @@ def run_diffusion_processsing(**args): t2_res = float(min(t2_zooms[:3])) t2_template_res = min(available_resolutions, key=lambda x:abs(float(x)-t2_res)) - fieldmap_set = layout.get_fieldmap(t1ws[0]) + fieldmap_set = layout.get_fieldmap(t1ws[0],return_list=True) fmap_args = {"fmapmag": "NONE", "fmapphase": "NONE", "echodiff": "NONE", @@ -445,7 +446,7 @@ def run_diffusion_processsing(**args): subject="sub-%s"%subject_label, posData=posdata, negData=negdata, - echospacing=echospacing * 1000., + echospacing=echospacing, n_cpus=args.n_cpus, PEdir=PEdir)) ]) From 1d87f1dd58aedf6ea8c7dfdb2533790d90d6eb51 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 7 Dec 2017 12:29:34 -0800 Subject: [PATCH 11/56] use jokes pipelines to check --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2bc6e52..9d34ba3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,7 @@ ENV CARET7DIR=/usr/bin # Install HCP Pipelines RUN apt-get -y update \ && apt-get install -y --no-install-recommends python-numpy && \ - wget https://github.com/Washington-University/Pipelines/archive/v3.17.0.tar.gz -O pipelines.tar.gz && \ + wget https://github.com/jokedurnez/Pipelines/archive/v3.23.0-jd.tar.gz -O pipelines.tar.gz && \ cd /opt/ && \ tar zxvf /pipelines.tar.gz && \ mv /opt/Pipelines-* /opt/HCP-Pipelines && \ From 3a30eaa4ea6e427c8d01c33e5245543f231d4365 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 8 Dec 2017 13:59:50 -0800 Subject: [PATCH 12/56] adapt to pybids 0.4.2 + add checks for consistency --- Dockerfile | 2 +- run.py | 111 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 63 insertions(+), 50 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9d34ba3..e5ebde2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,7 @@ ENV MSMBin=${HCPPIPEDIR}/MSMBinaries RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN pip install pybids==0.3.0 +RUN pip install pybids==0.4.2 ENV PYTHONPATH="" COPY run.py /run.py diff --git a/run.py b/run.py index 2ab2b50..98bf28d 100644 --- a/run.py +++ b/run.py @@ -252,6 +252,8 @@ def run_diffusion_processsing(**args): "seunwarpdir": "NONE"} if fieldmap_set: + fieldmap_trans = dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) + t1_spacing = layout.get_metadata(t1ws[0])["EffectiveEchoSpacing"] t2_spacing = layout.get_metadata(t2ws[0])["EffectiveEchoSpacing"] @@ -264,57 +266,66 @@ def run_diffusion_processsing(**args): "t2samplespacing": "%.8f"%t2_spacing, "unwarpdir": unwarpdir}) - if fieldmap_set["type"] == "phasediff": + if set(fieldmap_trans['type'])==set(['phasediff']): merged_file = "%s/tmp/%s/magfile.nii.gz"%(args.output_dir, subject_label) run("mkdir -p %s/tmp/%s/ && fslmerge -t %s %s %s"%(args.output_dir, subject_label, merged_file, - fieldmap_set["magnitude1"], - fieldmap_set["magnitude2"])) + fieldmap_trans["magnitude1"], + fieldmap_trans["magnitude2"])) - phasediff_metadata = layout.get_metadata(fieldmap_set["phasediff"]) + phasediff_metadata = layout.get_metadata(fieldmap_trans["phasediff"]) te_diff = phasediff_metadata["EchoTime2"] - phasediff_metadata["EchoTime1"] # HCP expects TE in miliseconds te_diff = te_diff*1000.0 fmap_args.update({"fmapmag": merged_file, - "fmapphase": fieldmap_set["phasediff"], + "fmapphase": fieldmap_trans["phasediff"], "echodiff": "%.6f"%te_diff, "avgrdcmethod": "SiemensFieldMap"}) - elif fieldmap_set["type"] == "epi": + + elif set(fieldmap_trans['type'])== set(["epi"]): SEPhaseNeg = None SEPhasePos = None - for fieldmap in fieldmap_set["epi"]: + seunwarpdir = None + for fieldmap in fieldmap_trans["epi"]: enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] if "-" in enc_dir: SEPhaseNeg = fieldmap else: SEPhasePos = fieldmap - seunwarpdir = layout.get_metadata(fieldmap_set["epi"][0])["PhaseEncodingDirection"] - seunwarpdir = seunwarpdir.replace("-", "").replace("i","x").replace("j", "y").replace("k", "z") - - #TODO check consistency of echo spacing instead of assuming it's all the same - if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap_set["epi"][0]): - echospacing = layout.get_metadata(fieldmap_set["epi"][0])["EffectiveEchoSpacing"] - elif "TotalReadoutTime" in layout.get_metadata(fieldmap_set["epi"][0]): - # HCP Pipelines do not allow users to specify total readout time directly - # Hence we need to reverse the calculations to provide echo spacing that would - # result in the right total read out total read out time - # see https://github.com/Washington-University/Pipelines/blob/master/global/scripts/TopupPreprocessingAll.sh#L202 - print("BIDS App wrapper: Did not find EffectiveEchoSpacing, calculating it from TotalReadoutTime") - # TotalReadoutTime = EffectiveEchoSpacing * (len(PhaseEncodingDirection) - 1) - total_readout_time = layout.get_metadata(fieldmap_set["epi"][0])["TotalReadoutTime"] - phase_len = nibabel.load(fieldmap_set["epi"][0]).shape[{"x": 0, "y": 1}[seunwarpdir]] - echospacing = total_readout_time / float(phase_len - 1) - else: - raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") + unwarpdir = enc_dir.replace("-", "").replace("i","x").replace("j", "y").replace("k", "z") + if seunwarpdir and not seunwarpdir == unwarpdir: + raise RuntimeError("Inconsistent unwarp directions.") + + fmap_arguments = {} + for fieldmap in fieldmap_set: + if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap["epi"]): + echospacing = layout.get_metadata(fieldmap["epi"])["EffectiveEchoSpacing"] + elif "TotalReadoutTime" in layout.get_metadata(fieldmap["epi"]): + # HCP Pipelines do not allow users to specify total readout time directly + # Hence we need to reverse the calculations to provide echo spacing that would + # result in the right total read out total read out time + # see https://github.com/Washington-University/Pipelines/blob/master/global/scripts/TopupPreprocessingAll.sh#L202 + print("BIDS App wrapper: Did not find EffectiveEchoSpacing, calculating it from TotalReadoutTime") + # TotalReadoutTime = EffectiveEchoSpacing * (len(PhaseEncodingDirection) - 1) + total_readout_time = layout.get_metadata(fieldmap["epi"])["TotalReadoutTime"] + phase_len = nibabel.load(fieldmap["epi"]).shape[{"x": 0, "y": 1}[seunwarpdir]] + echospacing = total_readout_time / float(phase_len - 1) + else: + raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") + if 'echospacing' in fmap_arguments.keys() and not fmap_arguments['echospacing']==echospacing: + raise RuntimeError("Inconsistent echospacing.") fmap_args.update({"SEPhaseNeg": SEPhaseNeg, "SEPhasePos": SEPhasePos, "echospacing": "%.6f"%echospacing, "seunwarpdir": seunwarpdir, "avgrdcmethod": "TOPUP"}) + else: + raise RuntimeError("Inconsistent fieldmap types or unknown type.") + #TODO add support for GE fieldmaps struct_stages_dict = OrderedDict([("PreFreeSurfer", partial(run_pre_freesurfer, @@ -352,30 +363,32 @@ def run_diffusion_processsing(**args): if not os.path.exists(fmriscout): fmriscout = "NONE" - fieldmap_set = layout.get_fieldmap(fmritcs) - if fieldmap_set and fieldmap_set["type"] == "epi": - SEPhaseNeg = None - SEPhasePos = None - for fieldmap in fieldmap_set["epi"]: - enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] - if "-" in enc_dir: - SEPhaseNeg = fieldmap - else: - SEPhasePos = fieldmap - echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] - unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] - unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") - if len(unwarpdir) == 2: - unwarpdir = "-" + unwarpdir[0] - dcmethod = "TOPUP" - biascorrection = "SEBASED" - else: - SEPhaseNeg = "NONE" - SEPhasePos = "NONE" - echospacing = "NONE" - unwarpdir = "NONE" - dcmethod = "NONE" - biascorrection = "NONE" + fieldmap_set = layout.get_fieldmap(fmritcs,return_list=True) + if fieldmap_set: + fieldmap_trans = dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) + if set(fieldmap_trans['type'])== set(["epi"]): + SEPhaseNeg = None + SEPhasePos = None + for fieldmap in fieldmap_set: + enc_dir = layout.get_metadata(fieldmap['epi'])["PhaseEncodingDirection"] + if "-" in enc_dir: + SEPhaseNeg = fieldmap['epi'] + else: + SEPhasePos = fieldmap['epi'] + echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] + unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] + unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") + if len(unwarpdir) == 2: + unwarpdir = "-" + unwarpdir[0] + dcmethod = "TOPUP" + biascorrection = "SEBASED" + else: + SEPhaseNeg = "NONE" + SEPhasePos = "NONE" + echospacing = "NONE" + unwarpdir = "NONE" + dcmethod = "NONE" + biascorrection = "NONE" zooms = nibabel.load(fmritcs).get_header().get_zooms() fmrires = float(min(zooms[:3])) From 49d523d0b4d258c750b73216d52b96f901820b66 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 8 Dec 2017 14:29:52 -0800 Subject: [PATCH 13/56] debug when fieldmaps are magnitude (hcp test) --- run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/run.py b/run.py index 98bf28d..3bc8332 100644 --- a/run.py +++ b/run.py @@ -252,7 +252,10 @@ def run_diffusion_processsing(**args): "seunwarpdir": "NONE"} if fieldmap_set: - fieldmap_trans = dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) + if len(fieldmap_set)>1: + fieldmap_trans = dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) + else: + fieldmap_trans = {k:[v] for k,v in fieldmap_set[0].iteritems()} t1_spacing = layout.get_metadata(t1ws[0])["EffectiveEchoSpacing"] t2_spacing = layout.get_metadata(t2ws[0])["EffectiveEchoSpacing"] @@ -271,10 +274,10 @@ def run_diffusion_processsing(**args): run("mkdir -p %s/tmp/%s/ && fslmerge -t %s %s %s"%(args.output_dir, subject_label, merged_file, - fieldmap_trans["magnitude1"], - fieldmap_trans["magnitude2"])) + " ".join(fieldmap_trans["magnitude1"]), + " ".join(fieldmap_trans["magnitude2"]))) - phasediff_metadata = layout.get_metadata(fieldmap_trans["phasediff"]) + phasediff_metadata = layout.get_metadata(fieldmap_trans["phasediff"][0]) te_diff = phasediff_metadata["EchoTime2"] - phasediff_metadata["EchoTime1"] # HCP expects TE in miliseconds te_diff = te_diff*1000.0 From 25af78fe415ab683b77e0c69e62da3c1496fc5a4 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 8 Dec 2017 16:51:42 -0800 Subject: [PATCH 14/56] debug fieldmap with magnitude --- run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 3bc8332..e2f884d 100644 --- a/run.py +++ b/run.py @@ -283,7 +283,7 @@ def run_diffusion_processsing(**args): te_diff = te_diff*1000.0 fmap_args.update({"fmapmag": merged_file, - "fmapphase": fieldmap_trans["phasediff"], + "fmapphase": fieldmap_trans["phasediff"][0], "echodiff": "%.6f"%te_diff, "avgrdcmethod": "SiemensFieldMap"}) @@ -301,6 +301,8 @@ def run_diffusion_processsing(**args): unwarpdir = enc_dir.replace("-", "").replace("i","x").replace("j", "y").replace("k", "z") if seunwarpdir and not seunwarpdir == unwarpdir: raise RuntimeError("Inconsistent unwarp directions.") + else: + seunwarpdir = copy.deepcopy(unwarpdir) fmap_arguments = {} for fieldmap in fieldmap_set: From 1af01237938001f3cb863a513506c7cc10462dbc Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 8 Dec 2017 18:01:07 -0800 Subject: [PATCH 15/56] debug consistency check --- run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/run.py b/run.py index e2f884d..aa03c29 100644 --- a/run.py +++ b/run.py @@ -4,6 +4,7 @@ import os import shutil import nibabel +import copy from glob import glob from subprocess import Popen, PIPE from shutil import rmtree From e3c02949b4832939f9aaf4606be671531dfc4caa Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 8 Dec 2017 19:40:53 -0800 Subject: [PATCH 16/56] restore validator --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index aa03c29..921d4f8 100644 --- a/run.py +++ b/run.py @@ -206,7 +206,7 @@ def run_diffusion_processsing(**args): -#run("bids-validator " + args.bids_dir) +run("bids-validator " + args.bids_dir) layout = BIDSLayout(args.bids_dir) subjects_to_analyze = [] From a316145d18cd176f6fb94ad549d4d61157ac403d Mon Sep 17 00:00:00 2001 From: Joke Durnez Date: Fri, 9 Feb 2018 15:42:59 -0800 Subject: [PATCH 17/56] update bids validator --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e5ebde2..993f438 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ apt-get install -y nodejs && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN npm install -g bids-validator@0.24.00 +RUN npm install -g bids-validator@0.25.07 # Download FreeSurfer RUN apt-get -y update \ From 2497e158dd3b184b9e7696db90a39eafddf91144 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 21 Feb 2018 13:52:34 -0800 Subject: [PATCH 18/56] update fsl to 5.0.10 to get surface processing working --- Dockerfile | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 993f438..a522dcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,27 +49,23 @@ ENV PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 ENV MNI_PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH -# Install FSL 5.0.9 -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl && \ - curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update && \ - apt-get install -y fsl-core=5.0.9-4~nd14.04+1 && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Install FSL 5.0.10 +RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \ + python fslinstaller.py -d /usr/local/fsl -D && \ + chmod +x $FSLDIR/etc/fslconf/fsl.sh && \ + $FSLDIR/etc/fslconf/fsl.sh -# Configure environment -ENV FSLDIR=/usr/share/fsl/5.0 +ENV FSLDIR=/usr/local/fsl +ENV PATH=$FSLDIR/bin:$PATH ENV FSL_DIR="${FSLDIR}" ENV FSLOUTPUTTYPE=NIFTI_GZ -ENV PATH=/usr/lib/fsl/5.0:$PATH +ENV PATH=$FSLDIR:$PATH ENV FSLMULTIFILEQUIT=TRUE -ENV POSSUMDIR=/usr/share/fsl/5.0 -ENV LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH +ENV POSSUMDIR=$FSLDIR +ENV LD_LIBRARY_PATH=$FSLDIR:$LD_LIBRARY_PATH ENV FSLTCLSH=/usr/bin/tclsh ENV FSLWISH=/usr/bin/wish ENV FSLOUTPUTTYPE=NIFTI_GZ -RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh # Install Connectome Workbench RUN apt-get update && apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 @@ -111,9 +107,5 @@ ENV PYTHONPATH="" COPY run.py /run.py RUN chmod +x /run.py -RUN wget -qO- https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/eddy-patch-fsl-5.0.9/centos6/eddy_cuda7.5 > $FSLDIR/bin/eddy_cuda -RUN wget -qO- https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/eddy-patch-fsl-5.0.9/centos6/eddy_openmp > $FSLDIR/bin/eddy_openmp -RUN chmod 775 $FSLDIR/bin/eddy_* - COPY version /version ENTRYPOINT ["/run.py"] From 105f768b5e14a2a6dc00466828ca9a6739a7f4ac Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 21 Feb 2018 14:01:52 -0800 Subject: [PATCH 19/56] set MSMbindir, aka the ony reason why I'm doing these changes --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a522dcf..dad086b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools: # Install FSL 5.0.10 RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \ - python fslinstaller.py -d /usr/local/fsl -D && \ + python fslinstaller.py -d /usr/local/fsl && \ chmod +x $FSLDIR/etc/fslconf/fsl.sh && \ $FSLDIR/etc/fslconf/fsl.sh @@ -97,7 +97,7 @@ ENV HCPPIPEDIR_dMRI=${HCPPIPEDIR}/DiffusionPreprocessing/scripts ENV HCPPIPEDIR_dMRITract=${HCPPIPEDIR}/DiffusionTractography/scripts ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts -ENV MSMBin=${HCPPIPEDIR}/MSMBinaries +ENV MSMBINDIR=${FSLDIR}/bin/ RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 16832a88828e6be92598e86d209dde362d71e57d Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 21 Feb 2018 14:46:41 -0800 Subject: [PATCH 20/56] missing FSLDIR --- Dockerfile | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index dad086b..81703b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,7 @@ RUN apt-get update && \ apt-get install -y curl && \ curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ apt-get remove -y curl && \ - apt-get install -y nodejs && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + apt-get install -y nodejs RUN npm install -g bids-validator@0.25.07 @@ -28,8 +27,7 @@ RUN apt-get -y update \ --exclude='freesurfer/average/mult-comp-cor' \ --exclude='freesurfer/lib/cuda' \ --exclude='freesurfer/lib/qt' && \ - apt-get install -y tcsh bc tar libgomp1 perl-modules curl && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + apt-get install -y tcsh bc tar libgomp1 perl-modules curl # Set up the environment ENV OS Linux @@ -52,8 +50,8 @@ ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools: # Install FSL 5.0.10 RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \ python fslinstaller.py -d /usr/local/fsl && \ - chmod +x $FSLDIR/etc/fslconf/fsl.sh && \ - $FSLDIR/etc/fslconf/fsl.sh + chmod +x $/usr/local/fsl/etc/fslconf/fsl.sh && \ + /usr/local/fsl/etc/fslconf/fsl.sh ENV FSLDIR=/usr/local/fsl ENV PATH=$FSLDIR/bin:$PATH @@ -80,8 +78,7 @@ RUN apt-get -y update \ tar zxvf /pipelines.tar.gz && \ mv /opt/Pipelines-* /opt/HCP-Pipelines && \ rm /pipelines.tar.gz && \ - cd / && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + cd / ENV HCPPIPEDIR=/opt/HCP-Pipelines ENV HCPPIPEDIR_Templates=${HCPPIPEDIR}/global/templates @@ -99,8 +96,7 @@ ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMBINDIR=${FSLDIR}/bin/ -RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools RUN pip install pybids==0.4.2 ENV PYTHONPATH="" From ac60bf66e49d228bcb5cc6dcfd504bd06e0aa247 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 21 Feb 2018 17:27:41 -0800 Subject: [PATCH 21/56] connectome workbench link + typo --- Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 81703b5..7d2f563 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools: # Install FSL 5.0.10 RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \ python fslinstaller.py -d /usr/local/fsl && \ - chmod +x $/usr/local/fsl/etc/fslconf/fsl.sh && \ + chmod +x /usr/local/fsl/etc/fslconf/fsl.sh && \ /usr/local/fsl/etc/fslconf/fsl.sh ENV FSLDIR=/usr/local/fsl @@ -64,9 +64,15 @@ ENV LD_LIBRARY_PATH=$FSLDIR:$LD_LIBRARY_PATH ENV FSLTCLSH=/usr/bin/tclsh ENV FSLWISH=/usr/bin/wish ENV FSLOUTPUTTYPE=NIFTI_GZ +ENV MSMBINDIR=${FSLDIR}/bin/ # Install Connectome Workbench -RUN apt-get update && apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ + apt-get update && \ + apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 ENV CARET7DIR=/usr/bin @@ -94,7 +100,6 @@ ENV HCPPIPEDIR_dMRI=${HCPPIPEDIR}/DiffusionPreprocessing/scripts ENV HCPPIPEDIR_dMRITract=${HCPPIPEDIR}/DiffusionTractography/scripts ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts -ENV MSMBINDIR=${FSLDIR}/bin/ RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools RUN pip install pybids==0.4.2 From ef96dd776c27692c19dda717ffa0115e836cb773 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 09:46:44 -0800 Subject: [PATCH 22/56] update circleCI with new data --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 5d4fd3f..e3be6f5 100644 --- a/circle.yml +++ b/circle.yml @@ -11,8 +11,8 @@ dependencies: - "~/data" override: - - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids.zip -d ${HOME}/data; fi - - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fc4eb83f6901f3b47a54" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fc4eb83f6901f3b47a54" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids.zip -d ${HOME}/data; fi - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - git describe --tags > version - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : From 0cf56eb5540e1f07404694755522b7e484f0823f Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 09:50:01 -0800 Subject: [PATCH 23/56] update version number for tests --- circle.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index e3be6f5..3c83869 100644 --- a/circle.yml +++ b/circle.yml @@ -11,8 +11,8 @@ dependencies: - "~/data" override: - - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids.zip -d ${HOME}/data; fi - - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fc4eb83f6901f3b47a54" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids_v3.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fc4eb83f6901f3b47a54" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids_v3.zip -d ${HOME}/data; fi - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - git describe --tags > version - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : @@ -23,9 +23,9 @@ dependencies: test: override: # print version - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version # participant level tests for single session dataset - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: timeout: 21600 deployment: hub: From bd328c9dc7fadea16cb9ce48374a69baefb6b289 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 09:52:38 -0800 Subject: [PATCH 24/56] change provider img --- circle.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index 3c83869..29220e6 100644 --- a/circle.yml +++ b/circle.yml @@ -12,7 +12,7 @@ dependencies: override: - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids_v3.zip -d ${HOME}/data; fi - - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fc4eb83f6901f3b47a54" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids_v3.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids_v3.zip -d ${HOME}/data; fi - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - git describe --tags > version - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : @@ -23,9 +23,9 @@ dependencies: test: override: # print version - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version # participant level tests for single session dataset - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: timeout: 21600 deployment: hub: From 19ab83c9443bfd88337757f4518bab44ef431ae9 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 09:59:08 -0800 Subject: [PATCH 25/56] actually not using lifespan --- circle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/circle.yml b/circle.yml index 29220e6..9c89d76 100644 --- a/circle.yml +++ b/circle.yml @@ -12,7 +12,6 @@ dependencies: override: - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids_v3.zip -d ${HOME}/data; fi - - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids_v3.zip -d ${HOME}/data; fi - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - git describe --tags > version - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : From 206294533023e050ffc5cbe40f237d3a2b301367 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 10:04:48 -0800 Subject: [PATCH 26/56] trigger circle --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7d2f563..f3482a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,7 @@ ENV LD_LIBRARY_PATH=$FSLDIR:$LD_LIBRARY_PATH ENV FSLTCLSH=/usr/bin/tclsh ENV FSLWISH=/usr/bin/wish ENV FSLOUTPUTTYPE=NIFTI_GZ + ENV MSMBINDIR=${FSLDIR}/bin/ # Install Connectome Workbench From 023e003c68b0dde5128f0abb7f4d3797cca449ac Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 10:40:50 -0800 Subject: [PATCH 27/56] add MSMCONFIGDIR --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f3482a4..67bfe95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,6 +101,7 @@ ENV HCPPIPEDIR_dMRI=${HCPPIPEDIR}/DiffusionPreprocessing/scripts ENV HCPPIPEDIR_dMRITract=${HCPPIPEDIR}/DiffusionTractography/scripts ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts +ENV MSMCONFIGDIR=${HCPPIPEDIR}/MSMConfig RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools RUN pip install pybids==0.4.2 From b08a02cfe354a95f920534f491c8f7bcbae70c33 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 10:41:04 -0800 Subject: [PATCH 28/56] increase timeout for docker build after upgrading fsl --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 9c89d76..5e9b8f8 100644 --- a/circle.yml +++ b/circle.yml @@ -15,7 +15,7 @@ dependencies: - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - git describe --tags > version - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : - timeout: 21600 + timeout: 40000 - mkdir -p ~/docker; docker save "bids/${CIRCLE_PROJECT_REPONAME,,}" > ~/docker/image.tar - mkdir -p ${HOME}/outputs From f809f936b4eed24bb9c2b21893781d119ce89243 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 13:44:04 -0800 Subject: [PATCH 29/56] get right files on OSF --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 5e9b8f8..f8f3733 100644 --- a/circle.yml +++ b/circle.yml @@ -11,7 +11,7 @@ dependencies: - "~/data" override: - - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids_v3.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids_v3.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5a8efffe91b689000c9f5ce5" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids.zip -d ${HOME}/data; fi - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi - git describe --tags > version - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : From 9918940bfbd201b427ed211e4f256c5ca401f13b Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 14:17:08 -0800 Subject: [PATCH 30/56] filename with MACOSX --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index f8f3733..8bcc34c 100644 --- a/circle.yml +++ b/circle.yml @@ -22,9 +22,9 @@ dependencies: test: override: # print version - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/__MACOSX/hcp_example_bids_v3:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version # participant level tests for single session dataset - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/__MACOSX/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: timeout: 21600 deployment: hub: From 7859a164d7d3750f1196f4f2a8a422fe1a29d81f Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 15:21:54 -0800 Subject: [PATCH 31/56] install latest connectome workbench --- Dockerfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 67bfe95..e08298b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,8 +48,9 @@ ENV MNI_PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH # Install FSL 5.0.10 -RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \ - python fslinstaller.py -d /usr/local/fsl && \ +RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py + +RUN python fslinstaller.py -d /usr/local/fsl && \ chmod +x /usr/local/fsl/etc/fslconf/fsl.sh && \ /usr/local/fsl/etc/fslconf/fsl.sh @@ -64,20 +65,20 @@ ENV LD_LIBRARY_PATH=$FSLDIR:$LD_LIBRARY_PATH ENV FSLTCLSH=/usr/bin/tclsh ENV FSLWISH=/usr/bin/wish ENV FSLOUTPUTTYPE=NIFTI_GZ - ENV MSMBINDIR=${FSLDIR}/bin/ -# Install Connectome Workbench +# Install connectome workbench (latest dev) +RUN wget http://brainvis.wustl.edu/workbench/workbench-source-dev_latest.zip +RUN apt-get install -y zip unzip cmake +RUN unzip workbench-source-dev_latest.zip +RUN mkdir build +WORKDIR build RUN apt-get update && \ - apt-get install -y --no-install-recommends curl && \ - curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update && \ - apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 - -ENV CARET7DIR=/usr/bin + apt-get install -y build-essential openssl libssl-dev libcurl4-openssl-dev g++ qtdeclarative5-dev +RUN cmake -D CMAKE_BUILD_TYPE=Release -D WORKBENCH_MESA_DIR=/usr -D WORKBENCH_USE_QT5=TRUE ../workbench/workbench_source/src # Install HCP Pipelines +WORKDIR / RUN apt-get -y update \ && apt-get install -y --no-install-recommends python-numpy && \ wget https://github.com/jokedurnez/Pipelines/archive/v3.23.0-jd.tar.gz -O pipelines.tar.gz && \ From 615cb8523ebf68c19bd2dfb6a453aa1bdeeac4b5 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 16:57:15 -0800 Subject: [PATCH 32/56] right OSF dataset --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 8bcc34c..2fd6d4f 100644 --- a/circle.yml +++ b/circle.yml @@ -22,9 +22,9 @@ dependencies: test: override: # print version - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/__MACOSX/hcp_example_bids_v3:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version # participant level tests for single session dataset - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/__MACOSX/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: timeout: 21600 deployment: hub: From fc9855e2f771cc6229b96482f9d02893a5cf4a57 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Thu, 22 Feb 2018 17:05:06 -0800 Subject: [PATCH 33/56] caret dir --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e08298b..5107fd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,6 +103,7 @@ ENV HCPPIPEDIR_dMRITract=${HCPPIPEDIR}/DiffusionTractography/scripts ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMCONFIGDIR=${HCPPIPEDIR}/MSMConfig +ENV CARET7DIR=/usr/bin RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools RUN pip install pybids==0.4.2 From 873db6abb186beb9e5c67c9fa43bf6d10f0892d7 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Tue, 27 Feb 2018 15:32:32 -0800 Subject: [PATCH 34/56] printcomm no longer option --- run.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/run.py b/run.py index 921d4f8..f917821 100644 --- a/run.py +++ b/run.py @@ -65,8 +65,7 @@ def run_pre_freesurfer(**args): '--unwarpdir="{unwarpdir}" ' + \ '--gdcoeffs="NONE" ' + \ '--avgrdcmethod={avgrdcmethod} ' + \ - '--topupconfig="{HCPPIPEDIR_Config}/b02b0.cnf" ' + \ - '--printcom=""' + '--topupconfig="{HCPPIPEDIR_Config}/b02b0.cnf" ' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) @@ -78,8 +77,7 @@ def run_freesurfer(**args): '--subjectDIR="{subjectDIR}" ' + \ '--t1="{path}/{subject}/T1w/T1w_acpc_dc_restore.nii.gz" ' + \ '--t1brain="{path}/{subject}/T1w/T1w_acpc_dc_restore_brain.nii.gz" ' + \ - '--t2="{path}/{subject}/T1w/T2w_acpc_dc_restore.nii.gz" ' + \ - '--printcom=""' + '--t2="{path}/{subject}/T1w/T2w_acpc_dc_restore.nii.gz" ' cmd = cmd.format(**args) if not os.path.exists(os.path.join(args["subjectDIR"], "fsaverage")): @@ -108,8 +106,7 @@ def run_post_freesurfer(**args): '--subcortgraylabels="{HCPPIPEDIR_Config}/FreeSurferSubcorticalLabelTableLut.txt" ' + \ '--freesurferlabels="{HCPPIPEDIR_Config}/FreeSurferAllLut.txt" ' + \ '--refmyelinmaps="{HCPPIPEDIR_Templates}/standard_mesh_atlases/Conte69.MyelinMap_BC.164k_fs_LR.dscalar.nii" ' + \ - '--regname="FS" ' + \ - '--printcom=""' + '--regname="FS" ' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) @@ -133,7 +130,6 @@ def run_generic_fMRI_volume_processsing(**args): '--dcmethod={dcmethod} ' + \ '--gdcoeffs="NONE" ' + \ '--topupconfig={HCPPIPEDIR_Config}/b02b0.cnf ' + \ - '--printcom="" ' + \ '--biascorrection={biascorrection} ' + \ '--mctype="MCFLIRT"' cmd = cmd.format(**args) @@ -164,8 +160,7 @@ def run_diffusion_processsing(**args): '--PEdir={PEdir} ' + \ '--gdcoeffs="NONE" ' + \ '--extra-eddy-arg="--data_is_shelled" ' + \ - '--no-gpu ' + \ - '--printcom=""' + '--no-gpu ' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) From 84c54c02eb40f6eba64ff08b1d3f1a163beedd18 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Tue, 27 Feb 2018 15:33:04 -0800 Subject: [PATCH 35/56] update pipelines, freesurfer, workbench --- Dockerfile | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5107fd0..4b917b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Use Ubuntu 14.04 LTS -FROM ubuntu:trusty-20170119 +# Use Ubuntu 17.10 LTS +FROM ubuntu:16.04 ## Install the validator RUN apt-get update && \ @@ -13,7 +13,7 @@ RUN npm install -g bids-validator@0.25.07 # Download FreeSurfer RUN apt-get -y update \ && apt-get install -y wget && \ - wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/5.3.0-HCP/freesurfer-Linux-centos4_x86_64-stable-pub-v5.3.0-HCP.tar.gz | tar zxv -C /opt \ + wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \ --exclude='freesurfer/trctrain' \ --exclude='freesurfer/subjects/fsaverage_sym' \ --exclude='freesurfer/subjects/fsaverage3' \ @@ -66,22 +66,20 @@ ENV FSLTCLSH=/usr/bin/tclsh ENV FSLWISH=/usr/bin/wish ENV FSLOUTPUTTYPE=NIFTI_GZ ENV MSMBINDIR=${FSLDIR}/bin/ +RUN chmod 770 -R $FSLDIR -# Install connectome workbench (latest dev) -RUN wget http://brainvis.wustl.edu/workbench/workbench-source-dev_latest.zip -RUN apt-get install -y zip unzip cmake -RUN unzip workbench-source-dev_latest.zip -RUN mkdir build -WORKDIR build RUN apt-get update && \ - apt-get install -y build-essential openssl libssl-dev libcurl4-openssl-dev g++ qtdeclarative5-dev -RUN cmake -D CMAKE_BUILD_TYPE=Release -D WORKBENCH_MESA_DIR=/usr -D WORKBENCH_USE_QT5=TRUE ../workbench/workbench_source/src + apt-get install -y --no-install-recommends curl && \ + curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ + apt-get update && \ + apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 # Install HCP Pipelines WORKDIR / RUN apt-get -y update \ && apt-get install -y --no-install-recommends python-numpy && \ - wget https://github.com/jokedurnez/Pipelines/archive/v3.23.0-jd.tar.gz -O pipelines.tar.gz && \ + wget https://github.com/jokedurnez/Pipelines/archive/v3.25.0-jd.tar.gz -O pipelines.tar.gz && \ cd /opt/ && \ tar zxvf /pipelines.tar.gz && \ mv /opt/Pipelines-* /opt/HCP-Pipelines && \ From f813d025e34189b2c70c1f964442f11eec87cea6 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 28 Feb 2018 08:46:58 -0800 Subject: [PATCH 36/56] revert to HCP-freesurfer version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4b917b1..cbf8c05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN npm install -g bids-validator@0.25.07 # Download FreeSurfer RUN apt-get -y update \ && apt-get install -y wget && \ - wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \ + wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/5.3.0-HCP/freesurfer-Linux-centos4_x86_64-stable-pub-v5.3.0-HCP.tar.gz | tar zxv -C /opt \ --exclude='freesurfer/trctrain' \ --exclude='freesurfer/subjects/fsaverage_sym' \ --exclude='freesurfer/subjects/fsaverage3' \ From d61f0dc559b809f58d2cab73e32a02b5f8b8fe69 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 2 Mar 2018 09:17:24 -0800 Subject: [PATCH 37/56] ubuntu 16.04 seems to work --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cbf8c05..b3b4bc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN npm install -g bids-validator@0.25.07 # Download FreeSurfer RUN apt-get -y update \ && apt-get install -y wget && \ - wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/5.3.0-HCP/freesurfer-Linux-centos4_x86_64-stable-pub-v5.3.0-HCP.tar.gz | tar zxv -C /opt \ + wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \ --exclude='freesurfer/trctrain' \ --exclude='freesurfer/subjects/fsaverage_sym' \ --exclude='freesurfer/subjects/fsaverage3' \ @@ -28,7 +28,6 @@ RUN apt-get -y update \ --exclude='freesurfer/lib/cuda' \ --exclude='freesurfer/lib/qt' && \ apt-get install -y tcsh bc tar libgomp1 perl-modules curl - # Set up the environment ENV OS Linux ENV FS_OVERRIDE 0 @@ -107,6 +106,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-pip pyth RUN pip install pybids==0.4.2 ENV PYTHONPATH="" +WORKDIR /opt/freesurfer/bin +RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/d26114a201333f812d2cef67a338e2685c004d00/scripts/recon-all.v6.hires && \ + chmod +x /opt/freesurfer/bin/recon-all.v6.hires + COPY run.py /run.py RUN chmod +x /run.py From a97b4773957a3e138fc1820d441982c8a9407279 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 2 Mar 2018 15:37:25 -0800 Subject: [PATCH 38/56] remote required but unused license argument --- run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/run.py b/run.py index f917821..02b9453 100644 --- a/run.py +++ b/run.py @@ -192,8 +192,6 @@ def run_diffusion_processsing(**args): default=['PreFreeSurfer', 'FreeSurfer', 'PostFreeSurfer', 'fMRIVolume', 'fMRISurface', 'DiffusionPreprocessing']) -parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', - required=True) parser.add_argument('-v', '--version', action='version', version='HCP Pipelines BIDS App version {}'.format(__version__)) From 5bd97b709b15f908297b3b163a5e0117b0a36db4 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 2 Mar 2018 16:12:22 -0800 Subject: [PATCH 39/56] remove license option in circle --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 2fd6d4f..b693101 100644 --- a/circle.yml +++ b/circle.yml @@ -24,7 +24,7 @@ test: # print version - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version # participant level tests for single session dataset - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids_v3:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --n_cpus 2: timeout: 21600 deployment: hub: From d8eb7ef400c8f3e6fb4ff49c9ed0661eaffebf35 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 2 Mar 2018 16:35:43 -0800 Subject: [PATCH 40/56] update Pipelines repo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b3b4bc2..03a1909 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,7 +78,7 @@ RUN apt-get update && \ WORKDIR / RUN apt-get -y update \ && apt-get install -y --no-install-recommends python-numpy && \ - wget https://github.com/jokedurnez/Pipelines/archive/v3.25.0-jd.tar.gz -O pipelines.tar.gz && \ + wget https://github.com/jokedurnez/Pipelines/archive/BIDS-app.tar.gz -O pipelines.tar.gz && \ cd /opt/ && \ tar zxvf /pipelines.tar.gz && \ mv /opt/Pipelines-* /opt/HCP-Pipelines && \ From 5626a8939dec621916c1d66928abc33caf450c4a Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Fri, 2 Mar 2018 17:11:12 -0800 Subject: [PATCH 41/56] add tess1mm manually --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 03a1909..eaf2b34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -109,6 +109,8 @@ ENV PYTHONPATH="" WORKDIR /opt/freesurfer/bin RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/d26114a201333f812d2cef67a338e2685c004d00/scripts/recon-all.v6.hires && \ chmod +x /opt/freesurfer/bin/recon-all.v6.hires +RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/dev/scripts/tess1mm && \ + chmod +x /opt/freesurfer/bin/tess1mm COPY run.py /run.py RUN chmod +x /run.py From dd2a1a3aae88fd366961c7dc443694d39f2ca8a0 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 5 Mar 2018 17:24:50 -0800 Subject: [PATCH 42/56] remove typo wb_command --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index eaf2b34..38c4c03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ RUN apt-get -y update \ --exclude='freesurfer/lib/cuda' \ --exclude='freesurfer/lib/qt' && \ apt-get install -y tcsh bc tar libgomp1 perl-modules curl + # Set up the environment ENV OS Linux ENV FS_OVERRIDE 0 @@ -106,6 +107,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-pip pyth RUN pip install pybids==0.4.2 ENV PYTHONPATH="" +# missing libraries +RUN echo deb http://security.ubuntu.com/ubuntu precise-security main >> /etc/apt/sources.list && \ + apt update && \ + apt install -y libxp6 libxmu6 + WORKDIR /opt/freesurfer/bin RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/d26114a201333f812d2cef67a338e2685c004d00/scripts/recon-all.v6.hires && \ chmod +x /opt/freesurfer/bin/recon-all.v6.hires From ccccc27162ade57c97f68aea266d57de6383f110 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 7 Mar 2018 15:14:05 -0800 Subject: [PATCH 43/56] remove print and remove no gpu --- run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run.py b/run.py index 02b9453..50a0c39 100644 --- a/run.py +++ b/run.py @@ -159,8 +159,7 @@ def run_diffusion_processsing(**args): '--echospacing="{echospacing}" '+ \ '--PEdir={PEdir} ' + \ '--gdcoeffs="NONE" ' + \ - '--extra-eddy-arg="--data_is_shelled" ' + \ - '--no-gpu ' + '--extra-eddy-arg="--data_is_shelled" ' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) @@ -433,7 +432,6 @@ def run_diffusion_processsing(**args): metadata = layout.get_metadata(dwi.filename) # get phaseencodingdirection phaseenc = metadata['PhaseEncodingDirection'] - print("hase: %s - ind: %i"%(phaseenc,idx)) acq = 1 if phaseenc[0]=='i' else 2 if not PEdir: PEdir = acq From 7a8ac378e54876a1ded8ff624268b516e911744d Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 7 Mar 2018 15:14:22 -0800 Subject: [PATCH 44/56] roll back --- Dockerfile | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38c4c03..8454115 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Use Ubuntu 17.10 LTS -FROM ubuntu:16.04 +# Use Ubuntu 14.04 LTS +FROM ubuntu:14.04 ## Install the validator RUN apt-get update && \ @@ -13,7 +13,7 @@ RUN npm install -g bids-validator@0.25.07 # Download FreeSurfer RUN apt-get -y update \ && apt-get install -y wget && \ - wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \ + wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/5.3.0-HCP/freesurfer-Linux-centos4_x86_64-stable-pub-v5.3.0-HCP.tar.gz | tar zxv -C /opt \ --exclude='freesurfer/trctrain' \ --exclude='freesurfer/subjects/fsaverage_sym' \ --exclude='freesurfer/subjects/fsaverage3' \ @@ -27,7 +27,9 @@ RUN apt-get -y update \ --exclude='freesurfer/average/mult-comp-cor' \ --exclude='freesurfer/lib/cuda' \ --exclude='freesurfer/lib/qt' && \ - apt-get install -y tcsh bc tar libgomp1 perl-modules curl + apt-get remove -y wget && \ + apt-get install -y tcsh bc tar libgomp1 perl-modules curl && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Set up the environment ENV OS Linux @@ -47,27 +49,28 @@ ENV PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 ENV MNI_PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH -# Install FSL 5.0.10 -RUN wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py - -RUN python fslinstaller.py -d /usr/local/fsl && \ - chmod +x /usr/local/fsl/etc/fslconf/fsl.sh && \ - /usr/local/fsl/etc/fslconf/fsl.sh +# Install FSL 5.0.9 +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ + apt-get update && \ + apt-get install -y fsl-core=5.0.9-4~nd14.04+1 && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -ENV FSLDIR=/usr/local/fsl -ENV PATH=$FSLDIR/bin:$PATH +ENV FSLDIR=/usr/share/fsl/5.0 ENV FSL_DIR="${FSLDIR}" ENV FSLOUTPUTTYPE=NIFTI_GZ -ENV PATH=$FSLDIR:$PATH +ENV PATH=/usr/lib/fsl/5.0:$PATH ENV FSLMULTIFILEQUIT=TRUE -ENV POSSUMDIR=$FSLDIR -ENV LD_LIBRARY_PATH=$FSLDIR:$LD_LIBRARY_PATH +ENV POSSUMDIR=/usr/share/fsl/5.0 +ENV LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH ENV FSLTCLSH=/usr/bin/tclsh ENV FSLWISH=/usr/bin/wish ENV FSLOUTPUTTYPE=NIFTI_GZ -ENV MSMBINDIR=${FSLDIR}/bin/ -RUN chmod 770 -R $FSLDIR +RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh +# Get connectome workbench RUN apt-get update && \ apt-get install -y --no-install-recommends curl && \ curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ @@ -79,7 +82,7 @@ RUN apt-get update && \ WORKDIR / RUN apt-get -y update \ && apt-get install -y --no-install-recommends python-numpy && \ - wget https://github.com/jokedurnez/Pipelines/archive/BIDS-app.tar.gz -O pipelines.tar.gz && \ + wget https://github.com/jokedurnez/Pipelines/archive/v0.1.0.tar.gz -O pipelines.tar.gz && \ cd /opt/ && \ tar zxvf /pipelines.tar.gz && \ mv /opt/Pipelines-* /opt/HCP-Pipelines && \ @@ -104,7 +107,10 @@ ENV MSMCONFIGDIR=${HCPPIPEDIR}/MSMConfig ENV CARET7DIR=/usr/bin RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools -RUN pip install pybids==0.4.2 +ADD requirements.txt requirements.txt +RUN pip install -r requirements.txt && \ + rm -rf ~/.cache/pip + ENV PYTHONPATH="" # missing libraries From 4f41911c5872a08dde3b5e8f589a435e12ac3d27 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Wed, 7 Mar 2018 15:14:51 -0800 Subject: [PATCH 45/56] add requirements --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..94ae07d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +grabbit==0.1.0 +pybids==0.4.2 From dd7200fe658268a26e7442f41b18f6e16f03a5fb Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 12 Mar 2018 12:15:49 -0700 Subject: [PATCH 46/56] add diffusion pipeline --- run.py | 83 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/run.py b/run.py index 50a0c39..9cbe588 100644 --- a/run.py +++ b/run.py @@ -4,7 +4,6 @@ import os import shutil import nibabel -import copy from glob import glob from subprocess import Popen, PIPE from shutil import rmtree @@ -64,6 +63,7 @@ def run_pre_freesurfer(**args): '--t2samplespacing="{t2samplespacing}" ' + \ '--unwarpdir="{unwarpdir}" ' + \ '--gdcoeffs="NONE" ' + \ + '--extra-eddy-arg="--data_is_shelled"' + \ '--avgrdcmethod={avgrdcmethod} ' + \ '--topupconfig="{HCPPIPEDIR_Config}/b02b0.cnf" ' cmd = cmd.format(**args) @@ -158,14 +158,13 @@ def run_diffusion_processsing(**args): '--subject="{subject}" ' + \ '--echospacing="{echospacing}" '+ \ '--PEdir={PEdir} ' + \ - '--gdcoeffs="NONE" ' + \ - '--extra-eddy-arg="--data_is_shelled" ' + '--gdcoeffs="NONE" ' cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) __version__ = open('/version').read() -parser = argparse.ArgumentParser(description='HCP Pipelines BIDS App (T1w, T2w, fMRI, DWI)') +parser = argparse.ArgumentParser(description='HCP Pipeliens BIDS App (T1w, T2w, fMRI, DWI)') parser.add_argument('bids_dir', help='The directory with the input dataset ' 'formatted according to the BIDS standard.') parser.add_argument('output_dir', help='The directory where the output files ' @@ -191,6 +190,8 @@ def run_diffusion_processsing(**args): default=['PreFreeSurfer', 'FreeSurfer', 'PostFreeSurfer', 'fMRIVolume', 'fMRISurface', 'DiffusionPreprocessing']) +parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', + required=True) parser.add_argument('-v', '--version', action='version', version='HCP Pipelines BIDS App version {}'.format(__version__)) @@ -246,7 +247,7 @@ def run_diffusion_processsing(**args): if fieldmap_set: if len(fieldmap_set)>1: - fieldmap_trans = dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) + fieldmap_trans=dict(zip(d,zip(*[d.values() for d in fieldmap_set]))) else: fieldmap_trans = {k:[v] for k,v in fieldmap_set[0].iteritems()} @@ -262,7 +263,7 @@ def run_diffusion_processsing(**args): "t2samplespacing": "%.8f"%t2_spacing, "unwarpdir": unwarpdir}) - if set(fieldmap_trans['type'])==set(['phasediff']): + if set(fieldmap_set["type"]) == set(["phasediff"]): merged_file = "%s/tmp/%s/magfile.nii.gz"%(args.output_dir, subject_label) run("mkdir -p %s/tmp/%s/ && fslmerge -t %s %s %s"%(args.output_dir, subject_label, @@ -279,8 +280,7 @@ def run_diffusion_processsing(**args): "fmapphase": fieldmap_trans["phasediff"][0], "echodiff": "%.6f"%te_diff, "avgrdcmethod": "SiemensFieldMap"}) - - elif set(fieldmap_trans['type'])== set(["epi"]): + elif set(fieldmap_set["type"]) == set(["epi"]): SEPhaseNeg = None SEPhasePos = None seunwarpdir = None @@ -291,40 +291,39 @@ def run_diffusion_processsing(**args): else: SEPhasePos = fieldmap - unwarpdir = enc_dir.replace("-", "").replace("i","x").replace("j", "y").replace("k", "z") + unwarpdir = enc_dir.replace('-', '').replace('i','x').replace('j','y').replace('k',z) if seunwarpdir and not seunwarpdir == unwarpdir: raise RuntimeError("Inconsistent unwarp directions.") else: - seunwarpdir = copy.deepcopy(unwarpdir) + raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w. Please fix your dataset.") + seunwarpdir = copy.deepcopy(unwarpdir) fmap_arguments = {} for fieldmap in fieldmap_set: - if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap["epi"]): - echospacing = layout.get_metadata(fieldmap["epi"])["EffectiveEchoSpacing"] - elif "TotalReadoutTime" in layout.get_metadata(fieldmap["epi"]): + if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap_set["epi"]): + echospacing = layout.get_metadata(fieldmap_set["epi"])["EffectiveEchoSpacing"] + elif "TotalReadoutTime" in layout.get_metadata(fieldmap_set["epi"]): # HCP Pipelines do not allow users to specify total readout time directly # Hence we need to reverse the calculations to provide echo spacing that would # result in the right total read out total read out time # see https://github.com/Washington-University/Pipelines/blob/master/global/scripts/TopupPreprocessingAll.sh#L202 print("BIDS App wrapper: Did not find EffectiveEchoSpacing, calculating it from TotalReadoutTime") # TotalReadoutTime = EffectiveEchoSpacing * (len(PhaseEncodingDirection) - 1) - total_readout_time = layout.get_metadata(fieldmap["epi"])["TotalReadoutTime"] - phase_len = nibabel.load(fieldmap["epi"]).shape[{"x": 0, "y": 1}[seunwarpdir]] + total_readout_time = layout.get_metadata(fieldmap_set["epi"])["TotalReadoutTime"] + phase_len = nibabel.load(fieldmap_set["epi"]).shape[{"x": 0, "y": 1}[seunwarpdir]] echospacing = total_readout_time / float(phase_len - 1) - else: - raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") - if 'echospacing' in fmap_arguments.keys() and not fmap_arguments['echospacing']==echospacing: - raise RuntimeError("Inconsistent echospacing.") + else: + raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") + if 'echospacing' in fmap_arguments.keys() and not fmap_arguments['echospacing']==echospacing: + raise RuntimeError("Inconsistent echospacing.") fmap_args.update({"SEPhaseNeg": SEPhaseNeg, "SEPhasePos": SEPhasePos, "echospacing": "%.6f"%echospacing, "seunwarpdir": seunwarpdir, "avgrdcmethod": "TOPUP"}) - else: - raise RuntimeError("Inconsistent fieldmap types or unknown type.") - - #TODO add support for GE fieldmaps + else: + raise RuntimeError("Inconsistent fieldmap types or unknown type.") struct_stages_dict = OrderedDict([("PreFreeSurfer", partial(run_pre_freesurfer, path=args.output_dir, @@ -364,29 +363,29 @@ def run_diffusion_processsing(**args): fieldmap_set = layout.get_fieldmap(fmritcs,return_list=True) if fieldmap_set: fieldmap_trans = dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) - if set(fieldmap_trans['type'])== set(["epi"]): + if set(fieldmap_trans['type']) == set(['epi']): SEPhaseNeg = None SEPhasePos = None - for fieldmap in fieldmap_set: - enc_dir = layout.get_metadata(fieldmap['epi'])["PhaseEncodingDirection"] + for fieldmap in fieldmap_set["epi"]: + enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] if "-" in enc_dir: - SEPhaseNeg = fieldmap['epi'] + SEPhaseNeg = fieldmap else: - SEPhasePos = fieldmap['epi'] - echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] - unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] - unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") - if len(unwarpdir) == 2: - unwarpdir = "-" + unwarpdir[0] - dcmethod = "TOPUP" - biascorrection = "SEBASED" - else: - SEPhaseNeg = "NONE" - SEPhasePos = "NONE" - echospacing = "NONE" - unwarpdir = "NONE" - dcmethod = "NONE" - biascorrection = "NONE" + SEPhasePos = fieldmap + echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] + unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] + unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") + if len(unwarpdir) == 2: + unwarpdir = "-" + unwarpdir[0] + dcmethod = "TOPUP" + biascorrection = "SEBASED" + else: + SEPhaseNeg = "NONE" + SEPhasePos = "NONE" + echospacing = "NONE" + unwarpdir = "NONE" + dcmethod = "NONE" + biascorrection = "NONE" zooms = nibabel.load(fmritcs).get_header().get_zooms() fmrires = float(min(zooms[:3])) From 6dbc4334aecc5777c2bdd711f34c6e701a213c43 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 12 Mar 2018 12:19:35 -0700 Subject: [PATCH 47/56] remove unused license arg --- run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/run.py b/run.py index 9cbe588..6778d12 100644 --- a/run.py +++ b/run.py @@ -190,8 +190,6 @@ def run_diffusion_processsing(**args): default=['PreFreeSurfer', 'FreeSurfer', 'PostFreeSurfer', 'fMRIVolume', 'fMRISurface', 'DiffusionPreprocessing']) -parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', - required=True) parser.add_argument('-v', '--version', action='version', version='HCP Pipelines BIDS App version {}'.format(__version__)) From 8bbc67a8af05562fea4830c826a109681fc94d3f Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 12 Mar 2018 12:22:21 -0700 Subject: [PATCH 48/56] debug fieldmaps --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 6778d12..367d653 100644 --- a/run.py +++ b/run.py @@ -245,7 +245,7 @@ def run_diffusion_processsing(**args): if fieldmap_set: if len(fieldmap_set)>1: - fieldmap_trans=dict(zip(d,zip(*[d.values() for d in fieldmap_set]))) + fieldmap_trans=dict(zip(fieldmap_set[0],zip(*[d.values() for d in fieldmap_set]))) else: fieldmap_trans = {k:[v] for k,v in fieldmap_set[0].iteritems()} From 935e85d5bffb04e05112f5f22e97faa103778dd8 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Sun, 18 Mar 2018 20:43:23 -0700 Subject: [PATCH 49/56] install eddy_openmp for dwi without gpu --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8454115..c36f112 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,6 +123,7 @@ RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/d26114a201333f8 chmod +x /opt/freesurfer/bin/recon-all.v6.hires RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/dev/scripts/tess1mm && \ chmod +x /opt/freesurfer/bin/tess1mm +RUN wget -qO- https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/eddy-patch-fsl-5.0.9/centos6/eddy_openmp > $FSLDIR/bin/eddy_openmp COPY run.py /run.py RUN chmod +x /run.py From 481884a041d1eeede21b013c1bd28e8b9f8546e9 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Sun, 18 Mar 2018 21:25:13 -0700 Subject: [PATCH 50/56] debug changes from pybids update --- run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index 367d653..147299d 100644 --- a/run.py +++ b/run.py @@ -261,7 +261,7 @@ def run_diffusion_processsing(**args): "t2samplespacing": "%.8f"%t2_spacing, "unwarpdir": unwarpdir}) - if set(fieldmap_set["type"]) == set(["phasediff"]): + if set(fieldmap_trans["type"]) == set(["phasediff"]): merged_file = "%s/tmp/%s/magfile.nii.gz"%(args.output_dir, subject_label) run("mkdir -p %s/tmp/%s/ && fslmerge -t %s %s %s"%(args.output_dir, subject_label, @@ -278,7 +278,7 @@ def run_diffusion_processsing(**args): "fmapphase": fieldmap_trans["phasediff"][0], "echodiff": "%.6f"%te_diff, "avgrdcmethod": "SiemensFieldMap"}) - elif set(fieldmap_set["type"]) == set(["epi"]): + elif set(fieldmap_trans["type"]) == set(["epi"]): SEPhaseNeg = None SEPhasePos = None seunwarpdir = None From 08850f40656985941e40775c006c85279e52556b Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 19 Mar 2018 08:45:24 -0700 Subject: [PATCH 51/56] debug unwarpdir dwi --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 147299d..cdd3ace 100644 --- a/run.py +++ b/run.py @@ -289,7 +289,7 @@ def run_diffusion_processsing(**args): else: SEPhasePos = fieldmap - unwarpdir = enc_dir.replace('-', '').replace('i','x').replace('j','y').replace('k',z) + unwarpdir = enc_dir.replace('-', '').replace('i','x').replace('j','y').replace('k','z') if seunwarpdir and not seunwarpdir == unwarpdir: raise RuntimeError("Inconsistent unwarp directions.") else: From ba8cc2bfba22f31d4e4ebbecb283ab942b4a1a54 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 19 Mar 2018 18:57:27 -0700 Subject: [PATCH 52/56] f*in indent --- run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/run.py b/run.py index cdd3ace..272ad45 100644 --- a/run.py +++ b/run.py @@ -289,12 +289,12 @@ def run_diffusion_processsing(**args): else: SEPhasePos = fieldmap - unwarpdir = enc_dir.replace('-', '').replace('i','x').replace('j','y').replace('k','z') - if seunwarpdir and not seunwarpdir == unwarpdir: - raise RuntimeError("Inconsistent unwarp directions.") - else: - raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w. Please fix your dataset.") - seunwarpdir = copy.deepcopy(unwarpdir) + unwarpdir = enc_dir.replace('-', '').replace('i','x').replace('j','y').replace('k','z') + if seunwarpdir and not seunwarpdir == unwarpdir: + raise RuntimeError("Inconsistent unwarp directions.") + else: + raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w. Please fix your dataset.") + seunwarpdir = copy.deepcopy(unwarpdir) fmap_arguments = {} for fieldmap in fieldmap_set: From 5979204ef19dd823965c9b34180322c169370350 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Mon, 19 Mar 2018 20:17:12 -0700 Subject: [PATCH 53/56] debug seunwarpdir for fmaps --- run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index 272ad45..52b1b68 100644 --- a/run.py +++ b/run.py @@ -293,8 +293,8 @@ def run_diffusion_processsing(**args): if seunwarpdir and not seunwarpdir == unwarpdir: raise RuntimeError("Inconsistent unwarp directions.") else: - raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w. Please fix your dataset.") - seunwarpdir = copy.deepcopy(unwarpdir) + #raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w. Please fix your dataset.") + seunwarpdir = copy.deepcopy(unwarpdir) fmap_arguments = {} for fieldmap in fieldmap_set: From a85785914b6048c46fd4c1916335bb3c18c0aea9 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Tue, 20 Mar 2018 06:40:18 -0700 Subject: [PATCH 54/56] debug fieldmap in case of multiple --- run.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/run.py b/run.py index 52b1b68..32e17a2 100644 --- a/run.py +++ b/run.py @@ -8,6 +8,7 @@ from subprocess import Popen, PIPE from shutil import rmtree import subprocess +import copy from bids.grabbids import BIDSLayout from functools import partial from collections import OrderedDict @@ -197,7 +198,7 @@ def run_diffusion_processsing(**args): -run("bids-validator " + args.bids_dir) +#run("bids-validator " + args.bids_dir) layout = BIDSLayout(args.bids_dir) subjects_to_analyze = [] @@ -297,23 +298,23 @@ def run_diffusion_processsing(**args): seunwarpdir = copy.deepcopy(unwarpdir) fmap_arguments = {} - for fieldmap in fieldmap_set: - if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap_set["epi"]): - echospacing = layout.get_metadata(fieldmap_set["epi"])["EffectiveEchoSpacing"] - elif "TotalReadoutTime" in layout.get_metadata(fieldmap_set["epi"]): + for fieldmap in fieldmap_trans['epi']: + if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap): + echospacing = layout.get_metadata(fieldmap)["EffectiveEchoSpacing"] + elif "TotalReadoutTime" in layout.get_metadata(fieldmap): # HCP Pipelines do not allow users to specify total readout time directly # Hence we need to reverse the calculations to provide echo spacing that would # result in the right total read out total read out time # see https://github.com/Washington-University/Pipelines/blob/master/global/scripts/TopupPreprocessingAll.sh#L202 print("BIDS App wrapper: Did not find EffectiveEchoSpacing, calculating it from TotalReadoutTime") # TotalReadoutTime = EffectiveEchoSpacing * (len(PhaseEncodingDirection) - 1) - total_readout_time = layout.get_metadata(fieldmap_set["epi"])["TotalReadoutTime"] - phase_len = nibabel.load(fieldmap_set["epi"]).shape[{"x": 0, "y": 1}[seunwarpdir]] + total_readout_time = layout.get_metadata(fieldmap)["TotalReadoutTime"] + phase_len = nibabel.load(fieldmap).shape[{"x": 0, "y": 1}[seunwarpdir]] echospacing = total_readout_time / float(phase_len - 1) - else: - raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") - if 'echospacing' in fmap_arguments.keys() and not fmap_arguments['echospacing']==echospacing: - raise RuntimeError("Inconsistent echospacing.") + else: + raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") + if 'echospacing' in fmap_arguments.keys() and not fmap_arguments['echospacing']==echospacing: + raise RuntimeError("Inconsistent echospacing.") fmap_args.update({"SEPhaseNeg": SEPhaseNeg, "SEPhasePos": SEPhasePos, @@ -364,7 +365,7 @@ def run_diffusion_processsing(**args): if set(fieldmap_trans['type']) == set(['epi']): SEPhaseNeg = None SEPhasePos = None - for fieldmap in fieldmap_set["epi"]: + for fieldmap in fieldmap_trans["epi"]: enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] if "-" in enc_dir: SEPhaseNeg = fieldmap From b22a435aafc1a1a6469e6be3afee0c04bd2f4ab2 Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Tue, 20 Mar 2018 06:59:03 -0700 Subject: [PATCH 55/56] uncomment validator --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 32e17a2..9964a46 100644 --- a/run.py +++ b/run.py @@ -198,7 +198,7 @@ def run_diffusion_processsing(**args): -#run("bids-validator " + args.bids_dir) +run("bids-validator " + args.bids_dir) layout = BIDSLayout(args.bids_dir) subjects_to_analyze = [] From 2c44cc0f6b3d829d4625b00665f7cbcac8e8119b Mon Sep 17 00:00:00 2001 From: jokedurnez Date: Tue, 20 Mar 2018 08:41:11 -0700 Subject: [PATCH 56/56] make eddy_openmp executable --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c36f112..7e53be9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,6 +124,7 @@ RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/d26114a201333f8 RUN wget https://raw.githubusercontent.com/freesurfer/freesurfer/dev/scripts/tess1mm && \ chmod +x /opt/freesurfer/bin/tess1mm RUN wget -qO- https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/eddy-patch-fsl-5.0.9/centos6/eddy_openmp > $FSLDIR/bin/eddy_openmp +RUN chmod +x $FSLDIR/bin/eddy_openmp COPY run.py /run.py RUN chmod +x /run.py