From ce2c6dbfe600164acbcb75a9fb83d54e440c15af Mon Sep 17 00:00:00 2001 From: Timothy Date: Wed, 28 Mar 2018 16:30:28 -0500 Subject: [PATCH 01/74] first commit to github.umn.edu of bids apps hcppipeline app --- Dockerfile | 110 +++++++++++++ README.md | 118 ++++++++++++++ Singularity | 61 ++++++++ circle.yml | 38 +++++ run.py | 439 ++++++++++++++++++++++++++++++++++++++++++++++++++++ version | 1 + 6 files changed, 767 insertions(+) create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 Singularity create mode 100644 circle.yml create mode 100644 run.py create mode 100644 version diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..89791f1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,110 @@ +# Use Ubuntu 14.04 LTS +FROM ubuntu:trusty-20170119 + +## Install the validator +RUN apt-get update && \ + apt-get install -y curl && \ + curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ + apt-get remove -y curl && \ + apt-get install -y nodejs + +RUN npm install -g bids-validator@0.19.2 + +# Download FreeSurfer +RUN apt-get -y update \ + && apt-get install -y wget && \ + wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/5.3.0-HCP/freesurfer-Linux-centos4_x86_64-stable-pub-v5.3.0-HCP.tar.gz | tar zxv -C /opt \ + --exclude='freesurfer/trctrain' \ + --exclude='freesurfer/subjects/fsaverage_sym' \ + --exclude='freesurfer/subjects/fsaverage3' \ + --exclude='freesurfer/subjects/fsaverage4' \ + --exclude='freesurfer/subjects/fsaverage5' \ + --exclude='freesurfer/subjects/fsaverage6' \ + --exclude='freesurfer/subjects/cvs_avg35' \ + --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \ + --exclude='freesurfer/subjects/bert' \ + --exclude='freesurfer/subjects/V1_average' \ + --exclude='freesurfer/average/mult-comp-cor' \ + --exclude='freesurfer/lib/cuda' \ + --exclude='freesurfer/lib/qt' && \ + apt-get install -y tcsh bc tar libgomp1 perl-modules curl + +# Set up the environment +ENV OS Linux +ENV FS_OVERRIDE 0 +ENV FIX_VERTEX_AREA= +ENV SUBJECTS_DIR /opt/freesurfer/subjects +ENV FSF_OUTPUT_FORMAT nii.gz +ENV MNI_DIR /opt/freesurfer/mni +ENV LOCAL_DIR /opt/freesurfer/local +ENV FREESURFER_HOME /opt/freesurfer +ENV FSFAST_HOME /opt/freesurfer/fsfast +ENV MINC_BIN_DIR /opt/freesurfer/mni/bin +ENV MINC_LIB_DIR /opt/freesurfer/mni/lib +ENV MNI_DATAPATH /opt/freesurfer/mni/data +ENV FMRI_ANALYSIS_DIR /opt/freesurfer/fsfast +ENV PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 +ENV MNI_PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5 +ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH + +# Install FSL 5.0.9 +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ + apt-get update && \ + apt-get install -y fsl-core=5.0.9-4~nd14.04+1 + +# Configure environment +ENV FSLDIR=/usr/share/fsl/5.0 +ENV FSL_DIR="${FSLDIR}" +ENV FSLOUTPUTTYPE=NIFTI_GZ +ENV PATH=/usr/lib/fsl/5.0:$PATH +ENV FSLMULTIFILEQUIT=TRUE +ENV POSSUMDIR=/usr/share/fsl/5.0 +ENV LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH +ENV FSLTCLSH=/usr/bin/tclsh +ENV FSLWISH=/usr/bin/wish +ENV FSLOUTPUTTYPE=NIFTI_GZ +RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh + +# Install Connectome Workbench +RUN apt-get update && apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 + +ENV CARET7DIR=/usr/bin + +# Install HCP Pipelines +RUN apt-get -y update \ + && apt-get install -y --no-install-recommends python-numpy && \ + wget https://github.com/Washington-University/Pipelines/archive/v3.17.0.tar.gz -O pipelines.tar.gz && \ + cd /opt/ && \ + tar zxvf /pipelines.tar.gz && \ + mv /opt/Pipelines-* /opt/HCP-Pipelines && \ + rm /pipelines.tar.gz && \ + cd / + +ENV HCPPIPEDIR=/opt/HCP-Pipelines +ENV HCPPIPEDIR_Templates=${HCPPIPEDIR}/global/templates +ENV HCPPIPEDIR_Bin=${HCPPIPEDIR}/global/binaries +ENV HCPPIPEDIR_Config=${HCPPIPEDIR}/global/config +ENV HCPPIPEDIR_PreFS=${HCPPIPEDIR}/PreFreeSurfer/scripts +ENV HCPPIPEDIR_FS=${HCPPIPEDIR}/FreeSurfer/scripts +ENV HCPPIPEDIR_PostFS=${HCPPIPEDIR}/PostFreeSurfer/scripts +ENV HCPPIPEDIR_fMRISurf=${HCPPIPEDIR}/fMRISurface/scripts +ENV HCPPIPEDIR_fMRIVol=${HCPPIPEDIR}/fMRIVolume/scripts +ENV HCPPIPEDIR_tfMRI=${HCPPIPEDIR}/tfMRI/scripts +ENV HCPPIPEDIR_dMRI=${HCPPIPEDIR}/DiffusionPreprocessing/scripts +ENV HCPPIPEDIR_dMRITract=${HCPPIPEDIR}/DiffusionTractography/scripts +ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts +ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts +ENV MSMBin=${HCPPIPEDIR}/MSMBinaries + +RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools +RUN pip install pybids==0.0.1 +ENV PYTHONPATH="" + +COPY run.py /run.py +RUN chmod +x /run.py + +COPY version /version +ENTRYPOINT ["/run.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..4b61aec --- /dev/null +++ b/README.md @@ -0,0 +1,118 @@ +## HCP Pipelines BIDS App +This a [BIDS App](https://bids-apps.neuroimaging.io) wrapper for [HCP Pipelines](https://github.com/Washington-University/Pipelines). +Like every BIDS App it consists of a container that includes all of the dependencies and run script that parses a [BIDS dataset](http://bids.neuroimaging.io). +BIDS Apps run on Windows, Linux, Mac as well as HCPs/clusters. + +To convert DICOMs from your HCP-Style (CMRR) acquisitions to BIDS try using [heudiconv](https://github.com/nipy/heudiconv) with this [heuristic file](https://github.com/nipy/heudiconv/blob/master/heuristics/cmrr_heuristic.py). + +### Description +The HCP Pipelines product is a set of tools (primarily, but not exclusively, +shell scripts) for processing MRI images for the [Human Connectome Project][HCP]. +Among other things, these tools implement the Minimal Preprocessing Pipeline +(MPP) described in [Glasser et al. 2013][GlasserEtAl]. + +**This BIDS App requires that each subject has at least one T1w and one T2w scan.** Lack fMRI or dMRI scans is handled robustly. Note that while anatomicals (T1w, T2w scans) can be processed without a fieldmap, a fieldmap is mandatory for processing fMRI scans. + + +### Documentation +[Release Notes, Installation, and Usage][release-install-use] + +### How to report errors +Discussion of HCP Pipeline usage and improvements can be posted to the +hcp-users discussion list. Sign up for hcp-users at +[http://humanconnectome.org/contact/#subscribe][hcp-users-subscribe] + +### Acknowledgements +Please cite [Glasser et al. 2013][GlasserEtAl] and [Smith et al. 2013][SmithEtAl]. + +### Usage +This App has the following command line arguments: + + PS C:\Users\filo> docker run -ti --rm -v /c/Users/filo/hcp_example_bids:/bids_dir:ro -v /c/Users/filo/test_output:/output bids/hcppipelines --help + usage: run.py [-h] + [--participant_label PARTICIPANT_LABEL [PARTICIPANT_LABEL ...]] + [--n_cpus N_CPUS] + [--stages {PreFreeSurfer,FreeSurfer,PostFreeSurfer,fMRIVolume,fMRISurface,DiffusionPreprocessing} [{PreFreeSurfer,FreeSurfer,PostFreeSurfer,fMRIVolume,fMRISurface,DiffusionPreprocessing} ...]] + --license_key LICENSE_KEY [-v] + bids_dir output_dir {participant} + + HCP Pipeliens BIDS App (T1w, T2w, fMRI) + + positional arguments: + bids_dir The directory with the input dataset formatted + according to the BIDS standard. + output_dir The directory where the output files should be stored. + If you are running group level analysis this folder + should be prepopulated with the results of + theparticipant level analysis. + {participant} Level of the analysis that will be performed. Multiple + participant level analyses can be run independently + (in parallel) using the same output_dir. + + optional arguments: + -h, --help show this help message and exit + --participant_label PARTICIPANT_LABEL [PARTICIPANT_LABEL ...] + The label of the participant that should be analyzed. + The label corresponds to sub- from + the BIDS spec (so it does not include "sub-"). If this + parameter is not provided all subjects should be + analyzed. Multiple participants can be specified with + a space separated list. + --n_cpus N_CPUS Number of CPUs/cores available to use. + --stages {PreFreeSurfer,FreeSurfer,PostFreeSurfer,fMRIVolume,fMRISurface,DiffusionPreprocessing} [{PreFreeSurfer,FreeSurfer,PostFreeSurfer,fMRIVolume,fMRISurface,DiffusionPreprocessing} ...] + Which stages to run. Space separated list. + --license_key LICENSE_KEY + FreeSurfer license key - letters and numbers after "*" + in the email you received after registration. To + register (for free) visit + https://surfer.nmr.mgh.harvard.edu/registration.html + -v, --version show program's version number and exit + +To run it in participant level mode (for one participant): + + docker run -i --rm \ + -v /Users/filo/data/ds005:/bids_dataset:ro \ + -v /Users/filo/outputs:/outputs \ + bids/hcppipelines \ + /bids_dataset /outputs participant --participant_label 01 --license_key "XXXXXX" + +### Commercial use + +In addition to [FreeSurfer](https://surfer.nmr.mgh.harvard.edu/) the following **non-free** Debian packages are part of this BIDS App: + + non-free/science fsl-5.0-core + non-free/science fsl-5.0-eddy-nonfree + non-free/science fsl-atlases + non-free/science fsl-bangor-cerebellar-atlas + non-free/science fsl-core + non-free/science fsl-harvard-oxford-atlases + non-free/science fsl-harvard-oxford-cortical-lateralized-atlas + non-free/science fsl-jhu-dti-whitematter-atlas + non-free/science fsl-juelich-histological-atlas + non-free/science fsl-mni-structural-atlas + non-free/science fsl-mni152-templates + non-free/science fsl-oxford-striatal-atlas + non-free/science fsl-oxford-thalamic-connectivity-atlas + non-free/science fsl-resting-connectivity-parcellation-atlases + non-free/science fsl-subthalamic-nucleus-atlas + non-free/science fsl-talairach-daemon-atlas + +If you are considering commercial use of this App please consult the relevant licenses. + +### TODO + + - [ ] Add DiffusionProcessing stage + - [ ] More testing for fMRI with different resolution + - [ ] Run fMRI runs in parallel (when n_cpus present) + - [ ] Add support for TOPUP and GE fieldmaps for structural scans (please get in touch if you can provide sample data) + - [ ] Add support for Siemens and GE fieldmaps for fMRI scans (please get in touch if you can provide sample data) + - [ ] Avoid copying fsaverage folder for every participant + - [ ] Add ICA FIX stage + - [ ] Add group level analysis + - [ ] Add task fMRI model fitting + +[HCP]: http://www.humanconnectome.org +[GlasserEtAl]: http://www.ncbi.nlm.nih.gov/pubmed/23668970 +[SmithEtAl]: http://www.ncbi.nlm.nih.gov/pubmed/23702415 +[release-install-use]: https://github.com/Washington-University/Pipelines/wiki/v3.4.0-Release-Notes,-Installation,-and-Usage +[hcp-users-subscribe]: http://humanconnectome.org/contact/#subscribe diff --git a/Singularity b/Singularity new file mode 100644 index 0000000..f25b326 --- /dev/null +++ b/Singularity @@ -0,0 +1,61 @@ +Bootstrap: docker +From: bids/hcppipelines + +%runscript + +exec /run.py "$@" + +%post + +# Make script executable +chmod +x /run.py + +# Make local folders +mkdir /share +mkdir /scratch +mkdir /local-scratch + +# add variables to the environment file +echo "" >> /environment +echo "export FSLDIR=/usr/share/fsl/5.0" >> /environment +echo 'export FSL_DIR="${FSLDIR}"' >> /environment +echo "export FSLOUTPUTTYPE=NIFTI_GZ" >> /environment +echo "export PATH=/usr/lib/fsl/5.0:$PATH" >> /environment +echo "export FSLMULTIFILEQUIT=TRUE" >> /environment +echo "export POSSUMDIR=/usr/share/fsl/5.0" >> /environment +echo "export LD_LIBRARY_PATH=/usr/lib/fsl/5.0:$LD_LIBRARY_PATH" >> /environment +echo "export FSLTCLSH=/usr/bin/tclsh" >> /environment +echo "export FSLWISH=/usr/bin/wish" >> /environment +echo "export FSLOUTPUTTYPE=NIFTI_GZ" >> /environment +echo "export CARET7DIR=/usr/bin" >> /environment +echo "export HCPPIPEDIR=/opt/HCP-Pipelines" >> /environment +echo "export HCPPIPEDIR_Templates=/opt/HCP-Pipelines/global/templates" >> /environment +echo "export HCPPIPEDIR_Bin=/opt/HCP-Pipelines/global/binaries" >> /environment +echo "export HCPPIPEDIR_Config=/opt/HCP-Pipelines/global/config" >> /environment +echo "export HCPPIPEDIR_PreFS=/opt/HCP-Pipelines/PreFreeSurfer/scripts" >> /environment +echo "export HCPPIPEDIR_FS=/opt/HCP-Pipelines/FreeSurfer/scripts" >> /environment +echo "export HCPPIPEDIR_PostFS=/opt/HCP-Pipelines/PostFreeSurfer/scripts" >> /environment +echo "export HCPPIPEDIR_fMRISurf=/opt/HCP-Pipelines/fMRISurface/scripts" >> /environment +echo "export HCPPIPEDIR_fMRIVol=/opt/HCP-Pipelines/fMRIVolume/scripts" >> /environment +echo "export HCPPIPEDIR_tfMRI=/opt/HCP-Pipelines/tfMRI/scripts" >> /environment +echo "export HCPPIPEDIR_dMRI=/opt/HCP-Pipelines/DiffusionPreprocessing/scripts" >> /environment +echo "export HCPPIPEDIR_dMRITract=/opt/HCP-Pipelines/DiffusionTractography/scripts" >> /environment +echo "export HCPPIPEDIR_Global=/opt/HCP-Pipelines/global/scripts" >> /environment +echo "export HCPPIPEDIR_tfMRIAnalysis=/opt/HCP-Pipelines/TaskfMRIAnalysis/scripts" >> /environment +echo "export MSMBin=/opt/HCP-Pipelines/MSMBinaries" >> /environment +echo "export PYTHONPATH=''" >> /environment +echo 'export OS=Linux' >> /environment +echo 'export FS_OVERRIDE=0' >> /environment +echo 'export SUBJECTS_DIR=/opt/freesurfer/subjects' >> /environment +echo 'export FSF_OUTPUT_FORMAT=nii.gz' >> /environment +echo 'export MNI_DIR=/opt/freesurfer/mni' >> /environment +echo 'export LOCAL_DIR=/opt/freesurfer/local' >> /environment +echo 'export FREESURFER_HOME=/opt/freesurfer' >> /environment +echo 'export FSFAST_HOME=/opt/freesurfer/fsfast' >> /environment +echo 'export MINC_BIN_DIR=/opt/freesurfer/mni/bin' >> /environment +echo 'export MINC_LIB_DIR=/opt/freesurfer/mni/lib' >> /environment +echo 'export MNI_DATAPATH=/opt/freesurfer/mni/data' >> /environment +echo 'export FMRI_ANALYSIS_DIR=/opt/freesurfer/fsfast' >> /environment +echo 'export PERL5LIB=/opt/freesurfer/mni/lib/perl5/5.8.5' >> /environment +echo 'export MNI_PERL5LIB=/opt/freesurfer/mni/lib/perl5/5.8.5' >> /environment +echo 'export PATH=/opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH' >> /environment diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..5d4fd3f --- /dev/null +++ b/circle.yml @@ -0,0 +1,38 @@ +general: + artifacts: + - "~/outputs1" +machine: + services: + - docker #don't use 1.10 - caching is broken + +dependencies: + cache_directories: + - "~/docker" + - "~/data" + + override: + - if [[ ! -d ~/data/hcp_example_bids ]]; then wget -c -O ${HOME}/hcp_example_bids.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fd039ad5a101f23cd94f" && mkdir -p ${HOME}/data && unzip ${HOME}/hcp_example_bids.zip -d ${HOME}/data; fi + - if [[ ! -d ~/data/lifespan_example_bids ]]; then wget -c -O ${HOME}/lifespan_example_bids.zip "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/58a1fc4eb83f6901f3b47a54" && mkdir -p ${HOME}/data && unzip ${HOME}/lifespan_example_bids.zip -d ${HOME}/data; fi + - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi + - git describe --tags > version + - docker build -t bids/${CIRCLE_PROJECT_REPONAME,,} . : + timeout: 21600 + - mkdir -p ~/docker; docker save "bids/${CIRCLE_PROJECT_REPONAME,,}" > ~/docker/image.tar + - mkdir -p ${HOME}/outputs + +test: + override: + # print version + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version + # participant level tests for single session dataset + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/hcp_example_bids:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 100307 --stages PreFreeSurfer --license_key="*CxjskRdd7" --n_cpus 2: + timeout: 21600 +deployment: + hub: + owner: BIDS-Apps + tag: /.*/ + commands: + - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker push bids/${CIRCLE_PROJECT_REPONAME,,}:latest; fi : + timeout: 21600 + - if [[ -n "$DOCKER_PASS" ]]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag bids/${CIRCLE_PROJECT_REPONAME,,} bids/${CIRCLE_PROJECT_REPONAME,,}:$CIRCLE_TAG && docker push bids/${CIRCLE_PROJECT_REPONAME,,}:$CIRCLE_TAG; fi : + timeout: 21600 diff --git a/run.py b/run.py new file mode 100644 index 0000000..9ef8c4a --- /dev/null +++ b/run.py @@ -0,0 +1,439 @@ +#!/usr/bin/python +from __future__ import print_function +import argparse +import os +import shutil +import nibabel +from glob import glob +from subprocess import Popen, PIPE +from shutil import rmtree +import subprocess +from bids.grabbids import BIDSLayout +from functools import partial +from collections import OrderedDict + +def run(command, env={}, cwd=None): + merged_env = os.environ + merged_env.update(env) + merged_env.pop("DEBUG", None) + print(command) + process = Popen(command, stdout=PIPE, stderr=subprocess.STDOUT, + shell=True, env=merged_env, cwd=cwd) + while True: + line = process.stdout.readline() + print(line) + line = str(line)[:-1] + if line == '' and process.poll() != None: + break + if process.returncode != 0: + raise Exception("Non zero return code: %d"%process.returncode) + +grayordinatesres = "2" # This is currently the only option for which the is an atlas +lowresmesh = 32 + +def run_pre_freesurfer(**args): + args.update(os.environ) + args["t1"] = "@".join(t1ws) + args["t2"] = "@".join(t2ws) + + cmd = '{HCPPIPEDIR}/PreFreeSurfer/PreFreeSurferPipeline.sh ' + \ + '--path="{path}" ' + \ + '--subject="{subject}" ' + \ + '--t1="{t1}" ' + \ + '--t2="{t2}" ' + \ + '--t1template="{HCPPIPEDIR_Templates}/MNI152_T1_{t1_template_res}mm.nii.gz" ' + \ + '--t1templatebrain="{HCPPIPEDIR_Templates}/MNI152_T1_{t1_template_res}mm_brain.nii.gz" ' + \ + '--t1template2mm="{HCPPIPEDIR_Templates}/MNI152_T1_2mm.nii.gz" ' + \ + '--t2template="{HCPPIPEDIR_Templates}/MNI152_T2_{t2_template_res}mm.nii.gz" ' + \ + '--t2templatebrain="{HCPPIPEDIR_Templates}/MNI152_T2_{t2_template_res}mm_brain.nii.gz" ' + \ + '--t2template2mm="{HCPPIPEDIR_Templates}/MNI152_T2_2mm.nii.gz" ' + \ + '--templatemask="{HCPPIPEDIR_Templates}/MNI152_T1_{t1_template_res}mm_brain_mask.nii.gz" ' + \ + '--template2mmmask="{HCPPIPEDIR_Templates}/MNI152_T1_2mm_brain_mask_dil.nii.gz" ' + \ + '--brainsize="150" ' + \ + '--fnirtconfig="{HCPPIPEDIR_Config}/T1_2_MNI152_2mm.cnf" ' + \ + '--fmapmag="{fmapmag}" ' + \ + '--fmapphase="{fmapphase}" ' + \ + '--fmapgeneralelectric="NONE" ' + \ + '--echodiff="{echodiff}" ' + \ + '--SEPhaseNeg="{SEPhaseNeg}" ' + \ + '--SEPhasePos="{SEPhasePos}" ' + \ + '--echospacing="{echospacing}" ' + \ + '--seunwarpdir="{seunwarpdir}" ' + \ + '--t1samplespacing="{t1samplespacing}" ' + \ + '--t2samplespacing="{t2samplespacing}" ' + \ + '--unwarpdir="{unwarpdir}" ' + \ + '--gdcoeffs="NONE" ' + \ + '--avgrdcmethod={avgrdcmethod} ' + \ + '--topupconfig="{HCPPIPEDIR_Config}/b02b0.cnf" ' + \ + '--printcom=""' + cmd = cmd.format(**args) + run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) + +def run_freesurfer(**args): + args.update(os.environ) + args["subjectDIR"] = os.path.join(args["path"], args["subject"], "T1w") + cmd = '{HCPPIPEDIR}/FreeSurfer/FreeSurferPipeline.sh ' + \ + '--subject="{subject}" ' + \ + '--subjectDIR="{subjectDIR}" ' + \ + '--t1="{path}/{subject}/T1w/T1w_acpc_dc_restore.nii.gz" ' + \ + '--t1brain="{path}/{subject}/T1w/T1w_acpc_dc_restore_brain.nii.gz" ' + \ + '--t2="{path}/{subject}/T1w/T2w_acpc_dc_restore.nii.gz" ' + \ + '--printcom=""' + cmd = cmd.format(**args) + + if not os.path.exists(os.path.join(args["subjectDIR"], "fsaverage")): + shutil.copytree(os.path.join(os.environ["SUBJECTS_DIR"], "fsaverage"), + os.path.join(args["subjectDIR"], "fsaverage")) + if not os.path.exists(os.path.join(args["subjectDIR"], "lh.EC_average")): + shutil.copytree(os.path.join(os.environ["SUBJECTS_DIR"], "lh.EC_average"), + os.path.join(args["subjectDIR"], "lh.EC_average")) + if not os.path.exists(os.path.join(args["subjectDIR"], "rh.EC_average")): + shutil.copytree(os.path.join(os.environ["SUBJECTS_DIR"], "rh.EC_average"), + os.path.join(args["subjectDIR"], "rh.EC_average")) + + run(cmd, cwd=args["path"], env={"NSLOTS": str(args["n_cpus"]), + "OMP_NUM_THREADS": str(args["n_cpus"])}) + +def run_post_freesurfer(**args): + args.update(os.environ) + cmd = '{HCPPIPEDIR}/PostFreeSurfer/PostFreeSurferPipeline.sh ' + \ + '--path="{path}" ' + \ + '--subject="{subject}" ' + \ + '--surfatlasdir="{HCPPIPEDIR_Templates}/standard_mesh_atlases" ' + \ + '--grayordinatesdir="{HCPPIPEDIR_Templates}/91282_Greyordinates" ' + \ + '--grayordinatesres="{grayordinatesres:s}" ' + \ + '--hiresmesh="164" ' + \ + '--lowresmesh="{lowresmesh:d}" ' + \ + '--subcortgraylabels="{HCPPIPEDIR_Config}/FreeSurferSubcorticalLabelTableLut.txt" ' + \ + '--freesurferlabels="{HCPPIPEDIR_Config}/FreeSurferAllLut.txt" ' + \ + '--refmyelinmaps="{HCPPIPEDIR_Templates}/standard_mesh_atlases/Conte69.MyelinMap_BC.164k_fs_LR.dscalar.nii" ' + \ + '--regname="FS" ' + \ + '--printcom=""' + cmd = cmd.format(**args) + run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) + +def run_generic_fMRI_volume_processsing(**args): + args.update(os.environ) + cmd = '{HCPPIPEDIR}/fMRIVolume/GenericfMRIVolumeProcessingPipeline.sh ' + \ + '--path={path} ' + \ + '--subject={subject} ' + \ + '--fmriname={fmriname} ' + \ + '--fmritcs={fmritcs} ' + \ + '--fmriscout={fmriscout} ' + \ + '--SEPhaseNeg={SEPhaseNeg} ' + \ + '--SEPhasePos={SEPhasePos} ' + \ + '--fmapmag="NONE" ' + \ + '--fmapphase="NONE" ' + \ + '--fmapgeneralelectric="NONE" ' + \ + '--echospacing={echospacing} ' + \ + '--echodiff="NONE" ' + \ + '--unwarpdir={unwarpdir} ' + \ + '--fmrires={fmrires:s} ' + \ + '--dcmethod={dcmethod} ' + \ + '--gdcoeffs="NONE" ' + \ + '--topupconfig={HCPPIPEDIR_Config}/b02b0.cnf ' + \ + '--printcom="" ' + \ + '--biascorrection={biascorrection} ' + \ + '--mctype="MCFLIRT"' + cmd = cmd.format(**args) + run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) + +def run_generic_fMRI_surface_processsing(**args): + args.update(os.environ) + cmd = '{HCPPIPEDIR}/fMRISurface/GenericfMRISurfaceProcessingPipeline.sh ' + \ + '--path={path} ' + \ + '--subject={subject} ' + \ + '--fmriname={fmriname} ' + \ + '--lowresmesh="{lowresmesh:d}" ' + \ + '--fmrires={fmrires:s} ' + \ + '--smoothingFWHM={fmrires:s} ' + \ + '--grayordinatesres="{grayordinatesres:s}" ' + \ + '--regname="FS"' + cmd = cmd.format(**args) + run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) + +def run_diffusion_processsing(**args): + args.update(os.environ) + cmd = '{HCPPIPEDIR}/DiffusionPreprocessing/DiffPreprocPipeline.sh ' + \ + '--posData="{posData}" ' +\ + '--negData="{negData}" ' + \ + '--path="{path}" ' + \ + '--subject="{subject}" ' + \ + '--echospacing="{echospacing}" '+ \ + '--PEdir={PEdir} ' + \ + '--gdcoeffs="NONE" ' + \ + '--printcom=""' + cmd = cmd.format(**args) + run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) + +__version__ = open('/version').read() + +parser = argparse.ArgumentParser(description='HCP Pipeliens BIDS App (T1w, T2w, fMRI)') +parser.add_argument('bids_dir', help='The directory with the input dataset ' + 'formatted according to the BIDS standard.') +parser.add_argument('output_dir', help='The directory where the output files ' + 'should be stored. If you are running group level analysis ' + 'this folder should be prepopulated with the results of the' + 'participant level analysis.') +parser.add_argument('analysis_level', help='Level of the analysis that will be performed. ' + 'Multiple participant level analyses can be run independently ' + '(in parallel) using the same output_dir.', + choices=['participant']) +parser.add_argument('--participant_label', help='The label of the participant that should be analyzed. The label ' + 'corresponds to sub- from the BIDS spec ' + '(so it does not include "sub-"). If this parameter is not ' + 'provided all subjects should be analyzed. Multiple ' + 'participants can be specified with a space separated list.', + nargs="+") +parser.add_argument('--n_cpus', help='Number of CPUs/cores available to use.', + default=1, type=int) +parser.add_argument('--stages', help='Which stages to run. Space separated list.', + nargs="+", choices=['PreFreeSurfer', 'FreeSurfer', + 'PostFreeSurfer', 'fMRIVolume', + 'fMRISurface', 'DiffusionPreprocessing'], + default=['PreFreeSurfer', 'FreeSurfer', 'PostFreeSurfer', + 'fMRIVolume', 'fMRISurface', + 'DiffusionPreprocessing']) +parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', + required=True) +parser.add_argument('-v', '--version', action='version', + version='HCP Pielines BIDS App version {}'.format(__version__)) + +args = parser.parse_args() + + + +run("bids-validator " + args.bids_dir) + +layout = BIDSLayout(args.bids_dir) +subjects_to_analyze = [] +# only for a subset of subjects +if args.participant_label: + subjects_to_analyze = args.participant_label +# for all subjects +else: + subject_dirs = glob(os.path.join(args.bids_dir, "sub-*")) + subjects_to_analyze = [subject_dir.split("-")[-1] for subject_dir in subject_dirs] + +# running participant level +if args.analysis_level == "participant": + # find all T1s and skullstrip them + for subject_label in subjects_to_analyze: + t1ws = [f.filename for f in layout.get(subject=subject_label, + type='T1w', + extensions=["nii.gz", "nii"])] + t2ws = [f.filename for f in layout.get(subject=subject_label, + type='T2w', + extensions=["nii.gz", "nii"])] + assert (len(t1ws) > 0), "No T1w files found for subject %s!"%subject_label + assert (len(t2ws) > 0), "No T2w files found for subject %s!"%subject_label + + available_resolutions = ["0.7", "0.8", "1"] + t1_zooms = nibabel.load(t1ws[0]).get_header().get_zooms() + t1_res = float(min(t1_zooms[:3])) + t1_template_res = min(available_resolutions, key=lambda x:abs(float(x)-t1_res)) + t2_zooms = nibabel.load(t2ws[0]).get_header().get_zooms() + t2_res = float(min(t2_zooms[:3])) + t2_template_res = min(available_resolutions, key=lambda x:abs(float(x)-t2_res)) + + fieldmap_set = layout.get_fieldmap(t1ws[0]) + fmap_args = {"fmapmag": "NONE", + "fmapphase": "NONE", + "echodiff": "NONE", + "t1samplespacing": "NONE", + "t2samplespacing": "NONE", + "unwarpdir": "NONE", + "avgrdcmethod": "NONE", + "SEPhaseNeg": "NONE", + "SEPhasePos": "NONE", + "echospacing": "NONE", + "seunwarpdir": "NONE"} + + if fieldmap_set: + t1_spacing = layout.get_metadata(t1ws[0])["EffectiveEchoSpacing"] + t2_spacing = layout.get_metadata(t2ws[0])["EffectiveEchoSpacing"] + + unwarpdir = layout.get_metadata(t1ws[0])["PhaseEncodingDirection"] + unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") + if len(unwarpdir) == 2: + unwarpdir = unwarpdir[0]+"-" + + fmap_args.update({"t1samplespacing": "%.8f"%t1_spacing, + "t2samplespacing": "%.8f"%t2_spacing, + "unwarpdir": unwarpdir}) + + if fieldmap_set["type"] == "phasediff": + merged_file = "%s/tmp/%s/magfile.nii.gz"%(args.output_dir, subject_label) + run("mkdir -p %s/tmp/%s/ && fslmerge -t %s %s %s"%(args.output_dir, + subject_label, + merged_file, + fieldmap_set["magnitude1"], + fieldmap_set["magnitude2"])) + + phasediff_metadata = layout.get_metadata(fieldmap_set["phasediff"]) + te_diff = phasediff_metadata["EchoTime2"] - phasediff_metadata["EchoTime1"] + # HCP expects TE in miliseconds + te_diff = te_diff*1000.0 + + fmap_args.update({"fmapmag": merged_file, + "fmapphase": fieldmap_set["phasediff"], + "echodiff": "%.6f"%te_diff, + "avgrdcmethod": "SiemensFieldMap"}) + elif fieldmap_set["type"] == "epi": + SEPhaseNeg = None + SEPhasePos = None + for fieldmap in fieldmap_set["epi"]: + enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] + if "-" in enc_dir: + SEPhaseNeg = fieldmap + else: + SEPhasePos = fieldmap + + seunwarpdir = layout.get_metadata(fieldmap_set["epi"][0])["PhaseEncodingDirection"] + seunwarpdir = seunwarpdir.replace("-", "").replace("i","x").replace("j", "y").replace("k", "z") + + #TODO check consistency of echo spacing instead of assuming it's all the same + if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap_set["epi"][0]): + echospacing = layout.get_metadata(fieldmap_set["epi"][0])["EffectiveEchoSpacing"] + elif "TotalReadoutTime" in layout.get_metadata(fieldmap_set["epi"][0]): + # HCP Pipelines do not allow users to specify total readout time directly + # Hence we need to reverse the calculations to provide echo spacing that would + # result in the right total read out total read out time + # see https://github.com/Washington-University/Pipelines/blob/master/global/scripts/TopupPreprocessingAll.sh#L202 + print("BIDS App wrapper: Did not find EffectiveEchoSpacing, calculating it from TotalReadoutTime") + # TotalReadoutTime = EffectiveEchoSpacing * (len(PhaseEncodingDirection) - 1) + total_readout_time = layout.get_metadata(fieldmap_set["epi"][0])["TotalReadoutTime"] + phase_len = nibabel.load(fieldmap_set["epi"][0]).shape[{"x": 0, "y": 1}[seunwarpdir]] + echospacing = total_readout_time / float(phase_len - 1) + else: + raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") + + fmap_args.update({"SEPhaseNeg": SEPhaseNeg, + "SEPhasePos": SEPhasePos, + "echospacing": "%.6f"%echospacing, + "seunwarpdir": seunwarpdir, + "avgrdcmethod": "TOPUP"}) + #TODO add support for GE fieldmaps + + struct_stages_dict = OrderedDict([("PreFreeSurfer", partial(run_pre_freesurfer, + path=args.output_dir, + subject="sub-%s"%subject_label, + t1ws=t1ws, + t2ws=t2ws, + n_cpus=args.n_cpus, + t1_template_res=t1_template_res, + t2_template_res=t2_template_res, + **fmap_args)), + ("FreeSurfer", partial(run_freesurfer, + path=args.output_dir, + subject="sub-%s"%subject_label, + n_cpus=args.n_cpus)), + ("PostFreeSurfer", partial(run_post_freesurfer, + path=args.output_dir, + subject="sub-%s"%subject_label, + grayordinatesres=grayordinatesres, + lowresmesh=lowresmesh, + n_cpus=args.n_cpus)) + ]) + for stage, stage_func in struct_stages_dict.iteritems(): + if stage in args.stages: + stage_func() + + bolds = [f.filename for f in layout.get(subject=subject_label, + type='bold', + extensions=["nii.gz", "nii"])] + for fmritcs in bolds: + fmriname = "_".join(fmritcs.split("sub-")[-1].split("_")[1:]).split(".")[0] + assert fmriname + + fmriscout = fmritcs.replace("_bold", "_sbref") + if not os.path.exists(fmriscout): + fmriscout = "NONE" + + fieldmap_set = layout.get_fieldmap(fmritcs) + if fieldmap_set and fieldmap_set["type"] == "epi": + SEPhaseNeg = None + SEPhasePos = None + for fieldmap in fieldmap_set["epi"]: + enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] + if "-" in enc_dir: + SEPhaseNeg = fieldmap + else: + SEPhasePos = fieldmap + echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] + unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] + unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") + if len(unwarpdir) == 2: + unwarpdir = "-" + unwarpdir[0] + dcmethod = "TOPUP" + biascorrection = "SEBASED" + else: + SEPhaseNeg = "NONE" + SEPhasePos = "NONE" + echospacing = "NONE" + unwarpdir = "NONE" + dcmethod = "NONE" + biascorrection = "NONE" + + zooms = nibabel.load(fmritcs).get_header().get_zooms() + fmrires = float(min(zooms[:3])) + fmrires = "2" # https://github.com/Washington-University/Pipelines/blob/637b35f73697b77dcb1d529902fc55f431f03af7/fMRISurface/scripts/SubcorticalProcessing.sh#L43 + # While running '/usr/bin/wb_command -cifti-create-dense-timeseries /scratch/users/chrisgor/hcp_output2/sub-100307/MNINonLinear/Results/EMOTION/EMOTION_temp_subject.dtseries.nii -volume /scratch/users/chrisgor/hcp_output2/sub-100307/MNINonLinear/Results/EMOTION/EMOTION.nii.gz /scratch/users/chrisgor/hcp_output2/sub-100307/MNINonLinear/ROIs/ROIs.2.nii.gz': + # ERROR: label volume has a different volume space than data volume + + + func_stages_dict = OrderedDict([("fMRIVolume", partial(run_generic_fMRI_volume_processsing, + path=args.output_dir, + subject="sub-%s"%subject_label, + fmriname=fmriname, + fmritcs=fmritcs, + fmriscout=fmriscout, + SEPhaseNeg=SEPhaseNeg, + SEPhasePos=SEPhasePos, + echospacing=echospacing, + unwarpdir=unwarpdir, + fmrires=fmrires, + dcmethod=dcmethod, + biascorrection=biascorrection, + n_cpus=args.n_cpus)), + ("fMRISurface", partial(run_generic_fMRI_surface_processsing, + path=args.output_dir, + subject="sub-%s"%subject_label, + fmriname=fmriname, + fmrires=fmrires, + n_cpus=args.n_cpus, + grayordinatesres=grayordinatesres, + lowresmesh=lowresmesh)) + ]) + for stage, stage_func in func_stages_dict.iteritems(): + if stage in args.stages: + stage_func() + + dwis = layout.get(subject=subject_label, type='dwi', + extensions=["nii.gz", "nii"]) + + # print(dwis) + # acqs = set(layout.get(target='acquisition', return_type='id', + # subject=subject_label, type='dwi', + # extensions=["nii.gz", "nii"])) + # print(acqs) + # posData = [] + # negData = [] + # for acq in acqs: + # pos = "EMPTY" + # neg = "EMPTY" + # dwis = layout.get(subject=subject_label, + # type='dwi', acquisition=acq, + # extensions=["nii.gz", "nii"]) + # assert len(dwis) <= 2 + # for dwi in dwis: + # dwi = dwi.filename + # if "-" in layout.get_metadata(dwi)["PhaseEncodingDirection"]: + # neg = dwi + # else: + # pos = dwi + # posData.append(pos) + # negData.append(neg) + # + # print(negData) + # print(posData) diff --git a/version b/version new file mode 100644 index 0000000..38f8e88 --- /dev/null +++ b/version @@ -0,0 +1 @@ +dev From 56f01cb22ae727108f0ad2f69b2d97234ba5d443 Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 29 Mar 2018 15:43:03 -0500 Subject: [PATCH 02/74] made changes to run.py in order to account for my BIDS infrastructure --- Dockerfile | 2 +- run.py | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 89791f1..d1344ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,7 @@ ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMBin=${HCPPIPEDIR}/MSMBinaries -RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools +RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools nano RUN pip install pybids==0.0.1 ENV PYTHONPATH="" diff --git a/run.py b/run.py index 9ef8c4a..7364384 100644 --- a/run.py +++ b/run.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python -v from __future__ import print_function import argparse import os @@ -225,6 +225,7 @@ def run_diffusion_processsing(**args): t2ws = [f.filename for f in layout.get(subject=subject_label, type='T2w', extensions=["nii.gz", "nii"])] + fieldmaps = [f for f in layout.get(subject=subject_label, type=['epi', 'phasediff'], extensions=["nii.gz", "nii"])] assert (len(t1ws) > 0), "No T1w files found for subject %s!"%subject_label assert (len(t2ws) > 0), "No T2w files found for subject %s!"%subject_label @@ -236,7 +237,9 @@ def run_diffusion_processsing(**args): t2_res = float(min(t2_zooms[:3])) t2_template_res = min(available_resolutions, key=lambda x:abs(float(x)-t2_res)) - fieldmap_set = layout.get_fieldmap(t1ws[0]) + fieldmap_set = fieldmaps + fieldmap_num = len(fieldmaps) + fmap_args = {"fmapmag": "NONE", "fmapphase": "NONE", "echodiff": "NONE", @@ -279,31 +282,33 @@ def run_diffusion_processsing(**args): "fmapphase": fieldmap_set["phasediff"], "echodiff": "%.6f"%te_diff, "avgrdcmethod": "SiemensFieldMap"}) - elif fieldmap_set["type"] == "epi": + elif fieldmap_set[0][1] == "epi": SEPhaseNeg = None SEPhasePos = None - for fieldmap in fieldmap_set["epi"]: + for i in len(fieldmap_set): + fieldmap = fieldmap_set[i][0] enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] if "-" in enc_dir: SEPhaseNeg = fieldmap else: SEPhasePos = fieldmap - seunwarpdir = layout.get_metadata(fieldmap_set["epi"][0])["PhaseEncodingDirection"] + + seunwarpdir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] seunwarpdir = seunwarpdir.replace("-", "").replace("i","x").replace("j", "y").replace("k", "z") #TODO check consistency of echo spacing instead of assuming it's all the same - if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap_set["epi"][0]): - echospacing = layout.get_metadata(fieldmap_set["epi"][0])["EffectiveEchoSpacing"] - elif "TotalReadoutTime" in layout.get_metadata(fieldmap_set["epi"][0]): + if "EffectiveEchoSpacing" in layout.get_metadata(fieldmap): + echospacing = layout.get_metadata(fieldmap)["EffectiveEchoSpacing"] + elif "TotalReadoutTime" in layout.get_metadata(fieldmap): # HCP Pipelines do not allow users to specify total readout time directly # Hence we need to reverse the calculations to provide echo spacing that would # result in the right total read out total read out time # see https://github.com/Washington-University/Pipelines/blob/master/global/scripts/TopupPreprocessingAll.sh#L202 print("BIDS App wrapper: Did not find EffectiveEchoSpacing, calculating it from TotalReadoutTime") # TotalReadoutTime = EffectiveEchoSpacing * (len(PhaseEncodingDirection) - 1) - total_readout_time = layout.get_metadata(fieldmap_set["epi"][0])["TotalReadoutTime"] - phase_len = nibabel.load(fieldmap_set["epi"][0]).shape[{"x": 0, "y": 1}[seunwarpdir]] + total_readout_time = layout.get_metadata(fieldmap)["TotalReadoutTime"] + phase_len = nibabel.load(fieldmap).shape[{"x": 0, "y": 1}[seunwarpdir]] echospacing = total_readout_time / float(phase_len - 1) else: raise RuntimeError("EffectiveEchoSpacing or TotalReadoutTime not defined for the fieldmap intended for T1w image. Please fix your BIDS dataset.") From 8f46b5bb7a01da269e9a5e72ecd292549bc1e101 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 3 Apr 2018 12:48:56 -0500 Subject: [PATCH 03/74] last commit prior to removing branch --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 7364384..f0e1574 100644 --- a/run.py +++ b/run.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -v +#!/usr/bin/python from __future__ import print_function import argparse import os From 53d615b39c54db467a0f30c6327a0dbbdab05dcd Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 3 Apr 2018 12:53:40 -0500 Subject: [PATCH 04/74] Added IntendedFor.py to create IntendedFor field if it does not exist within fieldmap json sidecars --- IntendedFor.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 IntendedFor.py diff --git a/IntendedFor.py b/IntendedFor.py new file mode 100755 index 0000000..51f4911 --- /dev/null +++ b/IntendedFor.py @@ -0,0 +1,47 @@ +import json +import os +from glob import glob + +""" +This script is designed to determine which field maps apply to discrete fMRI scans + +Author - Timothy J Hendrickson +""" + +#subject_path = #"/home/timothy/sandbox_DO_NOT_DELETE/BIDS/263_ETOH_BIDS/output/sub-8404" + +def setup(subject_path): + if "ses" in os.listdir(subject_path)[0]: + for item in (os.listdir(subject_path)): + session_path = subject_path + '/'+ item + IntendedFor(session_path) + IntendedFor(subject_path) + +def IntendedFor(data_path): + for fmap in glob(data_path+'/fmap/*.json'): + with open(fmap, 'r') as f: + fmap_json = json.load(f) + f.close() + if not "IntendedFor" in fmap_json: + shim_fmap = fmap_json["ShimSetting"] + func_list = [] + for func in glob(data_path+'/func/*bold.json'): + with open(func, 'r') as g: + func_json = json.load(g) + shim_func = func_json["ShimSetting"] + g.close() + if shim_fmap == shim_func: + func_nii = glob(data_path+'/func/' +func.split('/')[-1].split('.')[0]+".nii*")[0] + func_list.append(func_nii) + entry = {"IntendedFor": func_list} + + fmap_json.update(entry) + with open(fmap, 'w') as f: + json.dump(fmap_json, f) + f.close() + else: + print(fmap + " already exists for this session") +setup(subject_path) + + + From 4f98e772eb7a1a58427fd69eb6d6ff82e9186ec7 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 3 Apr 2018 16:20:40 -0500 Subject: [PATCH 05/74] added IntendedFor scrit to run.py so it will take care of adding IntendedFor field at time of execution --- Dockerfile | 1 + IntendedFor.py | 13 ++++++------- run.py | 12 ++++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index d1344ce..b3eeb4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,4 +107,5 @@ COPY run.py /run.py RUN chmod +x /run.py COPY version /version +COPY IntendedFor.py /IntendedFor.py ENTRYPOINT ["/run.py"] diff --git a/IntendedFor.py b/IntendedFor.py index 51f4911..d564ed8 100755 --- a/IntendedFor.py +++ b/IntendedFor.py @@ -8,8 +8,6 @@ Author - Timothy J Hendrickson """ -#subject_path = #"/home/timothy/sandbox_DO_NOT_DELETE/BIDS/263_ETOH_BIDS/output/sub-8404" - def setup(subject_path): if "ses" in os.listdir(subject_path)[0]: for item in (os.listdir(subject_path)): @@ -32,7 +30,12 @@ def IntendedFor(data_path): g.close() if shim_fmap == shim_func: func_nii = glob(data_path+'/func/' +func.split('/')[-1].split('.')[0]+".nii*")[0] - func_list.append(func_nii) + if "ses" in data_path: + func_nii = "/".join(func_nii.split("/")[-3:]) + func_list.append(func_nii) + else: + func_nii = "/".join(func_nii.split("/")[-2:]) + func_list.append(func_nii) entry = {"IntendedFor": func_list} fmap_json.update(entry) @@ -41,7 +44,3 @@ def IntendedFor(data_path): f.close() else: print(fmap + " already exists for this session") -setup(subject_path) - - - diff --git a/run.py b/run.py index f0e1574..20671ea 100644 --- a/run.py +++ b/run.py @@ -11,6 +11,7 @@ from bids.grabbids import BIDSLayout from functools import partial from collections import OrderedDict +from IntendedFor import setup, IntendedFor def run(command, env={}, cwd=None): merged_env = os.environ @@ -166,7 +167,8 @@ def run_diffusion_processsing(**args): cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) -__version__ = open('/version').read() +#__version__ = open('/version').read() +__version__ = open('/home/timothy/projects/BIDS_apps/HCPPipelines/version').read() parser = argparse.ArgumentParser(description='HCP Pipeliens BIDS App (T1w, T2w, fMRI)') parser.add_argument('bids_dir', help='The directory with the input dataset ' @@ -197,7 +199,7 @@ def run_diffusion_processsing(**args): parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', required=True) parser.add_argument('-v', '--version', action='version', - version='HCP Pielines BIDS App version {}'.format(__version__)) + version='HCP Pipelines BIDS App version {}'.format(__version__)) args = parser.parse_args() @@ -205,7 +207,7 @@ def run_diffusion_processsing(**args): run("bids-validator " + args.bids_dir) -layout = BIDSLayout(args.bids_dir) +#layout = BIDSLayout(args.bids_dir) subjects_to_analyze = [] # only for a subset of subjects if args.participant_label: @@ -217,8 +219,10 @@ def run_diffusion_processsing(**args): # running participant level if args.analysis_level == "participant": - # find all T1s and skullstrip them for subject_label in subjects_to_analyze: + # before doing anything else make sure that the fieldmap json files are set up correctly + setup(os.path.join(args.bids_dir, "sub-"+subject_label)) + t1ws = [f.filename for f in layout.get(subject=subject_label, type='T1w', extensions=["nii.gz", "nii"])] From 8ac3dad8a3659030aa725c48bca590cb1c05e772 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 3 Apr 2018 16:23:58 -0500 Subject: [PATCH 06/74] changed run.py for Dockerfile implementation --- run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 20671ea..55b116a 100644 --- a/run.py +++ b/run.py @@ -167,8 +167,8 @@ def run_diffusion_processsing(**args): cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) -#__version__ = open('/version').read() -__version__ = open('/home/timothy/projects/BIDS_apps/HCPPipelines/version').read() +__version__ = open('/version').read() +#__version__ = open('/home/timothy/projects/BIDS_apps/HCPPipelines/version').read() parser = argparse.ArgumentParser(description='HCP Pipeliens BIDS App (T1w, T2w, fMRI)') parser.add_argument('bids_dir', help='The directory with the input dataset ' @@ -207,7 +207,7 @@ def run_diffusion_processsing(**args): run("bids-validator " + args.bids_dir) -#layout = BIDSLayout(args.bids_dir) +layout = BIDSLayout(args.bids_dir) subjects_to_analyze = [] # only for a subset of subjects if args.participant_label: From 6c6178875b3f5c9f0cfc4650cc8bef0ed5ac3560 Mon Sep 17 00:00:00 2001 From: Timothy Date: Fri, 6 Apr 2018 08:30:01 -0500 Subject: [PATCH 07/74] added intendedfor script to run.py to add intendedfor field if it doesn't exist --- run.py | 68 +++++++++++++++++++++++++++------------------------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/run.py b/run.py index 55b116a..d7e416f 100644 --- a/run.py +++ b/run.py @@ -7,6 +7,7 @@ from glob import glob from subprocess import Popen, PIPE from shutil import rmtree +import pdb import subprocess from bids.grabbids import BIDSLayout from functools import partial @@ -229,7 +230,6 @@ def run_diffusion_processsing(**args): t2ws = [f.filename for f in layout.get(subject=subject_label, type='T2w', extensions=["nii.gz", "nii"])] - fieldmaps = [f for f in layout.get(subject=subject_label, type=['epi', 'phasediff'], extensions=["nii.gz", "nii"])] assert (len(t1ws) > 0), "No T1w files found for subject %s!"%subject_label assert (len(t2ws) > 0), "No T2w files found for subject %s!"%subject_label @@ -241,9 +241,7 @@ def run_diffusion_processsing(**args): t2_res = float(min(t2_zooms[:3])) t2_template_res = min(available_resolutions, key=lambda x:abs(float(x)-t2_res)) - fieldmap_set = fieldmaps - fieldmap_num = len(fieldmaps) - + fieldmap_set = layout.get_fieldmap(t1ws[0]) fmap_args = {"fmapmag": "NONE", "fmapphase": "NONE", "echodiff": "NONE", @@ -286,11 +284,10 @@ def run_diffusion_processsing(**args): "fmapphase": fieldmap_set["phasediff"], "echodiff": "%.6f"%te_diff, "avgrdcmethod": "SiemensFieldMap"}) - elif fieldmap_set[0][1] == "epi": + elif fieldmap_set["type"] == "epi": SEPhaseNeg = None SEPhasePos = None - for i in len(fieldmap_set): - fieldmap = fieldmap_set[i][0] + for fieldmap in fieldmap_set["epi"]: enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] if "-" in enc_dir: SEPhaseNeg = fieldmap @@ -354,7 +351,7 @@ def run_diffusion_processsing(**args): for fmritcs in bolds: fmriname = "_".join(fmritcs.split("sub-")[-1].split("_")[1:]).split(".")[0] assert fmriname - + pdb.set_trace() fmriscout = fmritcs.replace("_bold", "_sbref") if not os.path.exists(fmriscout): fmriscout = "NONE" @@ -417,32 +414,31 @@ def run_diffusion_processsing(**args): for stage, stage_func in func_stages_dict.iteritems(): if stage in args.stages: stage_func() - - dwis = layout.get(subject=subject_label, type='dwi', + pdb.set_trace() + dwis = layout.get(subject=subject_label, type='dwi', extensions=["nii.gz", "nii"]) - - # print(dwis) - # acqs = set(layout.get(target='acquisition', return_type='id', - # subject=subject_label, type='dwi', - # extensions=["nii.gz", "nii"])) - # print(acqs) - # posData = [] - # negData = [] - # for acq in acqs: - # pos = "EMPTY" - # neg = "EMPTY" - # dwis = layout.get(subject=subject_label, - # type='dwi', acquisition=acq, - # extensions=["nii.gz", "nii"]) - # assert len(dwis) <= 2 - # for dwi in dwis: - # dwi = dwi.filename - # if "-" in layout.get_metadata(dwi)["PhaseEncodingDirection"]: - # neg = dwi - # else: - # pos = dwi - # posData.append(pos) - # negData.append(neg) - # - # print(negData) - # print(posData) + print(dwis) + acqs = set(layout.get(target='acquisition', return_type='id', + subject=subject_label, type='dwi', + extensions=["nii.gz", "nii"])) + print(acqs) + posData = [] + negData = [] + for acq in acqs: + pos = "EMPTY" + neg = "EMPTY" + dwis = layout.get(subject=subject_label, + type='dwi', acquisition=acq, + extensions=["nii.gz", "nii"]) + assert len(dwis) <= 2 + for dwi in dwis: + dwi = dwi.filename + if "-" in layout.get_metadata(dwi)["PhaseEncodingDirection"]: + neg = dwi + else: + pos = dwi + posData.append(pos) + negData.append(neg) + + print(negData) + print(posData) \ No newline at end of file From bbab867540b88964f22a2d63f04a6e98140dd7ea Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 10 Apr 2018 09:17:32 -0500 Subject: [PATCH 08/74] changed set_fieldmap with return_list=True to allow for multiple field maps if they exist --- Dockerfile | 12 ++++++++---- IntendedFor.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ run.py | 6 +++++- 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100755 IntendedFor.py diff --git a/Dockerfile b/Dockerfile index 89791f1..8418a96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ + # Use Ubuntu 14.04 LTS FROM ubuntu:trusty-20170119 @@ -51,8 +52,8 @@ ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools: RUN apt-get update && \ apt-get install -y --no-install-recommends curl && \ curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update && \ + apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 +RUN apt-get update && \ apt-get install -y fsl-core=5.0.9-4~nd14.04+1 # Configure environment @@ -100,11 +101,14 @@ ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMBin=${HCPPIPEDIR}/MSMBinaries RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools -RUN pip install pybids==0.0.1 +RUN pip install pybids==0.5.1 +RUN pip install --upgrade pybids ENV PYTHONPATH="" COPY run.py /run.py -RUN chmod +x /run.py +RUN chmod 111 /run.py COPY version /version +COPY IntendedFor.py /IntendedFor.py +RUN chmod 111 /IntendedFor.py ENTRYPOINT ["/run.py"] diff --git a/IntendedFor.py b/IntendedFor.py new file mode 100755 index 0000000..a3f86ab --- /dev/null +++ b/IntendedFor.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +import json +import os +from glob import glob + +""" +This script is designed to determine which field maps apply to discrete fMRI scans + +Author - Timothy J Hendrickson +""" + +def setup(subject_path): + if "ses" in os.listdir(subject_path)[0]: + for item in (os.listdir(subject_path)): + session_path = subject_path + '/'+ item + IntendedFor(session_path) + IntendedFor(subject_path) + +def IntendedFor(data_path): + for fmap in glob(data_path+'/fmap/*.json'): + with open(fmap, 'r') as f: + fmap_json = json.load(f) + f.close() + if not "IntendedFor" in fmap_json: + shim_fmap = fmap_json["ShimSetting"] + func_list = [] + for func in glob(data_path+'/func/*bold.json'): + with open(func, 'r') as g: + func_json = json.load(g) + shim_func = func_json["ShimSetting"] + g.close() + if shim_fmap == shim_func: + func_nii = glob(data_path+'/func/' +func.split('/')[-1].split('.')[0]+".nii*")[0] + if "ses" in data_path: + func_nii = "/".join(func_nii.split("/")[-3:]) + func_list.append(func_nii) + else: + func_nii = "/".join(func_nii.split("/")[-2:]) + func_list.append(func_nii) + entry = {"IntendedFor": func_list} + + fmap_json.update(entry) + with open(fmap, 'w') as f: + json.dump(fmap_json, f) + f.close() + else: + print(fmap + " already exists for this session") diff --git a/run.py b/run.py index 9ef8c4a..b9f8215 100644 --- a/run.py +++ b/run.py @@ -11,6 +11,7 @@ from bids.grabbids import BIDSLayout from functools import partial from collections import OrderedDict +from IntendedFor import setup,IntendedFor def run(command, env={}, cwd=None): merged_env = os.environ @@ -219,6 +220,9 @@ def run_diffusion_processsing(**args): if args.analysis_level == "participant": # find all T1s and skullstrip them for subject_label in subjects_to_analyze: + + # before do anything else add IntendedFor field to fieldmap + setup(os.path.join(args.bids_dir, "sub-"+subject_label)) t1ws = [f.filename for f in layout.get(subject=subject_label, type='T1w', extensions=["nii.gz", "nii"])] @@ -350,7 +354,7 @@ def run_diffusion_processsing(**args): if not os.path.exists(fmriscout): fmriscout = "NONE" - fieldmap_set = layout.get_fieldmap(fmritcs) + fieldmap_set = layout.get_fieldmap(fmritcs, return_list=True) if fieldmap_set and fieldmap_set["type"] == "epi": SEPhaseNeg = None SEPhasePos = None From 4696caa8ecfcacde3796f18ebe04023d175549a3 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 10 Apr 2018 14:27:30 -0500 Subject: [PATCH 09/74] made changes to IntendedFor.py for user determined fudge factor between fieldmap and func shim settings --- Dockerfile | 28 +++++------------------- IntendedFor.py | 58 +++++++++++++++++++++++++++++++++----------------- run.py | 58 ++++++++++++++++++++++---------------------------- 3 files changed, 69 insertions(+), 75 deletions(-) diff --git a/Dockerfile b/Dockerfile index de38e62..5fea4ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,3 @@ -<<<<<<< HEAD - -======= ->>>>>>> ce2c6dbfe600164acbcb75a9fb83d54e440c15af # Use Ubuntu 14.04 LTS FROM ubuntu:trusty-20170119 @@ -31,7 +27,7 @@ RUN apt-get -y update \ --exclude='freesurfer/average/mult-comp-cor' \ --exclude='freesurfer/lib/cuda' \ --exclude='freesurfer/lib/qt' && \ - apt-get install -y tcsh bc tar libgomp1 perl-modules curl + apt-get install -y tcsh bc tar libgomp1 perl-modules curl # Set up the environment ENV OS Linux @@ -55,13 +51,8 @@ ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools: RUN apt-get update && \ apt-get install -y --no-install-recommends curl && \ curl -sSL http://neuro.debian.net/lists/trusty.us-ca.full >> /etc/apt/sources.list.d/neurodebian.sources.list && \ -<<<<<<< HEAD apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 -RUN apt-get update && \ -======= - apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update && \ ->>>>>>> ce2c6dbfe600164acbcb75a9fb83d54e440c15af +RUN apt-get update && \ apt-get install -y fsl-core=5.0.9-4~nd14.04+1 # Configure environment @@ -108,25 +99,16 @@ ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMBin=${HCPPIPEDIR}/MSMBinaries -RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools -<<<<<<< HEAD +RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools RUN pip install pybids==0.5.1 RUN pip install --upgrade pybids ENV PYTHONPATH="" COPY run.py /run.py -RUN chmod 111 /run.py +RUN chmod 555 /run.py COPY version /version COPY IntendedFor.py /IntendedFor.py -RUN chmod 111 /IntendedFor.py -======= -RUN pip install pybids==0.0.1 -ENV PYTHONPATH="" +RUN chmod 555 /IntendedFor.py -COPY run.py /run.py -RUN chmod +x /run.py - -COPY version /version ->>>>>>> ce2c6dbfe600164acbcb75a9fb83d54e440c15af ENTRYPOINT ["/run.py"] diff --git a/IntendedFor.py b/IntendedFor.py index a3f86ab..3e5f623 100755 --- a/IntendedFor.py +++ b/IntendedFor.py @@ -2,13 +2,14 @@ import json import os from glob import glob +import numpy """ This script is designed to determine which field maps apply to discrete fMRI scans Author - Timothy J Hendrickson """ - +subject_path = "/home/naxos2-raid8/mangia-data/DATA_R01_rsfMRI/analyzed/BIDS_output/sub-patient03" def setup(subject_path): if "ses" in os.listdir(subject_path)[0]: for item in (os.listdir(subject_path)): @@ -17,31 +18,50 @@ def setup(subject_path): IntendedFor(subject_path) def IntendedFor(data_path): - for fmap in glob(data_path+'/fmap/*.json'): + for fmap in sorted(glob(data_path+'/fmap/*.json')): with open(fmap, 'r') as f: fmap_json = json.load(f) f.close() - if not "IntendedFor" in fmap_json: - shim_fmap = fmap_json["ShimSetting"] - func_list = [] - for func in glob(data_path+'/func/*bold.json'): - with open(func, 'r') as g: - func_json = json.load(g) - shim_func = func_json["ShimSetting"] - g.close() - if shim_fmap == shim_func: - func_nii = glob(data_path+'/func/' +func.split('/')[-1].split('.')[0]+".nii*")[0] + if "IntendedFor" in fmap_json: + del fmap_json["IntendedFor"] + shim_fmap = fmap_json["ShimSetting"] + patient_pos_fmap = fmap_json["ImageOrientationPatientDICOM"] + func_list = [] + for func in sorted(glob(data_path+'/func/*bold.json')): + with open(func, 'r') as g: + func_json = json.load(g) + shim_func = func_json["ShimSetting"] + patient_pos_func = func_json["ImageOrientationPatientDICOM"] + g.close() + if shim_fmap == shim_func: + func_nii = glob(data_path+'/func/' +func.split('/')[-1].split('.')[0]+".nii*")[0] + if "ses" in data_path: + func_nii = "/".join(func_nii.split("/")[-3:]) + func_list.append(func_nii) + else: + func_nii = "/".join(func_nii.split("/")[-2:]) + func_list.append(func_nii) + elif patient_pos_fmap == patient_pos_func: + shim_func_np = numpy.array(shim_func) + shim_fmap_np = numpy.array(shim_fmap) + diff_shims = shim_fmap_np - shim_func_np + print("difference between fieldmap and functional scan is " + str(sum(abs(diff_shims)))) + user_input = raw_input("Do you accept this difference? [y/n]: ") + if user_input == 'y': + func_nii = glob(data_path + '/func/' + func.split('/')[-1].split('.')[0] + ".nii*")[0] if "ses" in data_path: func_nii = "/".join(func_nii.split("/")[-3:]) func_list.append(func_nii) else: func_nii = "/".join(func_nii.split("/")[-2:]) func_list.append(func_nii) - entry = {"IntendedFor": func_list} + else: + print(fmap + " does not have match yet...") + entry = {"IntendedFor": func_list} + + fmap_json.update(entry) + with open(fmap, 'w') as f: + json.dump(fmap_json, f) + f.close() - fmap_json.update(entry) - with open(fmap, 'w') as f: - json.dump(fmap_json, f) - f.close() - else: - print(fmap + " already exists for this session") +setup(subject_path) \ No newline at end of file diff --git a/run.py b/run.py index 8c70934..82ef3a8 100644 --- a/run.py +++ b/run.py @@ -7,14 +7,12 @@ from glob import glob from subprocess import Popen, PIPE from shutil import rmtree +import pdb import subprocess from bids.grabbids import BIDSLayout from functools import partial from collections import OrderedDict -<<<<<<< HEAD from IntendedFor import setup,IntendedFor -======= ->>>>>>> ce2c6dbfe600164acbcb75a9fb83d54e440c15af def run(command, env={}, cwd=None): merged_env = os.environ @@ -223,12 +221,8 @@ def run_diffusion_processsing(**args): if args.analysis_level == "participant": # find all T1s and skullstrip them for subject_label in subjects_to_analyze: -<<<<<<< HEAD - # before do anything else add IntendedFor field to fieldmap setup(os.path.join(args.bids_dir, "sub-"+subject_label)) -======= ->>>>>>> ce2c6dbfe600164acbcb75a9fb83d54e440c15af t1ws = [f.filename for f in layout.get(subject=subject_label, type='T1w', extensions=["nii.gz", "nii"])] @@ -360,34 +354,32 @@ def run_diffusion_processsing(**args): if not os.path.exists(fmriscout): fmriscout = "NONE" -<<<<<<< HEAD fieldmap_set = layout.get_fieldmap(fmritcs, return_list=True) -======= - fieldmap_set = layout.get_fieldmap(fmritcs) ->>>>>>> ce2c6dbfe600164acbcb75a9fb83d54e440c15af - if fieldmap_set and fieldmap_set["type"] == "epi": - SEPhaseNeg = None - SEPhasePos = None - for fieldmap in fieldmap_set["epi"]: - enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] - if "-" in enc_dir: - SEPhaseNeg = fieldmap + if fieldmap_set: + for item in fieldmap_set: + + if item["type"] == "epi": + + fieldmap = item["epi"] + enc_dir = layout.get_metadata(fieldmap)["PhaseEncodingDirection"] + if "-" in enc_dir: + SEPhaseNeg = fieldmap + else: + SEPhasePos = fieldmap + echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] + unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] + unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") + if len(unwarpdir) == 2: + unwarpdir = "-" + unwarpdir[0] + dcmethod = "TOPUP" + biascorrection = "SEBASED" else: - SEPhasePos = fieldmap - echospacing = layout.get_metadata(fmritcs)["EffectiveEchoSpacing"] - unwarpdir = layout.get_metadata(fmritcs)["PhaseEncodingDirection"] - unwarpdir = unwarpdir.replace("i","x").replace("j", "y").replace("k", "z") - if len(unwarpdir) == 2: - unwarpdir = "-" + unwarpdir[0] - dcmethod = "TOPUP" - biascorrection = "SEBASED" - else: - SEPhaseNeg = "NONE" - SEPhasePos = "NONE" - echospacing = "NONE" - unwarpdir = "NONE" - dcmethod = "NONE" - biascorrection = "NONE" + SEPhaseNeg = "NONE" + SEPhasePos = "NONE" + echospacing = "NONE" + unwarpdir = "NONE" + dcmethod = "NONE" + biascorrection = "NONE" zooms = nibabel.load(fmritcs).get_header().get_zooms() fmrires = float(min(zooms[:3])) From 197e040f2991c865facf78a146836d6d69e2476e Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 12 Apr 2018 11:42:21 -0500 Subject: [PATCH 10/74] added /bin/bash -c to entrypoint to fix bit set problem with udocker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 89791f1..886121c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,4 +107,4 @@ COPY run.py /run.py RUN chmod +x /run.py COPY version /version -ENTRYPOINT ["/run.py"] +ENTRYPOINT ["/bin/bash -c /run.py"] From 35e86d38f736f5981964dc80bec496016d10f7d3 Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 12 Apr 2018 13:05:40 -0500 Subject: [PATCH 11/74] remove entrypoint command so udocker will run --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6466a05..7f61684 100644 --- a/Dockerfile +++ b/Dockerfile @@ -109,5 +109,5 @@ RUN chmod +x /run.py COPY version /version COPY IntendedFor.py /IntendedFor.py -ENTRYPOINT ["/bin/bash -c /run.py"] +#ENTRYPOINT ["/bin/bash -c /run.py"] From ae79bb65d4b547ddcf9221c41739607c4de0aa13 Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 12 Apr 2018 13:12:13 -0500 Subject: [PATCH 12/74] remove entrypoint command so udocker will run --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7f61684..253e9c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,8 @@ ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMBin=${HCPPIPEDIR}/MSMBinaries RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools -RUN pip install pybids==0.0.1 +RUN pip install pybids==0.5.1 +RUN pip install --upgrade pybids ENV PYTHONPATH="" COPY run.py /run.py From 3f0c1eaeefbe100d7ec85ea4572934ba7305148a Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 12 Apr 2018 15:17:05 -0500 Subject: [PATCH 13/74] added diffusion pieces from jokedurnez fork commit 2c44cc0f6b --- run.py | 65 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/run.py b/run.py index ac5612c..3721cd9 100644 --- a/run.py +++ b/run.py @@ -412,32 +412,47 @@ def run_diffusion_processsing(**args): for stage, stage_func in func_stages_dict.iteritems(): if stage in args.stages: stage_func() + pdb.set_trace() dwis = layout.get(subject=subject_label, type='dwi', extensions=["nii.gz", "nii"]) - # print(dwis) - # acqs = set(layout.get(target='acquisition', return_type='id', - # subject=subject_label, type='dwi', - # extensions=["nii.gz", "nii"])) - # print(acqs) - # posData = [] - # negData = [] - # for acq in acqs: - # pos = "EMPTY" - # neg = "EMPTY" - # dwis = layout.get(subject=subject_label, - # type='dwi', acquisition=acq, - # extensions=["nii.gz", "nii"]) - # assert len(dwis) <= 2 - # for dwi in dwis: - # dwi = dwi.filename - # if "-" in layout.get_metadata(dwi)["PhaseEncodingDirection"]: - # neg = dwi - # else: - # pos = dwi - # posData.append(pos) - # negData.append(neg) - # - # print(negData) - # print(posData) + pos = []; neg = [] + PEdir = None; echospacing = None + + for idx, dwi in enumerate(dwis): + metadata = layout.get_metadata(dwi.filename) + # get phaseencodingdirection + phaseenc = metadata['PhaseEncodingDirection'] + acq = 1 if phaseenc[0]=='i' else 2 + if not PEdir: + PEdir = acq + if PEdir != acq: + raise RuntimeError("Not all dwi images have the same encoding direction (both LR and AP). Not implemented.") + # get pos/neg + if len(phaseenc)>1: + neg.append(dwi.filename) + else: + pos.append(dwi.filename) + # get echospacing + if not echospacing: + echospacing = metadata['EffectiveEchoSpacing']*1000. + if echospacing != metadata['EffectiveEchoSpacing']*1000.: + raise RuntimeError("Not all dwi images have the same echo spacing. Not implemented.") + + posdata = "@".join(pos) + negdata = "@".join(neg) + + dif_stages_dict = OrderedDict([("DiffusionPreprocessing", partial(run_diffusion_processsing, + path=args.output_dir, + subject="sub-%s"%subject_label, + posData=posdata, + negData=negdata, + echospacing=echospacing, + n_cpus=args.n_cpus, + PEdir=PEdir)) + ]) + + for stage, stage_func in dif_stages_dict.iteritems(): + if stage in args.stages: + stage_func() From 6378f36b6d5161094700fcdc3b2718ece1f36c4b Mon Sep 17 00:00:00 2001 From: Timothy Date: Thu, 12 Apr 2018 17:43:02 -0500 Subject: [PATCH 14/74] Added function to run ICA-FIX processing stream --- run.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/run.py b/run.py index 3721cd9..27a6c3f 100644 --- a/run.py +++ b/run.py @@ -169,6 +169,17 @@ def run_diffusion_processsing(**args): cmd = cmd.format(**args) run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) +def run_ICAFIX_processing(**args): + args.update(os.environ) + cmd = '{HCPPIPEDIR}/Examples/Scripts/IcaFixProcessingBatch.sh' + \ + '--StudyFolder="{path} ' + \ + '--Subject="{subject}" ' + \ + '--EnvironmentScript="{EnvironmentScript}"' + \ + '--FixDir="{FixDir}" ' + \ + '--RunLocal="{RunLocal}"' + cmd = cmd.format(**args) + run(cmd, cwd=args["path"], env={"OMP_NUM_THREADS": str(args["n_cpus"])}) + __version__ = open('/version').read() parser = argparse.ArgumentParser(description='HCP Pipeliens BIDS App (T1w, T2w, fMRI)') From dc33ab0bf8cfd34a6cc77533fba388319dd230be Mon Sep 17 00:00:00 2001 From: Timothy Hendrickson Date: Thu, 12 Apr 2018 21:48:43 -0500 Subject: [PATCH 15/74] install FIX along with dependencies to Dockerfile --- .idea/vcs.xml | 6 ++++++ Dockerfile | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 96dec14..c7e9ed9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ RUN apt-get update && apt-get -y install connectome-workbench=1.2.3-1~nd14.04+1 ENV CARET7DIR=/usr/bin # Install HCP Pipelines -RUN apt-get -y update \ +RUN apt-get update \ && apt-get install -y --no-install-recommends python-numpy && \ wget https://github.com/Washington-University/Pipelines/archive/v3.17.0.tar.gz -O pipelines.tar.gz && \ cd /opt/ && \ @@ -99,6 +99,18 @@ ENV HCPPIPEDIR_Global=${HCPPIPEDIR}/global/scripts ENV HCPPIPEDIR_tfMRIAnalysis=${HCPPIPEDIR}/TaskfMRIAnalysis/scripts ENV MSMBin=${HCPPIPEDIR}/MSMBinaries +# Install FIX, along with dependencies +RUN apt-get update && \ + cd /opt/ && \ + wget http://www.fmrib.ox.ac.uk/~steve/ftp/fix.tar.gz && \ + tar zxvf fix.tar.gz && \ + rm fix.tar.gz && \ + wget https://cran.r-project.org/src/base/R-3/R-3.4.4.tar.gz && \ + tar zxvf R-3.4.4.tar.gz && \ + cd R-3.4.4 && \ + ./configure && make && make check && make install + + RUN apt-get update && apt-get install -y --no-install-recommends python-pip python-six python-nibabel python-setuptools RUN pip install pybids==0.5.1 RUN pip install --upgrade pybids From 5fbf3232dcb0917e4840d65b5064b56b36cb6574 Mon Sep 17 00:00:00 2001 From: Timothy Date: Fri, 13 Apr 2018 17:50:18 -0500 Subject: [PATCH 16/74] added FIX along with dependencies such as R to dockerfile in order to run ICA-FIX pipeline --- .idea/HCPPipelines.iml | 14 +++ .idea/libraries/R_User_Library.xml | 6 ++ .idea/misc.xml | 4 + .idea/modules.xml | 8 ++ .idea/workspace.xml | 164 +++++++++++++++++++++++++++++ Dockerfile | 9 +- 6 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 .idea/HCPPipelines.iml create mode 100644 .idea/libraries/R_User_Library.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/HCPPipelines.iml b/.idea/HCPPipelines.iml new file mode 100644 index 0000000..8bdd4a0 --- /dev/null +++ b/.idea/HCPPipelines.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/R_User_Library.xml b/.idea/libraries/R_User_Library.xml new file mode 100644 index 0000000..71f5ff7 --- /dev/null +++ b/.idea/libraries/R_User_Library.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..7ba73c2 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..d659a47 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..3f1b2c2 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,164 @@ + + + + + + + + + + + + + + + + + + + + + + + + fsl/5.0 + $HOME + + + /opt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +