Skip to content

Commit

Permalink
Merge pull request #11 from mrfil/jq-jo-devel
Browse files Browse the repository at this point in the history
JSON-based documentation
  • Loading branch information
pcamach2 authored Apr 1, 2022
2 parents b3dba14 + 469c0bd commit 57e476c
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 177 deletions.
3 changes: 1 addition & 2 deletions docs/source/Install.rst
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,8 @@ This following commands can be used to build these required images for the pipel
# See README.md for more information on
# provided def files for ubuntu-jq, python3
SINGULARITY_NOHTTPS=1 singularity build ubuntu-jq-0.1.sif defjq
SINGULARITY_NOHTTPS=1 singularity build ubuntu-jqjo.sif jqjo.def
SINGULARITY_NOHTTPS=1 singularity build python3.sif defpy3
SINGULARITY_NOHTTPS=1 singularity build laynii-2.0.0.sif layniidef
SINGULARITY_NOHTTPS=1 singularity build ashs-1.0.0.sif ashsdef
#Start Docker registry for localhost
Expand Down
50 changes: 10 additions & 40 deletions jsoncrawler.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
#
#single-sub single-session script for adding intended for statements to fmap jsons before running BIDS apps
#
# requires JQ
#usage: jsoncrawler.sh {bids directory} {session} {subject}
#
#pbc
Expand All @@ -10,7 +10,6 @@
sesname=$2
seslen=${#sesname}
sub=$3
intend='"IntendedFor": '

seshs=($1/${sub}/${sesname})
for sesh in ${seshs[@]}; do
Expand All @@ -20,21 +19,15 @@ restfuncs=(./func/*rest*bold*nii.gz)

for restfuncmap in ${restfuncmaps[@]}; do
img=${restfuncs:1}
jq .IntendedFor $restfuncmap > tmpj
jq '.IntendedFor' $restfuncmap > tmpj
if [[ $(cat tmpj) == "${sesname}${img}" ]];
then
echo "IntendedFor found as "${sesname}$img""
exit 0
else
printf " " > test.txt
printf $intend >> test.txt
printf " " >> test.txt
printf '"' >> test.txt
printf "${sesname}""$img" >> test.txt
printf '",' >> test.txt
echo "" >> test.txt
echo "" > tmp
awk 'NR==1{a=$0}NR==FNR{next}FNR==32{print a}1' test.txt $restfuncmap >> tmp && mv tmp $restfuncmap
# replace the following with jq or jo
cat $restfuncmap | jq '.IntendedFor |= "${sesname}$img"' > tmp
mv tmp $restfuncmap
fi
done

Expand All @@ -48,22 +41,8 @@ taskfuncs=`find ./func -maxdepth 2 -type f \( -iname "*nback*bold*nii.gz" -a -no
task="*nback*"
if [[ "$img" == "$task" ]];
then
# jq .IntendedFor $taskfuncmap > tmpj
# if [[ "`cat tmpj`" == "${sesname}$img" ]];
# then
# echo "IntendedFor found as "${sesname}$img""
# exit 0
# else
printf " " > test.txt
printf $intend >> test.txt
printf " " >> test.txt
printf '"' >> test.txt
printf "${sesname}""$img" >> test.txt
printf '",' >> test.txt
echo "" >> test.txt
echo "" > tmp
awk 'NR==1{a=$0}NR==FNR{next}FNR==32{print a}1' test.txt $taskfuncmap >> tmp && mv tmp $taskfuncmap

cat $taskfuncmap | jq '.IntendedFor |= "${sesname}$img"' > tmp
mv tmp $taskfuncmap
fi
done

Expand All @@ -72,25 +51,16 @@ dwis=(./dwi/*dwi*nii.gz)

for dwimap in ${dwimaps[@]}; do
img=${dwis:1}
jq .IntendedFor $dwimap > tmpj
jq '.IntendedFor' $dwimap > tmpj
if [[ $(cat tmpj) == "${sesname}$img" ]];
then
echo "IntendedFor found as "${sesname}$img""
exit 0
else
printf " " > test.txt
printf $intend >> test.txt
printf " " >> test.txt
printf '"' >> test.txt
printf "${sesname}""$img" >> test.txt
printf '",' >> test.txt
echo "" >> test.txt
echo "" > tmp
awk 'NR==1{a=$0}NR==FNR{next}FNR==32{print a}1' test.txt $dwimap >> tmp && mv tmp $dwimap
cat $dwimap | jq '.IntendedFor |= "${sesname}$img"' > tmp
mv tmp $dwimap
fi
done
done

rm -f test.txt
rm -f tmp
rm -f tmpj
101 changes: 26 additions & 75 deletions project_doc.sh
Original file line number Diff line number Diff line change
@@ -1,112 +1,63 @@
#!/bin/bash
#
#script for generating a project xml file
#script for generating a project json file
#
# usage project_doc.sh {project} {subject} {session} {process called (heudiconv, mriqc, fmriprep, xcpengine, mridti, scfsl)} {is this a new file? (yes/no)}
# usage project_doc.sh {project} {subject} {session} {process called (heudiconv, mriqc, fmriprep, xcpengine, mridti, scfsl)} {is this a new file? (yes/no)} {app version}
#
# to be called as part of a pipeline for documentation of processing tool calls
#
# maybe set up to run on batches with reading in the txt files from batchprocd.sh to the project xml

project=$1
subject=$2
session=$3
proc=$4
newFile=$5
version=$6

#pull in versions from sif names var in script?

#Get date and time
NOW=$(date +"%m-%d-%Y-%T")

xmlfile="${project}_${subject}_${session}.xml"
jsonfile="${project}_${subject}_${session}.json"

if [ "${newFile}" == "yes" ];
then
echo '<?xml version="1.0" encoding="utf-8"?>' > "$xmlfile"
echo '<proc_log id="${project}">' >> "$xmlfile"

jo log="pipeline processing" project="${project}" > $jsonfile
fi

#if statements for each process (heudiconv, mriqc, fmriprep, xcpengine, mridti, scfsl)
if [ "${proc}" == "heudiconv" ];
then
echo '<proc id="HeuDiConv">' >> "$xmlfile"
echo '<source>nipy/heudiconv</source>' >> "$xmlfile"
echo '<title>HeuDiConv</title>' >> "$xmlfile"
echo '<version>0.9.0</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description> Heuristic conversion from DICOMs to BIDS-compatible nii.gz images with JSON sidecars.</description>' >> "$xmlfile"
echo '<conversion_tool>Chris Rordens dcm2niix</conversion_tool>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
cat $jsonfile | jq '.step1["process"] |= "HeuDiConv"' | jq '.step1["Docker"] |= "nipy/heudiconv:'${version}'"' | jq '.step1["date"] |= '"${NOW}"'' | jq '.step1["description"] |= "Heuristic-based conversion from DICOMs to BIDS-compatible nii.gz images with JSON sidecars with dcm2niix"' > tmp
mv tmp $jsonfile
elif [ "${proc}" == "mriqc" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="MRIQC">' >> "$xmlfile"
echo '<source>poldracklab/mriqc</source>' >> "$xmlfile"
echo '<title>MRIQC</title>' >> "$xmlfile"
echo '<version>0.16.1</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description> MRIQC extracts no-reference IQMs (image quality metrics) from structural (T1w and T2w) and functional MRI (magnetic resonance imaging) data </description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
cat $jsonfile | jq '.step2["process"] |= "MRIQC"' | jq '.step2["Docker"] |= "poldracklab/mriqc:'${version}'"' | jq '.step2["date"] |= '"${NOW}"'' | jq '.step2["description"] |= "MRIQC extracts no-reference IQMs (image quality metrics) from structural (T1w and T2w) and functional MRI (magnetic resonance imaging) data"' > tmp
mv tmp "$jsonfile"
elif [ "${proc}" == "fmriprep" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="fMRIPrep">' >> "$xmlfile"
echo '<source>niprep/fmriprep</source>' >> "$xmlfile"
echo '<title>fMRIPrep</title>' >> "$xmlfile"
echo '<version>21.0.1</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description>fMRIPrep is a functional magnetic resonance imaging (fMRI) data preprocessing pipeline that is designed to provide an easily accessible, state-of-the-art interface that is robust to variations in scan acquisition protocols and that requires minimal user input, while providing easily interpretable and comprehensive error and output reporting. It performs basic processing steps (coregistration, normalization, unwarping, noise component extraction, segmentation, skullstripping etc.) providing outputs that can be easily submitted to a variety of group level analyses, including task-based or resting-state fMRI, graph theory measures, surface or volume-based statistics, etc.</description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
cat $jsonfile | jq '.step3["process"] |= "fMRIPrep"' | jq '.step3["Docker"] |= "nipreps/fmriprep:'${version}'"' | jq '.step3["date"] |= '"${NOW}"'' | jq '.step3["description"] |= "fMRIPrep is a functional magnetic resonance imaging (fMRI) data preprocessing pipeline that is designed to provide an easily accessible, state-of-the-art interface that is robust to variations in scan acquisition protocols and that requires minimal user input, while providing easily interpretable and comprehensive error and output reporting. It performs basic processing steps (coregistration, normalization, unwarping, noise component extraction, segmentation, skullstripping etc.) providing outputs that can be easily submitted to a variety of group level analyses, including task-based or resting-state fMRI, graph theory measures, surface or volume-based statistics, etc."' > tmp
mv tmp "$jsonfile"
elif [ "${proc}" == "xcpengine" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="xcpEngine">' >> "$xmlfile"
echo '<source>PennBBL/xcpEngine</source>' >> "$xmlfile"
echo '<title>xcpEngine fc-36p, fc-36p_scrub, fc-aroma</title>' >> "$xmlfile"
echo '<version>1.2.4</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description> The XCP imaging pipeline (XCP system) is a free, open-source software package for processing of multimodal neuroimages. The XCP system uses a modular design to deploy analytic routines from leading MRI analysis platforms, including FSL, AFNI, and ANTs.</description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
cat $jsonfile | jq '.step4["process"] |= "xcpEngine"' | jq '.step4["Docker"] |= "pennbbl/xcpengine:'${version}'"' | jq '.step4["date"] |= '"${NOW}"'' | jq '.step4["description"] |= "The XCP imaging pipeline (XCP system) is a free, open-source software package for processing of multimodal neuroimages. The XCP system uses a modular design to deploy analytic routines from leading MRI analysis platforms, including FSL, AFNI, and ANTs"' > tmp
mv tmp "$jsonfile"
elif [ "${proc}" == "mridti" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="mridti">' >> "$xmlfile"
echo '<source>mrfil/mridti</source>' >> "$xmlfile"
echo '<title>MRI DTI with FSL</title>' >> "$xmlfile"
echo '<version>1.0.0</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description>Diffusion tensor processing pipeline with FSL bedpostx, probtrackx2.0</description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
elif [ "${proc}" == "scfsl" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="scfsl">' >> "$xmlfile"
echo '<source>mrfil/scfsl</source>' >> "$xmlfile"
echo '<title>Structural Connectivity with FSL </title>' >> "$xmlfile"
echo '<version>1.0.0</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description> FSL-based structural connectivity analysis with fibre-tracking between 68 cortical regions, 14 subcortical regions, and the left and right cerebellar cortices defined by the Freesurfer recon-all parcellation using the Desikan-Killiany Atlas</description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
cat $jsonfile | jq '.step5["process"] |= "MRIDTI FSL"' | jq '.step5["Docker"] |= "mrfilbi/neurodoc:'${version}'"' | jq '.step5["date"] |= '"${NOW}"'' | jq '.step5["description"] |= "Diffusion tensor processing pipeline with FSL "' > tmp
mv tmp "$jsonfile"
elif [ "${proc}" == "qsiprep" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="qsiprep">' >> "$xmlfile"
echo '<source>pennbbl/qsiprep</source>' >> "$xmlfile"
echo '<title>Diffusion Preprocessing with QSIprep</title>' >> "$xmlfile"
echo '<version>0.15.1</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description>The preprocessing pipelines are built based on the available BIDS inputs, ensuring that fieldmaps are handled correctly. The preprocessing workflow performs head motion correction, susceptibility distortion correction, MP-PCA denoising, coregistration to T1w images, spatial normalization using ANTs_ and tissue segmentation.</description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
#QSIPrep is preferred over mridti due to more complete preprocessing
cat $jsonfile | jq '.step5["process"] |= "QSIPrep preprocessing"' | jq '.step5["Docker"] |= "pennbbl/qsiprep:'${version}'"' | jq '.step5["date"] |= '"${NOW}"'' | jq '.step5["description"] |= "The preprocessing pipelines are built based on the available BIDS inputs, ensuring that fieldmaps are handled correctly. The preprocessing workflow performs head motion correction, susceptibility distortion correction, MP-PCA denoising, coregistration to T1w images, spatial normalization using ANTs_ and tissue segmentation. This requires step 6 to include the reorient_fslstd reconstruction method to use outputs in FSL space!"' > tmp
mv tmp "$jsonfile"
elif [ "${proc}" == "qsirecon" ];
then
head -n -1 "$xmlfile" > temp.txt ; mv temp.txt "$xmlfile"
echo '<proc id="qsirecon">' >> "$xmlfile"
echo '<source>pennbbl/qsiprep</source>' >> "$xmlfile"
echo '<title>Tractography with Anatomical Constrains using QSIprep MRtrix implementation </title>' >> "$xmlfile"
echo '<version>0.15.1</version>' >> "$xmlfile"
echo "<exec_date>${NOW}</exec_date>" >> "$xmlfile"
echo '<description> This workflow uses the msmt_csd algorithm [Jeurissen2014] to estimate FODs for white matter, gray matter and cerebrospinal fluid using multi-shell acquisitions. The white matter FODs are used for tractography and the T1w segmentation is used for anatomical constraints [Smith2012]. </description>' >> "$xmlfile"
echo '</proc>' >> "$xmlfile"
cat $jsonfile | jq '.step6["process"] |= "QSIPrep reconstruction"' | jq '.step6["Docker"] |= "pennbbl/qsiprep:'${version}'"' | jq '.step6["date"] |= '"${NOW}"'' | jq '.step6["description"] |= "QSIPrep reconstructions performed include: mrtrix_multishell_msmt_ACT-hsvs, dsi_studio_gqi, amico_noddi, and reorient_fslstd. More details at https://qsiprep.readthedocs.io/en/latest/reconstruction.html#"' > tmp
mv tmp "$jsonfile"
elif [ "${proc}" == "scfsl" ];
then
cat $jsonfile | jq '.step7["process"] |= "SCFSL GPU"' | jq '.step7["Docker"] |= "mrfilbi/scfsl_gpu:'${version}'"' | jq '.step7["date"] |= '"${NOW}"'' | jq '.step7["description"] |= "CUDA-accelerated FSL-based structural connectivity analysis with fibre-tracking between 68 cortical regions, 14 subcortical regions, and the left and right cerebellar cortices defined by the Freesurfer recon-all parcellation using the Desikan-Killiany Atlas"' > tmp
mv tmp "$jsonfile"
fi
echo "</proc_log>" >> "$xmlfile"

Loading

0 comments on commit 57e476c

Please sign in to comment.