Skip to content

Commit

Permalink
Merge pull request scilus#139 from arnaudbore/migrate_topup_eddy
Browse files Browse the repository at this point in the history
Migrate Topup and Eddy modules to nf-test
  • Loading branch information
AlexVCaron authored Apr 29, 2024
2 parents 56a79d9 + db30b17 commit 9a8a46e
Show file tree
Hide file tree
Showing 23 changed files with 283 additions and 164 deletions.
7 changes: 7 additions & 0 deletions modules/nf-scil/preproc/eddy/environment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
name: "preproc_eddy"
channels:
- Docker
- Apptainer
dependencies:
- "FSL"
- "scilpy"
45 changes: 22 additions & 23 deletions modules/nf-scil/preproc/eddy/main.nf
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@


process PREPROC_EDDY {
tag "$meta.id"
label 'process_single'

container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'https://scil.usherbrooke.ca/containers/scilus_1.6.0.sif':
'scilus/scilus:1.6.0' }"
"https://scil.usherbrooke.ca/containers/scilus_2.0.0.sif":
"scilus/scilus:2.0.0"}"

input:
tuple val(meta), path(dwi), path(bval), path(bvec), path(rev_dwi), path(rev_bval), path(rev_bvec), path(corrected_b0s), path(topup_fieldcoef), path(topup_movpart)
Expand Down Expand Up @@ -34,6 +32,7 @@ process PREPROC_EDDY {
def dilate_b0_mask_prelim_brain_extraction = task.ext.dilate_b0_mask_prelim_brain_extraction ? task.ext.dilate_b0_mask_prelim_brain_extraction : ""
def eddy_cmd = task.ext.eddy_cmd ? task.ext.eddy_cmd : "eddy_cpu"
def bet_prelim_f = task.ext.bet_prelim_f ? task.ext.bet_prelim_f : ""
def extra_args = task.ext.extra_args ?: ""

"""
export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1
Expand All @@ -46,11 +45,11 @@ process PREPROC_EDDY {
number_rev_dwi=0
if [[ -f "$rev_dwi" ]];
then
scil_concatenate_dwi.py ${prefix}__concatenated_dwi.nii.gz ${prefix}__concatenated_dwi.bval ${prefix}__concatenated_dwi.bvec -f\
scil_dwi_concatenate.py ${prefix}__concatenated_dwi.nii.gz ${prefix}__concatenated_dwi.bval ${prefix}__concatenated_dwi.bvec -f\
--in_dwis ${dwi} ${rev_dwi} --in_bvals ${bval} ${rev_bval}\
--in_bvecs ${bvec} ${rev_bvec}
number_rev_dwi=\$(scil_print_header.py ${rev_dwi} --key dim | sed "s/ / /g" | sed "s/ / /g" | rev | cut -d' ' -f4-4 | rev)
number_rev_dwi=\$(scil_header_print_info.py ${rev_dwi} --key dim | sed "s/ / /g" | sed "s/ / /g" | rev | cut -d' ' -f4-4 | rev)
dwi=${prefix}__concatenated_dwi.nii.gz
bval=${prefix}__concatenated_dwi.bval
Expand All @@ -68,7 +67,7 @@ process PREPROC_EDDY {
bet b0_corrected.nii.gz ${prefix}__b0_bet.nii.gz -m -R\
-f $bet_topup_before_eddy_f
scil_prepare_eddy_command.py \${dwi} \${bval} \${bvec} ${prefix}__b0_bet_mask.nii.gz\
scil_dwi_prepare_eddy_command.py \${dwi} \${bval} \${bvec} ${prefix}__b0_bet_mask.nii.gz\
--topup $prefix_topup --eddy_cmd $eddy_cmd\
--b0_thr $b0_thr_extract_b0\
--encoding_direction $encoding\
Expand All @@ -77,38 +76,38 @@ process PREPROC_EDDY {
--lsr_resampling\
$slice_drop_flag
else
scil_extract_b0.py \${dwi} \${bval} \${bvec} ${prefix}__b0.nii.gz --mean\
--b0_thr $b0_thr_extract_b0 --force_b0_threshold
scil_dwi_extract_b0.py \${dwi} \${bval} \${bvec} ${prefix}__b0.nii.gz --mean\
--b0_threshold $b0_thr_extract_b0 --skip_b0_check
bet ${prefix}__b0.nii.gz ${prefix}__b0_bet.nii.gz -m -R -f $bet_prelim_f
scil_image_math.py convert ${prefix}__b0_bet_mask.nii.gz ${prefix}__b0_bet_mask.nii.gz --data_type uint8 -f
scil_volume_math.py convert ${prefix}__b0_bet_mask.nii.gz ${prefix}__b0_bet_mask.nii.gz --data_type uint8 -f
maskfilter ${prefix}__b0_bet_mask.nii.gz dilate ${prefix}__b0_bet_mask_dilated.nii.gz\
--npass $dilate_b0_mask_prelim_brain_extraction -nthreads 1
scil_image_math.py multiplication ${prefix}__b0.nii.gz ${prefix}__b0_bet_mask_dilated.nii.gz\
scil_volume_math.py multiplication ${prefix}__b0.nii.gz ${prefix}__b0_bet_mask_dilated.nii.gz\
${prefix}__b0_bet.nii.gz --data_type float32 -f
scil_prepare_eddy_command.py \${dwi} \${bval} \${bvec} ${prefix}__b0_bet_mask.nii.gz\
scil_dwi_prepare_eddy_command.py \${dwi} \${bval} \${bvec} ${prefix}__b0_bet_mask.nii.gz\
--eddy_cmd $eddy_cmd --b0_thr $b0_thr_extract_b0\
--encoding_direction $encoding\
--readout $readout --out_script --fix_seed\
$slice_drop_flag
fi
echo "--very_verbose" >> eddy.sh
echo "--very_verbose $extra_args" >> eddy.sh
sh eddy.sh
scil_image_math.py lower_threshold dwi_eddy_corrected.nii.gz 0 ${prefix}__dwi_corrected.nii.gz
scil_volume_math.py lower_threshold dwi_eddy_corrected.nii.gz 0 ${prefix}__dwi_corrected.nii.gz
if [[ \$number_rev_dwi -eq 0 ]]
then
mv dwi_eddy_corrected.eddy_rotated_bvecs ${prefix}__dwi_eddy_corrected.bvec
mv \${orig_bval} ${prefix}__bval_eddy
else
scil_validate_and_correct_eddy_gradients.py dwi_eddy_corrected.eddy_rotated_bvecs \${bval} \${number_rev_dwi} ${prefix}__dwi_eddy_corrected.bvec ${prefix}__bval_eddy
scil_gradients_validate_correct_eddy.py dwi_eddy_corrected.eddy_rotated_bvecs \${bval} \${number_rev_dwi} ${prefix}__dwi_eddy_corrected.bvec ${prefix}__bval_eddy
fi
cat <<-END_VERSIONS > versions.yml
"${task.process}":
scilpy: 1.6.0
scilpy: 2.0.0
mrtrix: \$(dwidenoise -version 2>&1 | sed -n 's/== dwidenoise \\([0-9.]\\+\\).*/\\1/p')
fsl: \$(flirt -version 2>&1 | sed -n 's/FLIRT version \\([0-9.]\\+\\)/\\1/p')
END_VERSIONS
Expand All @@ -119,15 +118,15 @@ process PREPROC_EDDY {
def prefix = task.ext.prefix ?: "${meta.id}"

"""
scil_image_math.py -h
scil_volume_math.py -h
maskfilter -h
bet -h
scil_extract_b0.py -h
scil_validate_and_correct_eddy_gradients.py -h
scil_concatenate_dwi.py -h
scil_dwi_extract_b0.py -h
scil_gradients_validate_correct_eddy.py -h
scil_dwi_concatenate.py -h
mrconvert -h
scil_prepare_eddy_command.py -h
scil_print_header.py -h
scil_dwi_prepare_eddy_command.py -h
scil_header_print_info.py -h
touch ${prefix}__dwi_corrected.nii.gz
touch ${prefix}__bval_eddy
Expand All @@ -137,7 +136,7 @@ process PREPROC_EDDY {
cat <<-END_VERSIONS > versions.yml
"${task.process}":
scilpy: 1.6.0
scilpy: 2.0.0
mrtrix: \$(dwidenoise -version 2>&1 | sed -n 's/== dwidenoise \\([0-9.]\\+\\).*/\\1/p')
fsl: \$(flirt -version 2>&1 | sed -n 's/FLIRT version \\([0-9.]\\+\\)/\\1/p')
Expand Down
61 changes: 61 additions & 0 deletions modules/nf-scil/preproc/eddy/tests/main.nf.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
nextflow_process {

name "Test Process PREPROC_EDDY"
script "../main.nf"
process "PREPROC_EDDY"

tag "modules"
tag "modules_nfcore"
tag "preproc"
tag "preproc/eddy"

tag "subworkflows"
tag "subworkflows/load_test_data"

config "./nextflow.config"

setup {
run("LOAD_TEST_DATA", alias: "LOAD_DATA") {
script "../../../../../subworkflows/nf-scil/load_test_data/main.nf"
process {
"""
input[0] = Channel.from( [ "topup_eddy_light.zip" ] )
input[1] = "test.load-test-data"
"""
}
}
}

test("eddy") {
when {
process {
"""
input[0] = LOAD_DATA.out.test_data_directory
.map{ test_data_directory -> [
[ id:'test', single_end:false ], // meta map
file("\${test_data_directory}/sub-01_dir-AP_dwi.nii.gz", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-AP_dwi.bval", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-AP_dwi.bvec", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_dwi.nii.gz", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_dwi.bval", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_dwi.bvec", checkIfExists: true),
file("\${test_data_directory}/sub-01__corrected_b0s.nii.gz", checkIfExists: true),
file("\${test_data_directory}/topup_results_fieldcoef.nii.gz", checkIfExists: true),
file("\${test_data_directory}/topup_results_movpar.txt", checkIfExists: true)]}
"""
}
}

then {
assertAll(
{ assert process.success },
{ assert snapshot(
file(process.out.dwi_corrected.get(0).get(1)).name,
process.out.bval_corrected,
file(process.out.bvec_corrected.get(0).get(1)).name,
process.out.b0_mask,
).match() }
)
}
}
}
31 changes: 31 additions & 0 deletions modules/nf-scil/preproc/eddy/tests/main.nf.test.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"eddy": {
"content": [
"test__dwi_corrected.nii.gz",
[
[
{
"id": "test",
"single_end": false
},
"test__bval_eddy:md5,4c61c53078316c31b4d5daf446a3d6ac"
]
],
"test__dwi_eddy_corrected.bvec",
[
[
{
"id": "test",
"single_end": false
},
"test__b0_bet_mask.nii.gz:md5,fc1f94e2a5e2cf5197d7fdc28a83ca28"
]
]
],
"meta": {
"nf-test": "0.8.4",
"nextflow": "23.10.1"
},
"timestamp": "2024-04-29T16:11:13.561798"
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
process {
withName: "PREPROC_EDDY" {
publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
ext.prefix_topup = "topup_results"
ext.default_config_topup = "b02b0.cnf"
ext.encoding = "y"
Expand All @@ -11,5 +10,6 @@ process {
ext.eddy_cmd="eddy_cpu"
ext.dilate_b0_mask_prelim_brain_extraction = 5
ext.bet_prelim_f = 0.16
ext.extra_args = "--flm=linear --niter=2"
}
}
2 changes: 2 additions & 0 deletions modules/nf-scil/preproc/eddy/tests/tags.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
preproc/eddy:
- "modules/nf-scil/preproc/eddy/**"
7 changes: 7 additions & 0 deletions modules/nf-scil/preproc/topup/environment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
name: "preproc_topup"
channels:
- Docker
- Apptainer
dependencies:
- "FSL"
- "scilpy"
29 changes: 14 additions & 15 deletions modules/nf-scil/preproc/topup/main.nf
100755 → 100644
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@

process PREPROC_TOPUP {
tag "$meta.id"
label 'process_single'

container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'https://scil.usherbrooke.ca/containers/scilus_1.6.0.sif':
'scilus/scilus:1.6.0' }"
"https://scil.usherbrooke.ca/containers/scilus_2.0.0.sif":
"scilus/scilus:2.0.0"}"

input:
tuple val(meta), path(dwi), path(bval), path(bvec), path(b0), path(rev_dwi), path(rev_bval), path(rev_bvec), path(rev_b0)
Expand Down Expand Up @@ -40,23 +39,23 @@ process PREPROC_TOPUP {
if [[ -f "$b0" ]];
then
scil_image_math.py concatenate $b0 $b0 ${prefix}__concatenated_b0.nii.gz
scil_image_math.py mean ${prefix}__concatenated_b0.nii.gz ${prefix}__b0_mean.nii.gz
scil_volume_math.py concatenate $b0 $b0 ${prefix}__concatenated_b0.nii.gz
scil_volume_math.py mean ${prefix}__concatenated_b0.nii.gz ${prefix}__b0_mean.nii.gz
else
scil_extract_b0.py $dwi $bval $bvec ${prefix}__b0_mean.nii.gz --mean --b0_thr $b0_thr_extract_b0 --force_b0_threshold
scil_dwi_extract_b0.py $dwi $bval $bvec ${prefix}__b0_mean.nii.gz --mean ----b0_threshold $b0_thr_extract_b0 --skip_b0_check
fi
if [[ -f "$rev_b0" ]];
then
scil_image_math.py concatenate $rev_b0 $rev_b0 ${prefix}__concatenated_rev_b0.nii.gz
scil_image_math.py mean ${prefix}__concatenated_rev_b0.nii.gz ${prefix}__rev_b0_mean.nii.gz
scil_volume_math.py concatenate $rev_b0 $rev_b0 ${prefix}__concatenated_rev_b0.nii.gz
scil_volume_math.py mean ${prefix}__concatenated_rev_b0.nii.gz ${prefix}__rev_b0_mean.nii.gz
else
scil_extract_b0.py $rev_dwi $rev_bval $rev_bvec ${prefix}__rev_b0_mean.nii.gz --mean --b0_thr $b0_thr_extract_b0 --force_b0_threshold
scil_dwi_extract_b0.py $rev_dwi $rev_bval $rev_bvec ${prefix}__rev_b0_mean.nii.gz --mean ----b0_threshold $b0_thr_extract_b0 --skip_b0_check
fi
antsRegistrationSyNQuick.sh -d 3 -f ${prefix}__b0_mean.nii.gz -m ${prefix}__rev_b0_mean.nii.gz -o output -t r -e 1
mv outputWarped.nii.gz ${prefix}__rev_b0_warped.nii.gz
scil_prepare_topup_command.py ${prefix}__b0_mean.nii.gz ${prefix}__rev_b0_warped.nii.gz\
scil_dwi_prepare_topup_command.py ${prefix}__b0_mean.nii.gz ${prefix}__rev_b0_warped.nii.gz\
--config $config_topup\
--encoding_direction $encoding\
--readout $readout --out_prefix $prefix_topup\
Expand All @@ -66,7 +65,7 @@ process PREPROC_TOPUP {
cat <<-END_VERSIONS > versions.yml
"${task.process}":
scilpy: 1.6.0
scilpy: 2.0.0
antsRegistration: 2.4.3
fsl: \$(flirt -version 2>&1 | sed -n 's/FLIRT version \\([0-9.]\\+\\)/\\1/p')
Expand All @@ -78,10 +77,10 @@ process PREPROC_TOPUP {
def prefix = task.ext.prefix ?: "${meta.id}"

"""
scil_image_math.py -h
scil_extract_b0.py -h
scil_volume_math.py -h
scil_dwi_extract_b0.py -h
antsRegistrationSyNQuick.sh -h
scil_prepare_topup_command.py -h
scil_dwi_prepare_topup_command.py -h
touch ${prefix}__corrected_b0s.nii.gz
touch ${prefix}__rev_b0_warped.nii.gz
Expand All @@ -92,7 +91,7 @@ process PREPROC_TOPUP {
cat <<-END_VERSIONS > versions.yml
"${task.process}":
scilpy: 1.6.0
scilpy: 2.0.0
antsRegistration: 2.4.3
fsl: \$(flirt -version 2>&1 | sed -n 's/FLIRT version \\([0-9.]\\+\\)/\\1/p')
Expand Down
62 changes: 62 additions & 0 deletions modules/nf-scil/preproc/topup/tests/main.nf.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
nextflow_process {

name "Test Process PREPROC_TOPUP"
script "../main.nf"
process "PREPROC_TOPUP"

tag "modules"
tag "modules_nfcore"
tag "preproc"
tag "preproc/topup"

tag "subworkflows"
tag "subworkflows/load_test_data"

config "./nextflow.config"

setup {
run("LOAD_TEST_DATA", alias: "LOAD_DATA") {
script "../../../../../subworkflows/nf-scil/load_test_data/main.nf"
process {
"""
input[0] = Channel.from( [ "topup_eddy_light.zip" ] )
input[1] = "test.load-test-data"
"""
}
}
}

test("topup") {
when {
process {
"""
input[0] = LOAD_DATA.out.test_data_directory
.map{ test_data_directory -> [
[ id:'test', single_end:false ], // meta map
file("\${test_data_directory}/sub-01_dir-AP_dwi.nii.gz", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-AP_dwi.bval", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-AP_dwi.bvec", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-AP_sbref.nii.gz", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_dwi.nii.gz", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_dwi.bval", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_dwi.bvec", checkIfExists: true),
file("\${test_data_directory}/sub-01_dir-PA_sbref.nii.gz", checkIfExists: true)]}
input[1] = []
"""
}
}

then {
assertAll(
{ assert process.success },
{ assert snapshot(
file(process.out.topup_corrected_b0s.get(0).get(1)).name,
file(process.out.topup_fieldcoef.get(0).get(1)).name,
file(process.out.topup_movpart.get(0).get(1)).name,
file(process.out.rev_b0_warped.get(0).get(1)).name,
process.out.rev_b0_mean,
process.out.b0_mean).match() }
)
}
}
}
Loading

0 comments on commit 9a8a46e

Please sign in to comment.