Merge pull request scilus#73 from scilus/fix/singularity_runs

Fixes for workflows on singularity
Manonedde · Feb 19, 2024 · 0415b04 · 0415b04
2 parents f32210e + e03e165
commit 0415b04
Show file tree

Hide file tree

Showing 8 changed files with 38 additions and 33 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,6 +1,9 @@
 FROM scilus/scilus:1.6.0
 
+ENV NODE_MAJOR=18
+
 RUN apt update && apt install -y \
+        curl \
         git \
         python3-venv \
         wget \
@@ -9,3 +12,6 @@ RUN apt update && apt install -y \
 RUN python3 -m pip install pipx && \
     python3 -m pipx ensurepath && \
     pipx install poetry
+
+RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - &&\
+    apt-get install -y nodejs
diff --git a/.devcontainer/setup_container.sh b/.devcontainer/setup_container.sh
@@ -1,13 +1,8 @@
 #!/usr/bin/env bash
 
-NODE_MAJOR=18
-
 poetry install --no-root
 echo "export PROFILE=docker" >> ~/.bashrc
 
-curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - &&\
-apt-get install -y nodejs
-
 npm install -g --save-dev --save-exact prettier
 npm install -g editorconfig
 npm install -g --save-dev editorconfig-checker
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -75,7 +75,7 @@ jobs:
         id: filter
         with:
           filters: "tests/config/pytest_modules.yml"
-          token: ""
+          token: ${{ github.token }}
           base: ${{ steps.successful-ref.outputs.commit_hash }}
 
   lint-modules:
@@ -99,7 +99,7 @@ jobs:
       fail-fast: false
       matrix:
         module: ["${{ fromJson(needs.failing-modules.outputs.modules) }}"]
-        profile: ["docker", "singularity"]
+        profile: ["docker"] # , "singularity"]
         exclude:
           - module: "nf-test"
     uses: ./.github/workflows/test_module.yml

diff --git a/.github/workflows/test_module.yml b/.github/workflows/test_module.yml
@@ -70,20 +70,27 @@ jobs:
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
+      - name: Pull scilus image into cache
+        if: inputs.profile == 'singularity'
+        run: |
+          apptainer pull docker://scilus/scilus:1.6.0
+
       # Test the module
       - name: Run pytest-workflow
         # only use one thread for pytest-workflow to avoid race condition on conda cache.
-        run: TMPDIR=~/_work PROFILE=${{ inputs.profile }} pytest --tag ${{ inputs.module }} --symlink --kwdof --git-aware --color=yes
+        run: |
+            TMPDIR=/home/runner/_work PROFILE=${{ inputs.profile }} pytest --tag ${{ inputs.module }} --symlink --kwdof --git-aware --color=yes
 
       - name: Collate logs on failure
         if: failure()
         run: |
           sudo apt-get update > /dev/null
           sudo apt-get install bat > /dev/null
-          batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
+          batcat --decorations=always --color=always /home/runner/_work/pytest_workflow_*/*/log.{out,err}
 
       - name: Setting global variables
         uses: actions/github-script@v6
+        if: failure()
         id: parsed
         with:
           script: |
@@ -99,9 +106,7 @@ jobs:
             /home/runner/_work/pytest_workflow_*/*/.nextflow.log
             /home/runner/_work/pytest_workflow_*/*/log.out
             /home/runner/_work/pytest_workflow_*/*/log.err
-            /home/runner/_work/pytest_workflow_*/*/.command.log
-            /home/runner/_work/pytest_workflow_*/*/.command.sh
-            /home/runner/_work/pytest_workflow_*/*/.command.run
-            !/home/runner/_work/pytest_workflow_*/*/work/conda
-            !/home/runner/_work/pytest_workflow_*/*/work/singularity
+            /home/runner/_work/pytest_workflow_*/**/.command.log
+            /home/runner/_work/pytest_workflow_*/**/.command.sh
+            /home/runner/_work/pytest_workflow_*/**/.command.run
             !${{ github.workspace }}/.singularity
diff --git a/modules/nf-scil/tracking/localtracking/main.nf b/modules/nf-scil/tracking/localtracking/main.nf
@@ -49,17 +49,21 @@ process TRACKING_LOCALTRACKING {
             --data_type uint8
 
     elif [ "${local_tracking_mask}" == "fa" ]; then
-        mrcalc $fa $local_fa_tracking_mask_threshold -ge ${prefix}__local_tracking_mask.nii.gz\
-            --datatype uint8
+        scil_image_math.py lower_threshold $fa \
+            $local_fa_tracking_mask_threshold \
+            ${prefix}__local_tracking_mask.nii.gz \
+            --data_type uint8
     fi
 
     if [ "${local_seeding_mask}" == "wm" ]; then
         scil_image_math.py convert $wm ${prefix}__local_seeding_mask.nii.gz \
             --data_type uint8
 
     elif [ "${local_seeding_mask}" == "fa" ]; then
-        mrcalc $fa $local_fa_seeding_mask_threshold -ge ${prefix}__local_seeding_mask.nii.gz\
-            -datatype uint8
+        scil_image_math.py lower_threshold $fa \
+            $local_fa_seeding_mask_threshold \
+            ${prefix}__local_seeding_mask.nii.gz \
+            --data_type uint8
     fi
 
     scil_compute_local_tracking.py $fodf ${prefix}__local_seeding_mask.nii.gz ${prefix}__local_tracking_mask.nii.gz tmp.trk\
@@ -91,8 +95,6 @@ process TRACKING_LOCALTRACKING {
     "sh_basis": "${task.ext.basis}"}
     TRACKING_INFO
 
-    echo "remove, only to trigger tests"
-
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         scilpy: 1.6.0
@@ -105,7 +107,6 @@ process TRACKING_LOCALTRACKING {
     scil_compute_local_tracking.py -h
     scil_remove_invalid_streamlines.py -h
     scil_image_math -h
-    mrcalc -h
 
     touch ${prefix}__local_tracking.trk
     touch ${prefix}__local_tracking_config.json

diff --git a/modules/nf-scil/tracking/pfttracking/main.nf b/modules/nf-scil/tracking/pfttracking/main.nf
@@ -99,8 +99,6 @@ process TRACKING_PFTTRACKING {
     "sh_basis": "${task.ext.basis}"}
     TRACKING_INFO
 
-    echo "remove, only to trigger tests"
-
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         scilpy: 1.6.0

diff --git a/tests/modules/nf-scil/tracking/localtracking/nextflow.config b/tests/modules/nf-scil/tracking/localtracking/nextflow.config
@@ -13,7 +13,7 @@ process {
         ext.local_tracking_mask_type = "wm"
         ext.local_theta = 20
         ext.local_sfthres = 0.1
-        ext.local_min_len = 5
+        ext.local_min_len = 3
         ext.local_max_len = 40
         ext.basis = "descoteaux07"
     }
@@ -33,7 +33,7 @@ process {
         ext.local_tracking_mask_type = "fa"
         ext.local_theta = 20
         ext.local_sfthres = 0.1
-        ext.local_min_len = 5
+        ext.local_min_len = 3
         ext.local_max_len = 40
         ext.basis = "descoteaux07"
     }

diff --git a/tests/modules/nf-scil/tracking/localtracking/test.yml b/tests/modules/nf-scil/tracking/localtracking/test.yml
@@ -1,31 +1,31 @@
 - name: tracking localtracking test_tracking_localtracking_wm
   command: nextflow run ./tests/modules/nf-scil/tracking/localtracking -entry test_tracking_localtracking_wm -c ./tests/config/nextflow.config
   tags:
-    - tracking
     - tracking/localtracking
+    - tracking
   files:
     - path: output/tracking/test__local_seeding_mask.nii.gz
       md5sum: e4e926fce399749c2cd68c92eccf04be
     - path: output/tracking/test__local_tracking.trk
-      md5sum: bfc37c0d4bf0b1e8723ce0ffa3d42160
+      md5sum: 51cf5adf9d575b69b917d56a183779e3
     - path: output/tracking/test__local_tracking_config.json
-      md5sum: 2c580723a264df9016802b7594cf767f
+      md5sum: 5eac37e6262a85d8d1b33c92e1f35796
     - path: output/tracking/test__local_tracking_mask.nii.gz
       md5sum: e4e926fce399749c2cd68c92eccf04be
     - path: output/tracking/versions.yml
 
 - name: tracking localtracking test_tracking_localtracking_fa
   command: nextflow run ./tests/modules/nf-scil/tracking/localtracking -entry test_tracking_localtracking_fa -c ./tests/config/nextflow.config
   tags:
-    - tracking
     - tracking/localtracking
+    - tracking
   files:
     - path: output/tracking/test__local_seeding_mask.nii.gz
-      md5sum: 49c19bc6c0f47af32dea7af992b40490
+      md5sum: 8756cc4ae85dfa91dd70576ec581de92
     - path: output/tracking/test__local_tracking.trk
-      md5sum: 7d7f49582d9f1253ce6bf7e22f359260
+      md5sum: f733cd364bfaee913c23e1f91b8ff073
     - path: output/tracking/test__local_tracking_config.json
-      md5sum: 04229ce51ef839a62cc21a09851eeb04
+      md5sum: a082f8d2f2b92cb3f2912a796942a786
     - path: output/tracking/test__local_tracking_mask.nii.gz
-      md5sum: 49c19bc6c0f47af32dea7af992b40490
+      md5sum: 8756cc4ae85dfa91dd70576ec581de92
     - path: output/tracking/versions.yml