👷 Launch runs for regression tests from CI (2/2)

.github/workflows/build_and_test.yml

@@ -339,6 +339,20 @@ jobs:
     if: github.ref_name == 'develop' || github.ref_name == 'main'
     uses: ./.github/workflows/smoke_test_participant.yml
 
+  regtest-lite:
+    name: Run lite regression test
+    needs:
+      - C-PAC
+    secrets: inherit
+    if: contains(github.event.head_commit.message, '[run reg-suite]')
+    uses: ./.github/workflows/regression_test_lite.yml
+
+  regtest-full:
+    name: Run full regression test
+    needs:
+      - smoke-tests-participant
+    uses: ./.github/workflows/regression_test_full.yml
+
   Circle_tests:
     name: Run tests on CircleCI
     needs:

.github/workflows/on_push.yml

@@ -40,6 +40,7 @@ jobs:
           python-version: '3.9'
           cache: 'pip'
       - name: Check if version updated
+        id: version_updated
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "Theodore (Machine User)"

.github/workflows/regression_test_full.yml

@@ -0,0 +1,41 @@
+name: Run Regression Full Test
+
+on: 
+  workflow_call:
+
+jobs: 
+  test:
+    name: Regression Test - Full
+    runs-on: ubuntu-latest
+    steps: 
+      - name: Get C-PAC branch
+        run: |
+          GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
+          if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
+          then
+            TAG=${GITHUB_BRANCH//\//_}
+          elif [[ $GITHUB_BRANCH == 'develop' ]]
+          then
+            TAG=nightly
+          elif [[ $GITHUB_BRANCH == 'main' ]]
+          then
+            TAG=latest
+          fi
+
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Clone reg-suite
+        run: |
+          git clone https://github.com/amygutierrez/reg-suite.git
+
+      - name: Run Full Regression Test
+        if: ${{ github.event_name }} == "pull_request" && ${{ github.event.pull_request.state }} == "closed" && ${{ github.event.pull_request.merged }} == "true" && ${{ github.event.pull_request.base.ref }} == "main"
+        run: |
+          echo "Running full regression test"
+          echo "୧(๑•̀ヮ•́)૭ LET'S GO! ٩(^ᗜ^ )و "
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: logs
+          path: output/*/*/log/

.github/workflows/regression_test_lite.yml

@@ -0,0 +1,84 @@
+name: Launch lite regression test
+
+on:
+  pull_request:
+    branches:
+      - develop
+    types:
+      - opened
+      - ready_for_review
+      - reopened
+  workflow_call:
+    secrets:
+      GH_CLI_BIN_PATH:
+        description: 'path to directory containing GitHub CLI binary if not on default $PATH'
+        required: false
+      SSH_PRIVATE_KEY:
+        required: true
+      SSH_USER:
+        required: true
+      SSH_HOST:
+        required: true
+      SSH_WORK_DIR:
+        required: true
+
+jobs: 
+  test:
+    name: Regression Test - Lite
+    environment: ACCESS
+    runs-on: ubuntu-latest
+    steps: 
+      - name: Get C-PAC branch
+        run: |
+          GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
+          if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
+          then
+            TAG=${GITHUB_BRANCH//\//_}
+          elif [[ $GITHUB_BRANCH == 'develop' ]]
+          then
+            TAG=nightly
+          elif [[ $GITHUB_BRANCH == 'main' ]]
+          then
+            TAG=latest
+          fi
+          TAG=$TAG$VARIANT
+          echo DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}" | tr '[:upper:]' '[:lower:]'):$TAG >> $GITHUB_ENV
+          cat $GITHUB_ENV
+
+      - name: Install SSH Keys
+        run: |
+          mkdir -p ~/.ssh/
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -H -t rsa ${{ secrets.SSH_HOST }} > ~/.ssh/known_hosts
+
+      - name: Initiate check
+        uses: guibranco/[email protected]
+        with:
+          authToken: ${{ secrets.GITHUB_TOKEN }}
+          context: Launch lite regression test
+          description: launching
+          state: pending
+
+      - name: Connect and Run Regression Test Lite
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          command_timeout: 200m
+          script: |
+            cd ${{ secrets.SSH_WORK_DIR }}
+            if [ ! -d slurm_testing ] ; then
+                git clone https://github.com/${{ github.repository_owner }}/slurm_testing slurm_testing
+            else
+                cd slurm_testing
+                git pull origin regression/after_runs
+                cd ..
+            fi
+            mkdir -p ./logs/${{ github.sha }}
+            sbatch --export="HOME_DIR=${{ secrets.SSH_WORK_DIR }},IMAGE=${{ env.DOCKER_TAG }},OWNER=${{ github.repository_owner }},PATH_EXTRA=${{ secrets.GH_CLI_BIN_PATH }},REPO=$(echo ${{ github.repository }} | cut -d '/' -f 2),SHA=${{ github.sha }}" --output=${{ secrets.SSH_WORK_DIR }}/logs/${{ github.sha }}/out.log --error=${{ secrets.SSH_WORK_DIR }}/logs/${{ github.sha }}/error.log slurm_testing/.github/scripts/launch_regtest_lite.SLURM
+
+      - name: Cleanup SSH
+        run: |
+          rm -rf ~/.ssh

CPAC/utils/datasource.py

@@ -18,7 +18,7 @@
 import json
 import re
 from pathlib import Path
-from typing import Tuple, Union
+from typing import Union
 from nipype import logging
 from nipype.interfaces import utility as util
 from CPAC.pipeline import nipype_pipeline_engine as pe
@@ -369,6 +369,7 @@ def get_fmap_phasediff_metadata(data_config_scan_params):
             echo_time_one, echo_time_two)
 
 
+@Function.sig_imports(['from CPAC.utils.typing import TUPLE'])
 def calc_delta_te_and_asym_ratio(effective_echo_spacing: float,
                                  echo_times: list) -> TUPLE[float, float]:
     """Calcluate ``deltaTE`` and ``ees_asym_ratio`` from given metadata

CPAC/utils/monitoring/draw_gantt_chart.py

@@ -650,8 +650,9 @@ def _timing_timestamp(node):
     -------
     dict
     """
-    if node is None:
+    if node is None or node.items() is None:
         raise ProcessLookupError('No logged nodes have timing information.')
-    return {k: (datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f") if
-            '.' in v else datetime.fromisoformat(v)) if
-            k in {"start", "finish"} else v for k, v in node.items()}
+    return {k: (datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f") if '.' in v else
+                datetime.fromisoformat(v)) if (k in {"start", "finish"} and
+                                               isinstance(v, str)) else
+            v for k, v in node.items()}

requirements.txt

@@ -47,4 +47,4 @@ pip==23.1.2
 setuptools==68.0.0
 urllib3==1.26.15
 wheel==0.40.0
-zipp==3.16.0
+zipp==3.16.0