diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c7f8a16c3f..62a7a34f4b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -339,6 +339,20 @@ jobs: if: github.ref_name == 'develop' || github.ref_name == 'main' uses: ./.github/workflows/smoke_test_participant.yml + regtest-lite: + name: Run lite regression test + needs: + - C-PAC + secrets: inherit + if: contains(github.event.head_commit.message, '[run reg-suite]') + uses: ./.github/workflows/regression_test_lite.yml + + regtest-full: + name: Run full regression test + needs: + - smoke-tests-participant + uses: ./.github/workflows/regression_test_full.yml + Circle_tests: name: Run tests on CircleCI needs: diff --git a/.github/workflows/on_push.yml b/.github/workflows/on_push.yml index 3789236eb2..c4a5ceacd4 100644 --- a/.github/workflows/on_push.yml +++ b/.github/workflows/on_push.yml @@ -40,6 +40,7 @@ jobs: python-version: '3.9' cache: 'pip' - name: Check if version updated + id: version_updated run: | git config --global user.email "CMI_CPAC_Support@childmind.org" git config --global user.name "Theodore (Machine User)" diff --git a/.github/workflows/regression_test_full.yml b/.github/workflows/regression_test_full.yml new file mode 100644 index 0000000000..39fe11f5d2 --- /dev/null +++ b/.github/workflows/regression_test_full.yml @@ -0,0 +1,41 @@ +name: Run Regression Full Test + +on: + workflow_call: + +jobs: + test: + name: Regression Test - Full + runs-on: ubuntu-latest + steps: + - name: Get C-PAC branch + run: | + GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-) + if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]] + then + TAG=${GITHUB_BRANCH//\//_} + elif [[ $GITHUB_BRANCH == 'develop' ]] + then + TAG=nightly + elif [[ $GITHUB_BRANCH == 'main' ]] + then + TAG=latest + fi + + - name: Checkout Code + uses: actions/checkout@v2 + - name: Clone reg-suite + run: | + git clone https://github.com/amygutierrez/reg-suite.git + + - name: Run Full Regression Test + if: ${{ github.event_name }} == "pull_request" && ${{ github.event.pull_request.state }} == "closed" && ${{ github.event.pull_request.merged }} == "true" && ${{ github.event.pull_request.base.ref }} == "main" + run: | + echo "Running full regression test" + echo "୧(๑•̀ヮ•́)૭ LET'S GO! ٩(^ᗜ^ )و " + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: logs + path: output/*/*/log/ diff --git a/.github/workflows/regression_test_lite.yml b/.github/workflows/regression_test_lite.yml new file mode 100644 index 0000000000..25cb40f569 --- /dev/null +++ b/.github/workflows/regression_test_lite.yml @@ -0,0 +1,96 @@ +name: Launch lite regression test + +on: + pull_request: + branches: + - develop + types: + - opened + - ready_for_review + - reopened + pull_request_target: # Make sure environment protection is enabled! + branches: + - develop + types: + - opened + - ready_for_review + - reopened + workflow_call: + secrets: + GH_CLI_BIN_PATH: + description: 'path to directory containing GitHub CLI binary if not on default $PATH' + required: false + SSH_PRIVATE_KEY: + required: true + SSH_USER: + required: true + SSH_HOST: + required: true + SSH_WORK_DIR: + required: true + +jobs: + test: + name: Regression Test - Lite + environment: ACCESS + env: + SHA: ${{ github.event.pull_request.head.sha }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_HOST: ${{ secrets.SSH_HOST }} + if: (${{ github.event.pull_request.head.repo.full_name }} == ${{ github.repository }} && ${{ github.event_name }} == 'pull_request') || (${{ github.event.pull_request.head.repo.full_name }} != ${{ github.repository }} && ${{ github.event_name }} == 'pull_request_target') + runs-on: ubuntu-latest + steps: + - name: Get C-PAC branch + run: | + GITHUB_BRANCH=${{ github.event.pull_request.head.ref }} + if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]] + then + TAG=${GITHUB_BRANCH//\//_} + elif [[ $GITHUB_BRANCH == 'develop' ]] + then + TAG=nightly + elif [[ $GITHUB_BRANCH == 'main' ]] + then + TAG=latest + fi + TAG=$TAG$VARIANT + echo DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}" | tr '[:upper:]' '[:lower:]'):$TAG >> $GITHUB_ENV + cat $GITHUB_ENV + + - name: Install SSH Keys + run: | + mkdir -p ~/.ssh/ + echo "${{ env.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -H -t rsa "${{ env.SSH_HOST }}" > ~/.ssh/known_hosts + + - name: Initiate check + uses: guibranco/github-status-action-v2@v1.1.7 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + context: Launch lite regression test + description: launching + state: pending + + - name: Connect and Run Regression Test Lite + uses: appleboy/ssh-action@v1.0.0 + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USER }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + command_timeout: 200m + script: | + cd ${{ secrets.SSH_WORK_DIR }} + if [ ! -d slurm_testing ] ; then + git clone https://github.com/${{ github.repository_owner }}/slurm_testing slurm_testing + else + cd slurm_testing + git pull origin regression/after_runs + cd .. + fi + mkdir -p ./logs/${SHA} + sbatch --export="HOME_DIR=${{ secrets.SSH_WORK_DIR }},IMAGE=${{ env.DOCKER_TAG }},OWNER=${{ github.repository_owner }},PATH_EXTRA=${{ secrets.GH_CLI_BIN_PATH }},REPO=$(echo ${{ github.repository }} | cut -d '/' -f 2),SHA=${SHA}" --output=${{ secrets.SSH_WORK_DIR }}/logs/${SHA}/out.log --error=${{ secrets.SSH_WORK_DIR }}/logs/${SHA}/error.log slurm_testing/.github/scripts/launch_regtest_lite.SLURM + + - name: Cleanup SSH + run: | + rm -rf ~/.ssh diff --git a/CPAC/utils/datasource.py b/CPAC/utils/datasource.py index 42534e325f..8f5523394d 100644 --- a/CPAC/utils/datasource.py +++ b/CPAC/utils/datasource.py @@ -18,7 +18,7 @@ import json import re from pathlib import Path -from typing import Tuple, Union +from typing import Union from nipype import logging from nipype.interfaces import utility as util from CPAC.pipeline import nipype_pipeline_engine as pe @@ -369,6 +369,7 @@ def get_fmap_phasediff_metadata(data_config_scan_params): echo_time_one, echo_time_two) +@Function.sig_imports(['from CPAC.utils.typing import TUPLE']) def calc_delta_te_and_asym_ratio(effective_echo_spacing: float, echo_times: list) -> TUPLE[float, float]: """Calcluate ``deltaTE`` and ``ees_asym_ratio`` from given metadata diff --git a/CPAC/utils/monitoring/draw_gantt_chart.py b/CPAC/utils/monitoring/draw_gantt_chart.py index 7695a3a291..6440aa30d0 100644 --- a/CPAC/utils/monitoring/draw_gantt_chart.py +++ b/CPAC/utils/monitoring/draw_gantt_chart.py @@ -650,8 +650,9 @@ def _timing_timestamp(node): ------- dict """ - if node is None: + if node is None or node.items() is None: raise ProcessLookupError('No logged nodes have timing information.') - return {k: (datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f") if - '.' in v else datetime.fromisoformat(v)) if - k in {"start", "finish"} else v for k, v in node.items()} + return {k: (datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f") if '.' in v else + datetime.fromisoformat(v)) if (k in {"start", "finish"} and + isinstance(v, str)) else + v for k, v in node.items()} diff --git a/requirements.txt b/requirements.txt index 062606e1f6..2b4fa7f3df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,4 +47,4 @@ pip==23.1.2 setuptools==68.0.0 urllib3==1.26.15 wheel==0.40.0 -zipp==3.16.0 \ No newline at end of file +zipp==3.16.0