Merge pull request #6 from FZJ-INM1-BDA/update_CICD

Update CI/CD for GitHub

.github/workflows/check-maps-precompmesh.yml

@@ -0,0 +1,22 @@
+name: '[test] check maps precompmesh'
+
+on:
+  push:
+    paths:
+    - 'maps/**.json'
+
+jobs:
+  check_maps_precompmesh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Check precomputed mesh
+      run: python _ci/check_precompmesh.py

.github/workflows/check-schema.yml

@@ -0,0 +1,31 @@
+name: '[test] check schema'
+
+on: [push]
+jobs:
+  check_schema:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: checkout siibra-python
+      uses: actions/checkout@v4
+      with:
+        repository: FZJ-INM1-BDA/siibra-python
+        path:  siibra-python-${{ github.run_id }}-${{ github.run_number }}
+        fetch-depth: 1
+        clean: True
+        ref: 'main'
+
+    - name: move siibra-python one up from workspace
+      run: mv siibra-python-${{ github.run_id }}-${{ github.run_number }} ../siibra-python
+
+    - name: Install requirements
+      run: pip install -r ../siibra-python/config_schema/requirements.txt
+
+    - name: check schema
+      run: python ../siibra-python/config_schema/check_schema.py ./

.github/workflows/check_connectivity_json.yml

@@ -0,0 +1,38 @@
+name: '[test] check connectivity'
+
+on: 
+  pull_request:
+    paths:
+      - 'features/connectivity/**/*.json'
+
+jobs:
+  check_connectivity_json:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: checkout siibra-python
+      uses: actions/checkout@v4
+      with:
+        repository: FZJ-INM1-BDA/siibra-python
+        path:  siibra-python-${{ github.run_id }}-${{ github.run_number }}
+        fetch-depth: 1
+        clean: True
+        ref: 'main'
+
+    - name: move siibra-python one up from workspace
+      run: mv siibra-python-${{ github.run_id }}-${{ github.run_number }} ../siibra-python
+
+    - name: Install siibra-python testing requirements
+      run: |
+          pip install -r ../siibra-python/requirements-test.txt
+          pip install -r ../siibra-python/requirements.txt
+    
+    - name: connectivity e2e test
+      run: |
+        SIIBRA_USE_CONFIGURATION='./'
+        pytest ../siibra-python/e2e/features/connectivity/test_connectivity.py

.github/workflows/push-to-mirror.yml

@@ -0,0 +1,37 @@
+name: '[mirror] push to mirror'
+
+on:
+  push:
+    branches: ["master"]
+    tags: ["**"]
+
+env:
+  EBRAINS_GITLAB_HOST: ${{ vars.EBRAINS_GITLAB_HOST }}
+  EBRAINS_GITLAB_REPO: ri/tech-hub/apps/siibra/siibra-configurations-mirror.git
+
+jobs:
+  # n.b. we use access token, rather than https://github.com/valtech-sd/git-sync
+  # because it does not seem possible to add project wide ssh key
+  # and using personal key is not very flesible
+  push_to_mirror:
+    if: ${{ env.EBRAINS_GITLAB_HOST != '' }}
+    runs-on: ubuntu-latest
+    environment: gitlab_mirror
+    steps:
+    - name: before script
+      run: |
+        'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+        eval $(ssh-agent -s)
+        git config --global user.email "[email protected]"
+        git config --global user.name "inm1 bda - gitlab bot"
+        mkdir -p ~/.ssh
+        ssh-keyscan ${{ env.GITLAB_MIRROR_HOST }} >> gitlab-known-hosts
+        cat gitlab-known-hosts >> ~/.ssh/known_hosts
+
+    - name: script
+      run: |
+        git fetch --tags -f
+        git fetch origin master
+        if ! git ls-remote ebrains > /dev/null; then git remote add ebrains https://jugitpusher:${{ secrets.EBRAINS_GITLAB_PUSH_TOKEN }}@${{ env.GITLAB_MIRROR_HOST }}/${{ env.EBRAINS_GITLAB_REPO }}; fi
+        git push ebrains --tags -f
+        git push ebrains HEAD:master -f 

.github/workflows/release-checklist.yml

@@ -0,0 +1,125 @@
+name: '[ci] release-checklist'
+
+on:
+  push:
+    tags: ["**"]
+
+env:
+  RELEASE_CHECKLIST_TAG: release-checklist
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  REPO: ${{ github.repository }}
+  ACTOR: ${{ github.actor }}
+  TAG_NAME: ${{ github.ref_name }}
+
+jobs:
+
+  # workflow and siibra-python will likely break with tag including /
+  # workflow: see sed
+  # siibra-python: see reftag escaping
+  sanitiy-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: "checks tag does not contain slash"
+      run: |
+        echo ${{ env.TAG_NAME }} | grep -Fq "/" && exit 1 || exit 0
+
+  get-all-opened:
+    runs-on: ubuntu-latest
+    needs:
+    - sanitiy-check
+    if: success()
+    outputs:
+      OPEN_ISSUES: ${{ steps.get-all-issues.outputs.ALL_ISSUES }}
+    steps:
+    - name: 'Getting all opened issue matching ${{ env.RELEASE_CHECKLIST_TAG }} in ${{ env.REPO }}'
+      id: 'get-all-issues'
+      run: |
+        issue=$(gh issue list \
+          --repo ${{ env.REPO }} \
+          --label '${{ env.RELEASE_CHECKLIST_TAG }}' \
+          --json 'id,body,number,url,title')
+        echo Found:
+        echo $issue
+        echo "ALL_ISSUES=$issue" >> $GITHUB_OUTPUT
+  
+  update-existing-body:
+    runs-on: ubuntu-latest
+    needs:
+    - get-all-opened
+    steps:
+    - run: |
+        OPEN_ISSUES=${{ needs.get-all-opened.outputs.OPEN_ISSUES }}
+        echo Found open issues: $OPEN_ISSUES
+        for issue in $(echo $OPEN_ISSUES | jq -c '.[]')
+        then
+          url=$(echo $issue | jq -r '.url')
+
+          new_issue_body=$(echo $issue | jq -r '.body' | sed '1s/master/${{ env.TAG_NAME }}/')
+          new_title=$(echo $issue | jq -r '.title' | sed 's/v???/${{ env.TAG_NAME }}/')
+
+          echo update issue $url
+          echo new_title=$new_title
+          echo new_issue_body: $new_issue_body
+
+          echo -e $new_issue_body | gh issue edit "$url" -F -
+        fi
+
+  add-comment-existing:
+    runs-on: ubuntu-latest
+    needs:
+    - get-all-opened
+    steps:
+    - run: |
+        OPEN_ISSUES=${{ needs.get-all-opened.outputs.OPEN_ISSUES }}
+        echo Found open issues: $OPEN_ISSUES
+        for issue in $(echo $OPEN_ISSUES | jq -c '.[]')
+        then
+          url=$(echo $issue | jq -r '.url')
+
+          echo $issue | jq -r '.body' | grep -Fq "<person name>"
+          validated=$(echo $?)
+          
+          new_comment="Thank you ${{ env.ACTOR }} for the release."
+          if [[ "$validated" != "0" ]]
+          then
+            new_comment="$new_comment\\n\\nCan you double check you have validated the release?"
+          fi
+          
+          echo update issue $url
+          echo -e new_comment: \\n\\n bar $new_comment
+          echo -e $new_comment | gh issue comment $url -F -
+        fi
+
+  close-existing:
+    runs-on: ubuntu-latest
+    needs:
+    - get-all-opened
+    steps:
+    - name: "Add comment to issue"
+      run: |
+        OPEN_ISSUES=${{ needs.get-all-opened.outputs.OPEN_ISSUES }}
+        echo Found open issues: $OPEN_ISSUES
+        for issue in $(echo $OPEN_ISSUES | jq -c '.[]')
+        then
+          id=$(echo $issue | jq -r '.id')
+          url=$(echo $issue | jq -r '.url')
+
+          gh issue close $url -r completed
+        fi
+
+  open-new:
+    runs-on: ubuntu-latest
+    needs:
+    - get-all-opened
+    steps:
+    - uses: actions/checkout@v4
+    - run: |
+        new_issue=$(sed '1s/LATEST_TAG_PLACEHOLDER/${{ env.TAG_NAME }}/' release_checklist.md)
+
+        echo -e new_issue: $new_issue
+
+        echo -e new_issue | gh issue create \
+          -R ${{ env.REPO }} \
+          -l '${{ env.RELEASE_CHECKLIST_TAG }}' \
+          -t '[release checklist] v???' \
+          -F -

.github/workflows/test-configuration.yml

@@ -0,0 +1,79 @@
+name: '[test] test configuration'
+
+on: [push]
+jobs:
+  check_jsons:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify json
+      run: python _ci/verify_json.py
+
+  check_ebrains_id:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify EBRAINS IDs
+      run: python _ci/verify_ebrains_ids.py
+
+  region_attr_compliance:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Region attribute compliance
+      run: python _ci/region_attr_compliance.py
+
+  check_map_volume_idx:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify map volume indices
+      run: python _ci/verify_volume_idx.py
+
+  check_maps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify maps
+      run: python _ci/verify_maps.py
+