Update CI/CD for GitHub

.github/workflows/check-maps-precompmesh.yml

@@ -0,0 +1,22 @@
+name: '[test] check maps precompmesh'
+
+on:
+  push:
+    paths:
+    - 'maps/**.json'
+
+jobs:
+  check_maps_precompmesh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Check precomputed mesh
+      run: python _ci/check_precompmesh.py

.github/workflows/check-schema.yml

@@ -0,0 +1,31 @@
+name: '[test] check schema'
+
+on: [push]
+jobs:
+  check_schema:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: checkout siibra-python
+      uses: actions/checkout@v4
+      with:
+        repository: FZJ-INM1-BDA/siibra-python
+        path:  siibra-python-${{ github.run_id }}-${{ github.run_number }}
+        fetch-depth: 1
+        clean: True
+        ref: 'main'
+
+    - name: move siibra-python one up from workspace
+      run: mv siibra-python-${{ github.run_id }}-${{ github.run_number }} ../siibra-python
+
+    - name: Install requirements
+      run: pip install -r ../siibra-python/config_schema/requirements.txt
+
+    - name: check schema
+      run: python ../siibra-python/config_schema/check_schema.py ./

.github/workflows/check_connectivity_json.yml

@@ -0,0 +1,38 @@
+name: '[test] check connectivity'
+
+on: 
+  pull_request:
+    paths:
+      - 'features/connectivity/**/*.json'
+
+jobs:
+  check_connectivity_json:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: checkout siibra-python
+      uses: actions/checkout@v4
+      with:
+        repository: FZJ-INM1-BDA/siibra-python
+        path:  siibra-python-${{ github.run_id }}-${{ github.run_number }}
+        fetch-depth: 1
+        clean: True
+        ref: 'main'
+
+    - name: move siibra-python one up from workspace
+      run: mv siibra-python-${{ github.run_id }}-${{ github.run_number }} ../siibra-python
+
+    - name: Install siibra-python testing requirements
+      run: |
+          pip install -r ../siibra-python/requirements-test.txt
+          pip install -r ../siibra-python/requirements.txt
+
+    - name: connectivity e2e test
+      run: |
+        SIIBRA_USE_CONFIGURATION='./'
+        pytest ../siibra-python/e2e/features/connectivity/test_connectivity.py

.github/workflows/push-to-mirror.yml

@@ -0,0 +1,37 @@
+name: '[mirror] push to mirror'
+
+on:
+  push:
+    branches: ["master"]
+    tags: ["*"]
+
+env:
+  EBRAINS_GITLAB_HOST: ${{ vars.EBRAINS_GITLAB_HOST }}
+  EBRAINS_GITLAB_REPO: ri/tech-hub/apps/siibra/siibra-configurations-mirror.git
+
+jobs:
+  # n.b. we use access token, rather than https://github.com/valtech-sd/git-sync
+  # because it does not seem possible to add project wide ssh key
+  # and using personal key is not very flesible
+  push_to_mirror:
+    if: ${{ env.EBRAINS_GITLAB_HOST != '' }}
+    runs-on: ubuntu-latest
+    environment: gitlab_mirror
+    steps:
+    - name: before script
+      run: |
+        'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+        eval $(ssh-agent -s)
+        git config --global user.email "[email protected]"
+        git config --global user.name "inm1 bda - gitlab bot"
+        mkdir -p ~/.ssh
+        ssh-keyscan ${{ env.GITLAB_MIRROR_HOST }} >> gitlab-known-hosts
+        cat gitlab-known-hosts >> ~/.ssh/known_hosts
+
+    - name: script
+      run: |
+        git fetch --tags -f
+        git fetch origin master
+        if ! git ls-remote ebrains > /dev/null; then git remote add ebrains https://jugitpusher:${{ secrets.EBRAINS_GITLAB_PUSH_TOKEN }}@${{ env.GITLAB_MIRROR_HOST }}/${{ env.EBRAINS_GITLAB_REPO }}; fi
+        git push ebrains --tags -f
+        git push ebrains HEAD:master -f 

.github/workflows/release-checklist.yml

@@ -0,0 +1,108 @@
+name: '[ci] release-checklist'
+
+on:
+  push:
+    tags:
+      '**'
+  workflow_run:
+    workflows: ["test configuration"]
+    branches: [master]
+    types: 
+      - completed
+
+jobs:
+  generate-update:
+    runs-on: ubuntu-latest
+    environment:
+      RELEASE_CHECKLIST_TAG: release-checklist
+      RELEASE_TAG_ENV: '.latest-tag.env'
+      OPENED_IID_ENV: '.all-opened.env'
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+
+    steps:
+      - name: install dependencies
+        run: |
+          apt-get update -y && apt-get install -y jq
+          pip install requests
+
+      - name: get-latest-tag
+        if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+        run: |
+          url="$CI_API_V4_URL/projects/$CI_PROJECT_ID/repository/tags"
+          echo "url: $url"
+          newest_tag=$(curl -s $url | jq -r ".[0].name")
+          echo "newest_tag $newest_tag"
+          echo "export NEWEST_TAG=$newest_tag" >> $RELEASE_TAG_ENV
+
+      - name: get-all-opened
+        if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+        run: 
+
+      - name: update
+        run: |
+
+
+      - name: createnew
+        run: |
+          python ./_ci/create_new_issue.py
+
+      get-all-opened:
+        rules:
+        - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+        stage: collection
+        extends: .install-dep
+        artifacts:
+          paths:
+          - $OPENED_IID_ENV
+        script: |
+          url="$CI_API_V4_URL/projects/$CI_PROJECT_ID/issues?labels=${RELEASE_CHECKLIST_TAG}&state=opened"
+          echo "url: $url"
+          all_issues_id=$(curl -s $url | jq -r ".[].iid")
+          echo "all_issues_id $all_issues_id"
+          echo "ALL_IID=$all_issues_id" >> $OPENED_IID_ENV
+
+      close-existing-issues:
+        stage: update
+        extends: .install-dep
+        dependencies:
+        - get-all-opened
+        - get-latest-tag
+        rules:
+        - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+        script: |
+          if [[ -z "$RELEASE_CHECKLIST_CI_TOKEN" ]]
+          then
+            echo "RELEASE_CHECKLIST_CI_TOKEN not set, terminating!"
+            exit 1
+          fi
+          cat $RELEASE_TAG_ENV
+          cat $OPENED_IID_ENV
+
+          source $OPENED_IID_ENV
+          echo ALL_IID: $ALL_IID
+
+          for iid in $ALL_IID
+          do
+            url="$CI_API_V4_URL/projects/$CI_PROJECT_ID/issues/$iid"
+            echo Closing issue id: $iid with url: $url
+            curl -v -XPUT -H "PRIVATE-TOKEN: $RELEASE_CHECKLIST_CI_TOKEN" ${url}?state_event=close
+          done
+
+      open-new-issue:
+        stage: createnew
+        extends: .install-dep
+        dependencies:
+        - get-latest-tag
+        rules:
+        - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+        script: |
+          if [[ -z "$RELEASE_CHECKLIST_CI_TOKEN" ]]
+          then
+            echo "RELEASE_CHECKLIST_CI_TOKEN not set, terminating!"
+            exit 1
+          fi
+          cat $RELEASE_TAG_ENV
+          source $RELEASE_TAG_ENV 
+          python ./_ci/create_new_issue.py

.github/workflows/test-configuration.yml

@@ -0,0 +1,79 @@
+name: '[test] test configuration'
+
+on: [push]
+jobs:
+  check_jsons:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify json
+      run: python _ci/verify_json.py
+
+  check_ebrains_id:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify EBRAINS IDs
+      run: python _ci/verify_ebrains_ids.py
+
+  region_attr_compliance:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Region attribute compliance
+      run: python _ci/region_attr_compliance.py
+
+  check_map_volume_idx:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify map volume indices
+      run: python _ci/verify_volume_idx.py
+
+  check_maps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install requirements
+      run: |
+          pip install -U pip
+          pip install -r _ci/requirements.txt
+    - name: Verify maps
+      run: python _ci/verify_maps.py
+