full-test.yml

name: Full spikeinterface tests

on:
  pull_request:
    types: [synchronize, opened, reopened]
    branches:
      - main

concurrency:  # Cancel previous workflows on the same pull request
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:  #  For the sortingview backend
  KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
  KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}

jobs:
  full-tests-depending-on-changed-files:
    name: Test on (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        # "macos-latest", "windows-latest"
        os: ["ubuntu-latest", ]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: Get current year-month
        id: date
        run: echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
      - name: Restore cached virtual environment with dependencies
        uses: actions/cache/restore@v3
        id: cache-venv
        with:
          path: ${{ github.workspace }}/test_env
          key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml') }}-${{ steps.date.outputs.date }}
      - name: Get ephy_testing_data current head hash
        # the key depends on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
        id: vars
        run: |
          echo "HASH_EPHY_DATASET=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
      - name: Restore cached gin data for extractors tests
        uses: actions/cache/restore@v3
        id: cache-datasets
        env:
          # the key depends on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
          HASH_EPHY_DATASET: git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1
        with:
          path: ~/spikeinterface_datasets
          key: ${{ runner.os }}-datasets-${{ steps.vars.outputs.HASH_EPHY_DATASET }}
          restore-keys: ${{ runner.os }}-datasets
      - name: Install packages
        uses: ./.github/actions/build-test-environment
      - name: Shows installed packages by pip, git-annex and cached testing files
        uses: ./.github/actions/show-test-environment
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v35
      - name: Module changes
        id: modules-changed
        run: |
          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
            if [[ $file == *"pyproject.toml" ]]; then
              echo "pyproject.toml changed"
              echo "CORE_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/core/"* || $file == *"/extractors/neoextractors/neobaseextractor.py" ]]; then
              echo "Core changed"
              echo "CORE_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/extractors/"* ]]; then
              echo "Extractors changed"
              echo "EXTRACTORS_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/preprocessing/"* ]]; then
              echo "Preprocessing changed"
              echo "PREPROCESSING_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/postprocessing/"* ]]; then
              echo "Postprocessing changed"
              echo "POSTPROCESSING_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/qualitymetrics/"* ]]; then
              echo "Quality metrics changed"
              echo "QUALITYMETRICS_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/sorters/"* && $file != *"/sorters/internal/"* && $file != *"/sorters/external/"* ]]; then
              echo "Sorters changed"
              echo "SORTERS_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/sorters/external"* ]]; then
              echo "External sorters changed"
              echo "SORTERS_EXTERNAL_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/sorters/internal"* ]]; then
              echo "Internal sorters changed"
              echo "SORTERS_INTERNAL_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/comparison/"* ]]; then
              echo "Comparison changed"
              echo "COMPARISON_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/curation/"* ]]; then
              echo "Curation changed"
              echo "CURATION_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/widgets/"* ]]; then
              echo "Widgets changed"
              echo "WIDGETS_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/exporters/"* ]]; then
              echo "Exporters changed"
              echo "EXPORTERS_CHANGED=true" >> $GITHUB_OUTPUT
            fi
            if [[ $file == *"/sortingcomponents/"* ]]; then
              echo "Sortingcomponents changed"
              echo "SORTINGCOMPONENTS_CHANGED=true" >> $GITHUB_OUTPUT
            fi
          done
      - name: Set execute permissions on run_tests.sh
        run: chmod +x .github/run_tests.sh
      - name: Test core
        run: ./.github/run_tests.sh core
      - name: Test extractors
        if: ${{ steps.modules-changed.outputs.EXTRACTORS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh "extractors and not streaming_extractors"
      - name: Test preprocessing
        if: ${{ steps.modules-changed.outputs.PREPROCESSING_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh preprocessing
      - name: Test postprocessing
        if: ${{ steps.modules-changed.outputs.POSTPROCESSING_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh postprocessing
      - name: Test quality metrics
        if: ${{ steps.modules-changed.outputs.QUALITYMETRICS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh qualitymetrics
      - name: Test core sorters
        if: ${{ steps.modules-changed.outputs.SORTERS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh sorters
      - name: Test comparison
        if: ${{ steps.modules-changed.outputs.COMPARISON_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh comparison
      - name: Test curation
        if: ${{ steps.modules-changed.outputs.CURATION_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh curation
      - name: Test widgets
        if: ${{ steps.modules-changed.outputs.WIDGETS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' || steps.modules-changed.outputs.QUALITYMETRICS_CHANGED == 'true' ||  steps.modules-changed.outputs.PREPROCESSING_CHANGED == 'true'}}
        run: ./.github/run_tests.sh widgets
      - name: Test exporters
        if: ${{ steps.modules-changed.outputs.EXPORTERS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' || steps.modules-changed.outputs.WIDGETS_CHANGED == 'true' }}
        run: ./.github/run_tests.sh exporters
      - name: Test sortingcomponents
        if: ${{ steps.modules-changed.outputs.SORTINGCOMPONENTS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh sortingcomponents
      - name: Test internal sorters
        if: ${{ steps.modules-changed.outputs.SORTERS_INTERNAL_CHANGED == 'true' || steps.modules-changed.outputs.SORTINGCOMPONENTS_CHANGED || steps.modules-changed.outputs.CORE_CHANGED == 'true' }}
        run: ./.github/run_tests.sh sorters_internal