build-pr.yml

name: Build Pull Request
run-name: Build PR ${{ github.event.number }} «${{ github.event.pull_request.title }}» (${{ github.actor}})
on:
  pull_request:
    types:
      - opened
      - synchronize
  workflow_call:
    inputs:
      is_called_workflow:
        required: true
        default: false
        type: boolean
        description: |
          Should be set to true. This is used to determine of this workflow was called directly or
          from another workflow.
      do_instrumentation:
        required: false
        default: false
        type: boolean
        description: |
          Set to true to enable instrumentation of Igor code. The execution will be approx. 30%
          slower.
    secrets:
      GHA_MIES_CERTIFICATE_PIN:
        required: true
      GHA_MIES_FTP_ARTEFACT_STORAGE_USER:
        required: true
      GHA_MIES_FTP_ARTEFACT_STORAGE_PWD:
        required: true
defaults:
  run:
    shell: bash
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
jobs:

  BuildInstaller:
    name: 🏗 Build ${{ matrix.kind }} installer
    strategy:
      matrix:
        kind:
          # the normal installer
          - user
          # This installer is *only* intended for developer internal use!
          - dev
    runs-on: [ self-hosted, Windows, Certificate ]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0 # load all commits
      - name: Initial repo config
        run: tools/initial-repo-config.sh
      - name: Create installer
        run: tools/create-installer.sh ${{ fromJSON('["", "dev"]')[ matrix.kind == 'dev' ] }}
      - name: Sign installer
        run: tools/sign-installer.sh -p "${{ secrets.GHA_MIES_CERTIFICATE_PIN }}"
      - name: upload artifacts
        uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: BuildInstaller-${{ matrix.kind }}-assets
          path: |
            tools/installer/MIES-*.exe
          if-no-files-found: error

  Linting:
    name: 🔎 Linting
    runs-on: [ self-hosted, Linux, Docker ]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Code Checks
        run: tools/check-code.sh
        # straight from the documentation, see https://pre-commit.com/#github-actions-example
      - name: set PY
        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Pre commit
        run: tools/pre-commit/run.sh

  CompilationTest:
    name: |
      👩🏾‍🔬 Compilation: ${{ matrix.source }} ${{ fromJSON('["","with XOP"]')[matrix.hardware] }}
    needs:
      - BuildInstaller
    strategy:
      matrix:
        source: [ git, installer ]
        hardware: [ false, true ]
        exclude:
          # the git source is always checked with hardware support
          - source: git
            hardware: false
    uses: ./.github/workflows/test-igor-workflow.yml
    with:
      job_name: |
        👩🏾‍🔬 Compilation: ${{ matrix.source }} ${{ fromJSON('["","with XOP"]')[matrix.hardware] }}
      overwrite_job_name: ${{ inputs.is_called_workflow || false }}
      experiment: Packages/tests/Compilation/CompilationTester.pxp
      installer_artifact_name: BuildInstaller-dev-assets
      installer_flags: ${{ fromJSON('["-x skipHardwareXOPs",""]')[matrix.hardware] }} -s ${{ matrix.source }}
      artifact_name: CompilationTest-${{ matrix.source }}-${{ fromJSON('["no-hardware","hardware"]')[matrix.hardware] }}-assets
      timeout_minutes: 60

  Documentation:
    name: 👷 Documentation
    runs-on: [ self-hosted, Linux, Docker ]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0 # load all commits
      - name: Initial repo config
        run: tools/initial-repo-config.sh
      - name: Build documentation
        run: tools/documentation/run.sh
      - name: upload artifacts
        uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: Documentation-assets
          path: |
            Packages/doc/*.zip
          if-no-files-found: error

  TestWithoutHardware:
    name: 🧪 Test ${{ matrix.name }}
    needs:
      - BuildInstaller
      - Linting
      - CompilationTest
    strategy:
      matrix:
        include:
          - experiment: Packages/tests/Basic/Basic.pxp
            name: Basic
          - experiment: Packages/tests/PAPlot/PAPlot.pxp
            name: PAPlot
          - experiment: Packages/tests/HistoricData/HistoricData.pxp
            name: HistoricData
    uses: ./.github/workflows/test-igor-workflow.yml
    with:
      job_name: 🧪 Test ${{ matrix.name }}
      overwrite_job_name: ${{ inputs.is_called_workflow || false }}
      experiment: ${{ matrix.experiment }}
      artifact_name: test-without-hw-assets
      expensive_checks: "0"
      instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }}
      timeout_minutes: 60

  TestNI:
    name: 🧪 Test NI ${{ matrix.name }}
    needs:
      - BuildInstaller
      - CompilationTest
      - TestWithoutHardware
    strategy:
      matrix:
        include:
          - experiment: Packages/tests/HardwareBasic/HardwareBasic-NI.pxp
            name: HardwareBasic
          - experiment: Packages/tests/HardwareAnalysisFunctions/HardwareAnalysisFunctions-NI.pxp
            name: HardwareAnalysisFunctions
      fail-fast: false
    uses: ./.github/workflows/test-igor-workflow.yml
    with:
      job_name: 🧪 Test NI ${{ matrix.name }}
      overwrite_job_name: ${{ inputs.is_called_workflow || false }}
      experiment: ${{ matrix.experiment }}
      target: "[ 'self-hosted', 'Windows', 'IgorPro', 'NI' ]"
      artifact_name: test-ni-assets
      expensive_checks: "0"
      instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }}
      timeout_minutes: 180

  TestITC18:
    name: 🧪 Test ITC18-USB ${{ matrix.name }}
    needs:
      - BuildInstaller
      - CompilationTest
      - TestWithoutHardware
    strategy:
      matrix:
        include:
          - experiment: Packages/tests/HardwareBasic/HardwareBasic.pxp
            name: HardwareBasic
          - experiment: Packages/tests/HardwareAnalysisFunctions/HardwareAnalysisFunctions.pxp
            name: HardwareAnalysisFunctions
      fail-fast: false
    uses: ./.github/workflows/test-igor-workflow.yml
    with:
      job_name: 🧪 Test ITC18-USB ${{ matrix.name }}
      overwrite_job_name: ${{ inputs.is_called_workflow || false }}
      experiment: ${{ matrix.experiment }}
      target: "[ 'self-hosted', 'Windows', 'IgorPro', 'ITC' ]"
      artifact_name: test-itc18-assets
      expensive_checks: "0"
      instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }}
      timeout_minutes: 180

  TestITC1600:
    name: 🧪 Test ITC1600 ${{ matrix.name }}
    needs:
      - BuildInstaller
      - CompilationTest
      - TestWithoutHardware
    strategy:
      matrix:
        include:
          - experiment: Packages/tests/HardwareBasic/HardwareBasic-ITC1600.pxp
            name: HardwareBasic
          - experiment: Packages/tests/HardwareAnalysisFunctions/HardwareAnalysisFunctions-ITC1600.pxp
            name: HardwareAnalysisFunctions
      fail-fast: false
    uses: ./.github/workflows/test-igor-workflow.yml
    with:
      job_name: 🧪 Test ITC1600 ${{ matrix.name }}
      overwrite_job_name: ${{ inputs.is_called_workflow || false }}
      experiment: ${{ matrix.experiment }}
      target: "[ 'self-hosted', 'Windows', 'IgorPro', 'ITC1600' ]"
      artifact_name: test-itc1600-assets
      expensive_checks: "0"
      instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }}
      timeout_minutes: 180

  ValidateNwb:
    name: 👮🏼 Validate NWBv2
    runs-on: [ self-hosted, Linux, Docker ]
    needs:
      - TestITC18
      - TestITC1600
      - TestNI
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - name: Initial repo config
        run: tools/initial-repo-config.sh
      - name: Download ITC18-USB artifacts
        uses: actions/download-artifact@v3
        with:
          name: test-itc18-assets
      - name: Download ITC1600 artifacts
        uses: actions/download-artifact@v3
        with:
          name: test-itc1600-assets
      - name: Download NI artifacts
        uses: actions/download-artifact@v3
        with:
          name: test-ni-assets
      - name: Validate and read NWBv2 files
        run: tools/nwb-read-tests/run.sh

  FTPUpload:
    name: 📦 FTP Upload Artifacts
    runs-on: [ self-hosted, Linux, Docker ]
    if: ${{ !cancelled() && !inputs.is_called_workflow}}
    needs:
      # All jobs that create new artifacts
      - BuildInstaller
      - CompilationTest
      - Documentation
      - TestWithoutHardware
      - TestNI
      - TestITC18
      - TestITC1600
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Initial repo config
        run: tools/initial-repo-config.sh
      - name: Download all artifacts
        id: download
        uses: actions/download-artifact@v3
        with:
          path: artifacts
      - name: Flatten artifact structure
        run: |
          find "${{ steps.download.outputs.download-path }}" -mindepth 1 -maxdepth 1 -type d |\
            xargs -I {} tools/ftp-upload/flatten-files.sh "{}"
      - name: Compress NWB artifacts
        run: |
          for dir in test-itc18-assets test-itc1600-assets test-ni-assets
          do
            tar --remove-files --use-compress-program=zstd -cvf $dir/NWB.tar.zst $dir/*nwb
          done
        working-directory: ${{ steps.download.outputs.download-path }}
      - name: Upload artifacts using FTP
        run: |
          tools/ftp-upload/upload-files.sh \
            -s "${{ vars.GHA_MIES_FTP_ARTEFACT_STORAGE_DNS }}" \
            -u "${{ secrets.GHA_MIES_FTP_ARTEFACT_STORAGE_USER }}" \
            -p "${{ secrets.GHA_MIES_FTP_ARTEFACT_STORAGE_PWD }}" \
            -d "${{ steps.download.outputs.download-path }}" \
            -t "pr/$(echo "${{ github.event.pull_request.head.ref }}" | sed "s@/@_@g")/${{ github.event.pull_request.head.sha }}/${{ github.run_attempt }}"