CI.yml

name: CI

on:
  push:
    # don't build commits to branches used by dependency autoupdater, as these
    # are redundant with the pull request builds
    branches-ignore:
      - 'autoupdate-*'
    # don't build changes to files that are irrelevant to the build
    paths-ignore:
      - 'demos/**'
      - 'docs/**'
      - 'language_development/**'
      - '!language_development/ref_testing.py'
      - '.gitignore'
      - 'LICENSE'
      - '**.md'
      - '.github/workflows/dependency-autoupdate*.yml'
      - '.github/workflows/clean-cache.yml'
      - 'actions/update*.py'
      - 'actions/clean_cache.py'
  pull_request:
  workflow_dispatch:
    inputs:
      FORCE_DEPLOY:
        type: boolean
        description: Force deployment (Allowed from master branch only)
        default: false

env:
  ICUDIR: ${{ github.workspace }}/thirdparty/icu
  JSONDIR: ${{ github.workspace }}/thirdparty/json
  JSON_INCLUDE: ${{ github.workspace }}/thirdparty/json/single_include

jobs:
  manylinux2014_x86_64:
    if: github.event_name != 'pull_request' || !startsWith(github.head_ref, 'autoupdate-') || github.head_ref != 'autoupdate-ccache' && github.head_ref != 'autoupdate-python-windows' && github.head_ref != 'autoupdate-python-osx' && github.head_ref != 'autoupdate-python-macosuniversal'
    runs-on: ubuntu-24.04
    outputs:
      REF_TESTING_PASSED: ${{ steps.tests.outputs.REF_TESTING_PASSED }}
    steps:
      - name: check out repository
        uses: actions/checkout@v4
      - name: get list of dependencies
        run: actions/dependencies.sh
      - name: download cached ICU installation
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-ICU-${{ env.ICU_URL }}
          path: ${{ env.ICUDIR }}
      - name: download cached JSON for Modern C++ installation
        uses: actions/cache@v4
        with:
          key: anyjob-JSON-${{ env.JSON_URL }}
          path: ${{ env.JSONDIR }}
      - name: download cached ccache and pip files
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-run-${{ github.run_number }}
          restore-keys: ${{ github.job }}-run-
          path: |
            ~/ccache
            ~/pipcache
      - name: build and run C++ unit tests
        run: docker container run --rm -e CCACHE_DIR=/ccache -e PIP_CACHE_DIR=/pipcache -e CCACHE_MAXSIZE=500M -e ICU_URL -e JSON_URL -e CYTHON_VERSION -v ~/ccache:/ccache -v ~/pipcache:/pipcache -v $GITHUB_WORKSPACE:/iknow quay.io/pypa/manylinux2014_x86_64:$MANYLINUX2014_X86_64_TAG /iknow/actions/build_manylinux.sh
      - name: upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-wheel
          path: ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
      - name: install
        run: |
          python -m venv venv
          ${{ github.workspace }}/venv/bin/pip install ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
      - name: run tests
        id: tests
        working-directory: ${{ github.workspace }}/language_development
        run: ${{ github.workspace }}/venv/bin/python ref_testing.py
      - name: upload test results
        if: steps.tests.outputs.REF_TESTING_PASSED == '0'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test
          path: |
            ${{ github.workspace }}/reference_materials/new_output
            ${{ github.workspace }}/reference_materials/reports
  manylinux2014_aarch64:
    if: github.event_name != 'pull_request' || !startsWith(github.head_ref, 'autoupdate-') || github.head_ref != 'autoupdate-ccache' && github.head_ref != 'autoupdate-python-windows' && github.head_ref != 'autoupdate-python-osx' && github.head_ref != 'autoupdate-python-macosuniversal'
    runs-on: ubuntu-24.04
    steps:
      - name: check out repository
        uses: actions/checkout@v4
      - name: get list of dependencies
        run: actions/dependencies.sh
      - name: download cached ICU installation
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-ICU-${{ env.ICU_URL }}
          path: ${{ env.ICUDIR }}
      - name: download cached JSON for Modern C++ installation
        uses: actions/cache@v4
        with:
          key: anyjob-JSON-${{ env.JSON_URL }}
          path: ${{ env.JSONDIR }}
      - name: download cached ccache and pip files
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-run-${{ github.run_number }}
          restore-keys: ${{ github.job }}-run-
          path: |
            ~/ccache
            ~/pipcache
      - name: install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y qemu-user-static binfmt-support
      - name: build and run C++ unit tests
        run: docker container run --rm -e CCACHE_DIR=/ccache -e PIP_CACHE_DIR=/pipcache -e CCACHE_MAXSIZE=500M -e ICU_URL -e JSON_URL -e CYTHON_VERSION -v ~/ccache:/ccache -v ~/pipcache:/pipcache -v $GITHUB_WORKSPACE:/iknow quay.io/pypa/manylinux2014_aarch64:$MANYLINUX2014_AARCH64_TAG /iknow/actions/build_manylinux.sh
      - name: upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-wheel
          path: ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
  manylinux2014_ppc64le:
    if: github.event_name != 'pull_request' || !startsWith(github.head_ref, 'autoupdate-') || github.head_ref != 'autoupdate-ccache' && github.head_ref != 'autoupdate-python-windows' && github.head_ref != 'autoupdate-python-osx' && github.head_ref != 'autoupdate-python-macosuniversal'
    runs-on: ubuntu-24.04
    steps:
      - name: check out repository
        uses: actions/checkout@v4
      - name: get list of dependencies
        run: actions/dependencies.sh
      - name: download cached ICU installation
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-ICU-${{ env.ICU_URL }}
          path: ${{ env.ICUDIR }}
      - name: download cached JSON for Modern C++ installation
        uses: actions/cache@v4
        with:
          key: anyjob-JSON-${{ env.JSON_URL }}
          path: ${{ env.JSONDIR }}
      - name: download cached ccache and pip files
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-run-${{ github.run_number }}
          restore-keys: ${{ github.job }}-run-
          path: |
            ~/ccache
            ~/pipcache
      - name: install dependencies
        run:  |
          sudo apt-get update
          sudo apt-get install -y qemu-user-static binfmt-support
      - name: build and run C++ unit tests
        run: docker container run --rm -e CCACHE_DIR=/ccache -e PIP_CACHE_DIR=/pipcache -e CCACHE_MAXSIZE=500M -e ICU_URL -e JSON_URL -e CYTHON_VERSION -v ~/ccache:/ccache -v ~/pipcache:/pipcache -v $GITHUB_WORKSPACE:/iknow quay.io/pypa/manylinux2014_ppc64le:$MANYLINUX2014_PPC64LE_TAG /iknow/actions/build_manylinux.sh
      - name: upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-wheel
          path: ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
  macos_10_9_universal:
    if: github.event_name != 'pull_request' || !startsWith(github.head_ref, 'autoupdate-') || github.head_ref != 'autoupdate-ccache' && github.head_ref != 'autoupdate-manylinux' && github.head_ref != 'autoupdate-python-windows' && github.head_ref != 'autoupdate-python-osx'
    runs-on: macos-14
    outputs:
      REF_TESTING_PASSED: ${{ steps.tests.outputs.REF_TESTING_PASSED }}
    steps:
      - name: check out repository
        uses: actions/checkout@v4
      - name: set up environment
        run: |
          echo "XCODE_SELECTED=/Applications/Xcode_15.4.app" >> $GITHUB_ENV
          echo "HOMEBREW_NO_AUTO_UPDATE=1" >> $GITHUB_ENV
          echo "HOMEBREW_NO_INSTALL_CLEANUP=1" >> $GITHUB_ENV
          echo "MACOSX_DEPLOYMENT_TARGET=10.9" >> $GITHUB_ENV
          echo "MACOSX_VERSION=$(sw_vers -productVersion | cut -d '.' -f 1,2)" >> $GITHUB_ENV
          mkdir -p $HOME/cache
          echo "CCACHE_DIR=$HOME/ccache" >> $GITHUB_ENV
          echo "CCACHE_MAXSIZE=400M" >> $GITHUB_ENV
          echo /Library/Frameworks/Python.framework/Versions/Current/bin >> $GITHUB_PATH
      - name: get list of dependencies
        run: actions/dependencies.sh
      - name: download cached ICU installation
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-${{ env.MACOSX_VERSION }}-ICU-${{ env.ICU_URL }}-${{ env.XCODE_SELECTED }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}
          path: ${{ env.ICUDIR }}
      - name: download cached JSON for Modern C++ installation
        uses: actions/cache@v4
        with:
          key: anyjob-JSON-${{ env.JSON_URL }}
          path: ${{ env.JSONDIR }}
      - name: download cached ccache files
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-${{ env.MACOSX_VERSION }}-ccache-${{ env.XCODE_SELECTED }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}-run-${{ github.run_number }}
          restore-keys: ${{ github.job }}-${{ env.MACOSX_VERSION }}-ccache-${{ env.XCODE_SELECTED }}-${{ env.MACOSX_DEPLOYMENT_TARGET }}-run-
          path: ${{ env.CCACHE_DIR }}
      - name: install dependencies
        run: actions/install_macosuniversal.sh
      - name: build and run C++ unit tests
        run: actions/build_macosuniversal.sh
      - name: upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-wheel
          path: ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
      - name: install
        run: pip3 install ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
      - name: run tests
        id: tests
        working-directory: ${{ github.workspace }}/language_development
        run: python3 ref_testing.py
      - name: upload test results
        if: steps.tests.outputs.REF_TESTING_PASSED == '0'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test
          path: |
            ${{ github.workspace }}/reference_materials/new_output
            ${{ github.workspace }}/reference_materials/reports
  windows_x86_64:
    if: github.event_name != 'pull_request' || !startsWith(github.head_ref, 'autoupdate-') || github.head_ref != 'autoupdate-manylinux' && github.head_ref != 'autoupdate-python-osx' && github.head_ref != 'autoupdate-python-macosuniversal'
    runs-on: windows-2022
    defaults:
      run:
        shell: bash
    outputs:
      REF_TESTING_PASSED: ${{ steps.tests.outputs.REF_TESTING_PASSED }}
    steps:
      - name: check out repository
        uses: actions/checkout@v4
      - name: set up environment
        run: |
          mkdir -p $HOME/ccache_exe
          echo "CCACHE_EXE_DIR=$HOME/ccache_exe" >> $GITHUB_ENV
          mkdir -p $HOME/ccache
          echo "CCACHE_DIR=$HOME/ccache" >> $GITHUB_ENV
          echo "CCACHE_MAXSIZE=2G" >> $GITHUB_ENV
          mkdir -p $HOME/python
          echo "PYINSTALL_DIR=$HOME/python" >> $GITHUB_ENV
      - name: Use GNU tar to speed up cache restore
        shell: cmd
        run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
      - name: get list of dependencies
        run: actions/dependencies.sh
      - name: download cached ICU installation
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-ICU-${{ env.ICU_URL }}
          path: ${{ env.ICUDIR }}
      - name: download cached ccache installation
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-ccache-${{ env.CCACHE_URL }}
          path: ~/ccache_exe
      - name: download cached JSON for Modern C++ installation
        uses: actions/cache@v4
        with:
          key: anyjob-JSON-${{ env.JSON_URL }}
          path: ${{ env.JSONDIR }}
      - name: download cached ccache files, Python installations
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-run-${{ github.run_number }}
          restore-keys: ${{ github.job }}-run-
          path: |
            ~/ccache
            ~/python
      - name: install dependencies
        run: actions/install_windows.sh
      - name: build and run C++ unit tests
        run: actions/build_windows.sh
      - name: upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-wheel
          path: ${{ github.workspace }}/modules/iknowpy/wheelhouse/iknowpy-*.whl
      - name: install
        shell: powershell
        run: pip install $(ls ${{ github.workspace }}\modules\iknowpy\wheelhouse\iknowpy-*.whl | % {$_.FullName})
      - name: run tests
        id: tests
        working-directory: ${{ github.workspace }}/language_development
        run: python ref_testing.py
      - name: upload test results
        if: steps.tests.outputs.REF_TESTING_PASSED == '0'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test
          path: |
            ${{ github.workspace }}/reference_materials/new_output
            ${{ github.workspace }}/reference_materials/reports

  test_result:
    needs: [manylinux2014_x86_64, manylinux2014_aarch64, manylinux2014_ppc64le, macos_10_9_universal, windows_x86_64]
    runs-on: ubuntu-22.04
    if: needs.manylinux2014_x86_64.outputs.REF_TESTING_PASSED == '0' || needs.macos_10_9_universal.outputs.REF_TESTING_PASSED == '0' || needs.windows_x86_64.outputs.REF_TESTING_PASSED == '0'
    steps:
      - name: create comment
        uses: peter-evans/commit-comment@v3
        with:
          body: >-
            **Error**: Reference testing failed. See the build artifacts at
            https://github.com/${{ github.repository}}/actions/runs/${{ github.run_id}}
            for details.

  deploy:
    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'master'
    needs: [manylinux2014_x86_64, manylinux2014_aarch64, manylinux2014_ppc64le, macos_10_9_universal, windows_x86_64]
    runs-on: ubuntu-22.04
    env:
      PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
      TESTPYPI_TOKEN: ${{ secrets.TESTPYPI_TOKEN }}
      COMMIT_RANGE: ${{ join(github.event.commits.*.id, ' ') }}
      FORCE_DEPLOY: ${{ github.event.inputs.FORCE_DEPLOY }}
    steps:
      - name: check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 21  # github.event.commits has limit of 20 commits, so we need at least 21 to perform diffs
      - name: download cached pip files
        uses: actions/cache@v4
        with:
          key: ${{ github.job }}-run-${{ github.run_number }}
          restore-keys: ${{ github.job }}-run-
          path: ~/.cache/pip
      - name: download manylinux2014_x86_64-wheel
        uses: actions/download-artifact@v4
        with:
          name: manylinux2014_x86_64-wheel
          path: ~/wheels
      - name: download manylinux2014_aarch64-wheel
        uses: actions/download-artifact@v4
        with:
          name: manylinux2014_aarch64-wheel
          path: ~/wheels
      - name: download manylinux2014_ppc64le-wheel
        uses: actions/download-artifact@v4
        with:
          name: manylinux2014_ppc64le-wheel
          path: ~/wheels
      - name: download macos_10_9_universal-wheel
        uses: actions/download-artifact@v4
        with:
          name: macos_10_9_universal-wheel
          path: ~/wheels
      - name: download windows_x86_64-wheel
        uses: actions/download-artifact@v4
        with:
          name: windows_x86_64-wheel
          path: ~/wheels
      - name: deploy if appropriate
        run: actions/deploy.sh
      - name: warn if version.py was changed but not in last commit
        if: env.WARN_COMMIT
        uses: peter-evans/commit-comment@v3
        with:
          sha: ${{ env.WARN_COMMIT }}
          body: >-
            **Warning:** The version bump must be part of the final commit in a
            push to `master` in order to trigger deployment. This is by design
            because we do not want commits that were made after a version bump
            to affect the new release. The final commit in the push was
            ${{ github.sha }}, so automatic deployment was not triggered.



            If the final commit in the push was a merge commit that `git`
            automatically created, you can do the following to prevent a merge
            commit from interfering with deployment the next time you bump the
            version number.

            - Use `git pull --rebase` instead of `git pull` when updating your
            local repository.

            - Do not bump the version number in a fork or non-`master` branch
            before merging into the `master` branch of `intersystems/iknow`.
            Instead, bump the version number directly in the `master` branch
            after the merge.



            If you want, you can still deploy the wheels for this release by
            downloading them from the **Artifacts** section of
            https://github.com/${{ github.repository}}/actions/runs/${{ github.run_id}}
            and manually uploading them to PyPI. Note that these wheels contain
            all commits in the push, including those that were made after the
            version bump. Alternatively, you can use the **Run workflow**
            button at https://github.com/${{ github.repository}}/actions?query=workflow%3ACI
            to force a deployment.
      - name: check PyPI project size
        if: env.DEPLOY_OCCURRED == '1'
        run: ${{ github.workspace }}/actions/check_pypi_limit.py ${{ secrets.GITHUB_TOKEN }}
      - name: create issue if PyPI project size is near limit
        if: env.DEPLOY_OCCURRED == '1' && env.CREATE_ISSUE == '1'
        uses: peter-evans/create-issue-from-file@v5
        with:
          title: "[check-pypi-limit] The PyPI size limit has almost been reached"
          content-filepath: ${{ env.HOME }}/issue.md
          assignees: isc-adang