main.yaml

name: Continuous integration

on:
  push:
    branches:
      - master
      - '[0-9]+.[0-9]+'
    tags:
      - '*'
  pull_request:

env:
  HAS_SECRETS: ${{ secrets.HAS_SECRETS }}

jobs:
  not-failed-backport:
    name: Test that's not a failed backport
    runs-on: ubuntu-24.04
    timeout-minutes: 5

    steps:
      - run: 'false'
        if: github.event.head_commit.message == '[skip ci] Add instructions to finish the backport.'

  main:
    name: Continuous integration
    runs-on: ubuntu-24.04
    timeout-minutes: 135
    if: "!startsWith(github.event.head_commit.message, '[skip ci] ')"

    env:
      MAIN_BRANCH: master
      MAJOR_VERSION: '2.10'
      # When we upgrade this we should also upgrade the requirements
      # in the documentation: doc/integrator/requirements.rst
      # and the first pyupgrade pre-commit hook in .pre-commit-config.yaml
      MIN_PYTHON_VERSION: '3.10'

    steps:
      - run: '! ls BACKPORT_TODO'
      - run: df -h
      - name: Clean all docker images
        run: docker system prune --all --force
      # Inspired by https://github.com/jlumbroso/free-disk-space/
      - run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet "$AGENT_TOOLSDIRECTORY"
      - run: df -h

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ secrets.GOPASS_CI_GITHUB_TOKEN }}
        if: env.HAS_SECRETS == 'HAS_SECRETS'
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
        if: env.HAS_SECRETS != 'HAS_SECRETS'

      - uses: camptocamp/initialise-gopass-summon-action@v2
        with:
          ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}}
          github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}}
          patterns: pypi docker transifex
        if: env.HAS_SECRETS == 'HAS_SECRETS'

      - uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - run: python3 -m pip install --requirement=ci/requirements.txt
      - name: Environment information
        run: c2cciutils-env
        env:
          # To display the content of the github object
          GITHUB_EVENT: ${{ toJson(github) }}

      - run: c2cciutils-download-applications --applications-file=ci/applications.yaml --versions-file=ci/applications-versions.yaml
      - id: version
        run: scripts/get-version --auto-increment --github

      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
          restore-keys: "pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}\npre-commit-"
      - run: pre-commit run --all-files
      - run: git diff --exit-code --patch > /tmp/pre-commit.patch || true
        if: failure()
      - uses: actions/upload-artifact@v4
        with:
          name: Apply pre-commit fix.patch
          path: /tmp/pre-commit.patch
          retention-days: 1
        if: failure()

      - run: python3 -m pip install --requirement=requirements.txt

        # Build images
      - run: make build-runner
      - run: make build-tools
      - run: make checks
        if: always()
      - run: make build-config

      # Build and lint QGIS images
      - run: QGIS_VERSION=3.34-gdal3.8 make build-qgisserver
      - run: QGIS_VERSION=3.34-gdal3.8 make prospector-qgisserver

      - run: make preparetest

      # Auto generate Alembic script
      - run: |
          docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=main check \
            || docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=main \
            revision --autogenerate --message='${{ github.event.pull_request.title }}'
          docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=static check \
            || docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=static \
            revision --autogenerate --message='${{ github.event.pull_request.title }}'
          sudo chmod go+rw -R commons/c2cgeoportal_commons/alembic/
          # Replace 'main' by schema and 'main. by f'{schema}.
          sed -i "s/'main'/schema/g" commons/c2cgeoportal_commons/alembic/*/*.py
          sed -i "s/'main\.'/f'{schema}./g" commons/c2cgeoportal_commons/alembic/*/*.py
          # Replace 'main_static' by staticschema and 'main_static. by f'{staticschema}.
          sed -i "s/'main_static'/staticschema/g" commons/c2cgeoportal_commons/alembic/*/*.py
          sed -i "s/'main_static\.'/f'{staticschema}./g" commons/c2cgeoportal_commons/alembic/*/*.py
          git checkout commons/c2cgeoportal_commons/alembic/main/ee25d267bf46_main_interface_desktop.py
          git checkout commons/c2cgeoportal_commons/alembic/main/415746eb9f6_changes_for_v2.py
          git add commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py
          pre-commit run --files commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py || true
          git add commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py
          git diff --staged --patch > /tmp/alembic.patch
          git diff --staged --exit-code
          git reset --hard
      - uses: actions/upload-artifact@v4
        with:
          name: Add Alembic upgrade script.patch
          path: /tmp/alembic.patch
          retention-days: 1
        if: failure()

      # Tests
      - run: c2cciutils-docker-logs
      - run: make tests-commons
      - run: c2cciutils-docker-logs
      - run: make tests-geoportal
      - run: c2cciutils-docker-logs
      - run: make tests-admin
      - run: c2cciutils-docker-logs
      - run: make tests-qgisserver
      - run: c2cciutils-docker-logs
        if: always()
      - run: docker compose down

      - name: Test version generation for the changelog
        run: scripts/updated_version latest latest

      - run: sudo git clean -fdx

        # Documentation
      - run: >
          docker build --tag=camptocamp/geomapfish-doc
          --build-arg=MAJOR_VERSION=${MAJOR_VERSION}
          --build-arg=MAIN_BRANCH=${MAIN_BRANCH}
          doc
        env:
          DOCKER_BUILDKIT: '1'

      - name: Extract documentation
        run: ci/extract-documentation artifacts/documentations/ || true
        if: always()

      - uses: actions/upload-artifact@v4
        with:
          name: Documentation
          path: artifacts/documentations/
          if-no-files-found: ignore
          retention-days: 5
        if: always()

      # Test App with Docker compose version 2.x
      - timeout-minutes: 30
        run: ci/test-app
      - name: Docker logs
        continue-on-error: true
        run: |
          cd ${HOME}/workspace/testgeomapfishapp/
          c2cciutils-docker-logs
        if: failure()

      # Use minimal version from the documentation
      - uses: actions/setup-python@v5
        with:
          python-version: '${{ env.MIN_PYTHON_VERSION }}'
      - run: pip install --requirement=ci/requirements-project.txt
      - run: docker compose version

      # Test App
      - timeout-minutes: 30
        run: ci/test-app
      - name: Docker logs
        continue-on-error: true
        run: |
          cd ${HOME}/workspace/testgeomapfishapp/
          c2cciutils-docker-logs
        if: failure()

      - run: git stash
      - run: git pull --ff-only origin ${{ env.MAIN_BRANCH }}
        if: >
          github.ref == format('refs/heads/{0}', env.MAIN_BRANCH)
          && env.HAS_SECRETS == 'HAS_SECRETS'
      - run: git stash pop || true

        # Test Upgrade
      - run: DOCKER_TAG=${{ steps.version.outputs.full }} make build-tools
      - run: DOCKER_TAG=${{ steps.version.outputs.full }} make build-runner
      - run: DOCKER_TAG=${{ steps.version.outputs.full }} make build-config
      - run: docker images | grep "<none>" | awk '{print $3}' | xargs --no-run-if-empty docker rmi || true
      - run: ci/test-upgrade init ${HOME}/workspace
      - run: ci/test-upgrade 270 ${HOME}/workspace
      - run: ci/test-upgrade 280 ${HOME}/workspace
      - run: ci/test-upgrade 29 ${HOME}/workspace
      - run: ci/test-upgrade cleanup ${HOME}/workspace

      - uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - run: pip install --requirement=ci/requirements-back.txt
      - run: docker compose version

      - name: Init Git
        run:
          git remote set-url origin https://${GITHUB_ACTOR}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository
          }}
      - run: make build-tools
      - run: make build-runner
      - run: make build-config

      - run: ci/create-new-project ${HOME}/workspace geomapfishapp
      - run: (cd ${HOME}/workspace/geomapfishapp/; ./build)

      - name: Update the changelog
        run: ci/changelog ${{ steps.version.outputs.full }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - run: git diff CHANGELOG.md

      - name: Push version and changelog
        run: |
          set -eux
          git add ci/ci.yaml ci/changelog.yaml CHANGELOG.md
          git diff --staged --quiet || (\
            git commit -m "[skip ci] Update the minor version"; \
            git push origin HEAD:${{ env.MAIN_BRANCH }} \
          )
        if: >
          github.ref == format('refs/heads/{0}', env.MAIN_BRANCH)
          && env.HAS_SECRETS == 'HAS_SECRETS'

      - run: git reset --hard
      - name: Publish
        run: >
          c2cciutils-publish
          --docker-versions=${{ steps.version.outputs.versions }}
          --snyk-version=${{ steps.version.outputs.snyk_version }}
        if: >
          env.HAS_SECRETS == 'HAS_SECRETS'
          && steps.version.outputs.versions != ''
      - name: Publish version branch to pypi
        run: |
          c2cciutils-publish --group=pypi --type=version_tag --version=${{ steps.version.outputs.full }}
        if: >
          github.ref == format('refs/heads/{0}', env.MAIN_BRANCH)
          && env.HAS_SECRETS == 'HAS_SECRETS'
          && env.MAIN_BRANCH != 'master'
      - run: git diff --exit-code --patch > /tmp/dpkg-versions.patch || true
        if: failure()
      - uses: actions/upload-artifact@v4
        with:
          name: Update dpkg versions list.patch
          path: /tmp/dpkg-versions.patch
          retention-days: 1
        if: failure()

      - name: Notify demo
        run: >
          curl --request POST --header "Content-Type: application/json"
          --header 'Accept: application/vnd.github.v3+json'
          --header "Authorization: token ${{ secrets.GOPASS_CI_GITHUB_TOKEN }}"
          https://api.github.com/repos/camptocamp/demo_geomapfish/dispatches
          --data '{"event_type": "geomapfish_${{ env.MAJOR_VERSION }}_updated",
            "client_payload": {"version": "'"${{ steps.version.outputs.upgrade_version }}"'"}}'
        if: >
          github.ref == format('refs/heads/{0}', env.MAIN_BRANCH)
          && env.HAS_SECRETS == 'HAS_SECRETS'

      - name: Publish to Transifex
        run: |
          git diff || true
          git status || true
          git status --ignored || true
          make build-tools
          docker run --name=transifex -ti --rm --detach --volume=${HOME}:/root camptocamp/geomapfish-tools tail -f /dev/null
          docker exec transifex bash -c \
              '(cd /opt/c2cgeoportal; make --makefile=dependencies.mk transifex-send)'
          docker stop transifex
        env:
          DOCKER_BUILDKIT: '1'
        if: >
          github.ref == format('refs/heads/{0}', env.MAIN_BRANCH)
          && env.HAS_SECRETS == 'HAS_SECRETS'

      - name: Publish documentation to GitHub.io
        run: ci/publish-documentation
        if: >
          github.ref == format('refs/heads/{0}', env.MAIN_BRANCH)
          && env.HAS_SECRETS == 'HAS_SECRETS'

      - run: docker images
        if: always()
      - run: docker system df
        if: always()
      - run: df -h
        if: always()