ci.yml

name: CI

on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch:
    # Needed so we can run it manually

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  RUST_BACKTRACE: 1
  CARGO_TERM_COLOR: always

jobs:
  ci-everything:
    name: All CI stages
    runs-on: ubuntu-latest
    needs:
      - lint
      - rust-tests
      - rust-fuzz
      - rust-rustdoc-wasm
      - python-tests
      - wasm-tests
      - js-lint
    if: ${{ success() || failure() }}  # Run this job even if a dependency has failed.
    steps:
      - name: Job outcomes
        run: |
          echo "lint: ${{ needs.lint.result }}"
          echo "rust-tests: ${{ needs.rust-tests.result }}"
          echo "rust-fuzz: ${{ needs.rust-fuzz.result }}"
          echo "python-tests: ${{ needs.python-tests.result }}"
          echo "wasm-tests: ${{ needs.wasm-tests.result }}"
          echo "js-lint: ${{ needs.js-lint.result }}"
          echo "rust-rustdoc-wasm: ${{ needs.rust-rustdoc-wasm.result }}"

      # Fail this required job if any of its dependent jobs have failed.
      #
      # Do not attempt to consolidate these steps into one step, it won't work.
      # Multi-line `if` clauses are not evaluated properly: see the intermediate commits in
      # https://github.com/obi1kenobi/cargo-semver-checks/pull/405
      - if: ${{ needs.lint.result != 'success' }}
        run: exit 1
      - if: ${{ needs.rust-tests.result != 'success' }}
        run: exit 1
      - if: ${{ needs.rust-fuzz.result != 'success' }}
        run: exit 1
      - if: ${{ needs.python-tests.result != 'success' }}
        run: exit 1
      - if: ${{ needs.wasm-tests.result != 'success' }}
        run: exit 1
      - if: ${{ needs.js-lint.result != 'success' }}
        run: exit 1
      - if: ${{ needs.rust-rustdoc-wasm.result != 'success' }}
        run: exit 1

  python-tests:
    name: Python tests and maturin build
    runs-on: ubuntu-latest
    needs:
      - rust-tests
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13.0-rc - 3.13"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust + caching
        uses: actions-rust-lang/setup-rust-toolchain@v1

      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: pytrustfall/.venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: |
          cd pytrustfall
          poetry install --no-interaction --no-root

      - name: maturin build and test
        run: |
          cd pytrustfall
          source .venv/bin/activate
          maturin develop
          pytest
          maturin build --interpreter python -o target/wheels/

      - name: mypy
        run: |
          cd pytrustfall
          source .venv/bin/activate
          mypy

      - name: Set environment variables
        id: branch-info
        run: |
          set -ex
          echo "COMMIT=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
          echo "TAG_NAME=$(.github/workflows/get_py_prerelease_name.sh)" >> $GITHUB_OUTPUT
          echo "PYTHON_BINDINGS_CHANGED=$(.github/workflows/python_bindings_changed.sh)" >> $GITHUB_OUTPUT

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        if: steps.branch-info.outputs.PYTHON_BINDINGS_CHANGED != 0
        with:
          name: pytrustfall-wheels
          path: pytrustfall/target/wheels/
          retention-days: 1

      - name: Make a new prerelease
        uses: ncipollo/release-action@v1
        if: steps.branch-info.outputs.BRANCH_NAME == 'main' && steps.branch-info.outputs.PYTHON_BINDINGS_CHANGED != 0
        with:
          artifacts: "pytrustfall/target/wheels/*.whl,pytrustfall/target/wheels/*.tar.gz"
          commit: ${{ steps.branch-info.outputs.COMMIT }}
          tag: ${{ steps.branch-info.outputs.TAG_NAME }}
          generateReleaseNotes: true
          prerelease: true
          token: ${{ secrets.GITHUB_TOKEN }}

  lint:
    name: Check lint and rustfmt
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust + caching
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: rustfmt, clippy

      - name: cargo clippy
        run: cargo clippy --workspace --all-features --all-targets --no-deps -- -D warnings --allow deprecated

      - name: cargo fmt
        run: cargo fmt -- --check

      - name: cargo doc
        env:
          RUSTDOCFLAGS: -D warnings
        run: cargo doc --workspace --all-features --no-deps --document-private-items

  rust-tests:
    name: Run tests
    runs-on: ubuntu-latest
    env:
      CARGO_INCREMENTAL: 0
      SCCACHE_GHA_ENABLED: "true"
      RUSTC_WRAPPER: "sccache"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: true

      - name: Install rust + caching
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          rustflags: ""

      - name: Set up sccache
        uses: mozilla-actions/sccache-action@v0.0.6

      - name: Clear sccache stats
        run: sccache --zero-stats

      # Test all crates except trustfall_stubgen,
      # which is only tested if it has changed since its tests are a bit long.
      #
      # `--all-targets` also tests example and bench targets, but explicitly excludes doctests.
      # We'll test doctests separately in their own step.
      # https://github.com/rust-lang/cargo/issues/6669
      - name: cargo test
        run: cargo test --workspace --all-targets --all-features --exclude trustfall_stubgen

      # Run doctests separately, since `--all-targets` above explicitly excludes doctests.
      # https://github.com/rust-lang/cargo/issues/6669
      - name: run doctests
        run: cargo test --workspace --all-features --doc

      - name: test trustfall_stubgen if it has changed
        run: |
          git fetch origin main

          # `git diff --quiet` exits non-zero if there are changes
          git diff --quiet HEAD origin/main -- ./trustfall_stubgen || (cd trustfall_stubgen/ && cargo test --all-features)

      - name: Show sccache stats
        run: sccache --show-stats

  rust-fuzz:
    name: Check fuzz targets
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: |
            ./trustfall_core/fuzz

      - name: cargo clippy
        run: |
          cd trustfall_core/fuzz
          cargo clippy --workspace --all-features --all-targets --no-deps -- -D warnings --allow deprecated

      - name: cargo fmt
        run: |
          cd trustfall_core/fuzz
          cargo fmt -- --check

      - name: cargo doc
        env:
          RUSTDOCFLAGS: -D warnings
        run: |
          cd trustfall_core/fuzz
          cargo doc --workspace --all-features --no-deps --document-private-items

  rust-rustdoc-wasm:
    name: Check rustdoc WASM target
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: |
            ./experiments/trustfall_rustdoc

      - name: cargo clippy
        run: |
          cd experiments/trustfall_rustdoc
          cargo clippy --all-features --all-targets --no-deps -- -D warnings --allow deprecated

      - name: cargo fmt
        run: |
          cd experiments/trustfall_rustdoc
          cargo fmt -- --check

      - name: cargo doc
        env:
          RUSTDOCFLAGS: -D warnings
        run: |
          cd experiments/trustfall_rustdoc
          cargo doc --all-features --no-deps --document-private-items

  wasm-tests:
    name: WASM tests
    runs-on: ubuntu-latest
    needs:
     - rust-tests
    permissions:
      contents: write
    env:
      CARGO_INCREMENTAL: 0
      SCCACHE_GHA_ENABLED: "true"
      RUSTC_WRAPPER: "sccache"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust + caching
        uses: actions-rust-lang/setup-rust-toolchain@v1

      - name: Install wasm-pack
        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh

      - name: Set up sccache
        uses: mozilla-actions/sccache-action@v0.0.6

      - name: Clear sccache stats
        run: sccache --zero-stats

      - name: Run cargo test and wasm-pack tests
        run: |
          cd trustfall_wasm
          wasm-pack test --headless --firefox
          wasm-pack test --headless --chrome

      - name: Set environment variables
        id: branch-info
        run: |
          set -ex
          echo "COMMIT=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
          echo "TAG_NAME=$(.github/workflows/get_wasm_prerelease_name.sh)" >> $GITHUB_OUTPUT
          echo "WASM_CHANGED=$(.github/workflows/wasm_pkg_changed.sh)" >> $GITHUB_OUTPUT

      - name: Build WASM module
        if: steps.branch-info.outputs.WASM_CHANGED != 0
        run: |
          cd trustfall_wasm
          wasm-pack build --no-typescript
          cp src/trustfall_wasm.d.ts pkg/
          cd pkg/
          tar -czvf ../trustfall_wasm.tar.gz --exclude='.gitignore' .

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        if: steps.branch-info.outputs.WASM_CHANGED != 0
        with:
          name: wasm-module
          path: trustfall_wasm/pkg/
          retention-days: 1

      - name: Make a new prerelease
        uses: ncipollo/release-action@v1
        if: steps.branch-info.outputs.BRANCH_NAME == 'main' && steps.branch-info.outputs.WASM_CHANGED != 0
        with:
          artifacts: "trustfall_wasm/trustfall_wasm.tar.gz"
          commit: ${{ steps.branch-info.outputs.COMMIT }}
          tag: ${{ steps.branch-info.outputs.TAG_NAME }}
          generateReleaseNotes: true
          prerelease: true
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Show sccache stats
        run: sccache --show-stats

  js-lint:
    name: Run JS linters
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: experiments/schemaless_wasm/www/
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust + caching  # necessary for `npm run build:wasm` later
        uses: actions-rust-lang/setup-rust-toolchain@v1

      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install dependencies
        run: npm ci

      - name: Build wasm package
        run: npm run build:wasm

      - name: Run linters
        run: npm run lint

  pre-publish-checks:
    name: pre-publish checks
    if: github.ref == 'refs/heads/main'
    needs:
      - ci-everything
      - check-trustfall-examples-and-expensive-tests
    runs-on: ubuntu-latest
    steps:
      - run: exit 0

  check-trustfall-examples-and-expensive-tests:
    name: pre-publish checks
    if: github.ref == 'refs/heads/main'
    needs:
      - rust-tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Install rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false
          rustflags: ""

      - name: Publishing?
        id: version
        run: |
          set +e
          .github/workflows/is_crate_version_already_uploaded.sh trustfall
          export TRUSTFALL="$?"
          .github/workflows/is_crate_version_already_uploaded.sh trustfall_core
          export TRUSTFALL_CORE="$?"
          .github/workflows/is_crate_version_already_uploaded.sh trustfall_derive
          export TRUSTFALL_DERIVE="$?"
          if [[ "$TRUSTFALL" == "7" && "$TRUSTFALL_CORE" == "7" && "$TRUSTFALL_DERIVE" == "7" ]]; then
            echo 'is_new_version=no' >> $GITHUB_OUTPUT
          elif [[ "$TRUSTFALL" == "0" || "$TRUSTFALL_CORE" == "0" || "$TRUSTFALL_DERIVE" == "0" ]]; then
            echo 'is_new_version=yes' >> $GITHUB_OUTPUT
          else
            # Unexpected outcome, indicates a bug.
            exit 1
          fi

      - uses: Swatinem/rust-cache@v2
        if: steps.version.outputs.is_new_version == 'yes'

      - name: feeds example
        if: steps.version.outputs.is_new_version == 'yes'
        run: |
          cd trustfall
          cargo run --example feeds refresh
          cargo run --example feeds query ./examples/feeds/example_queries/feed_content.ron
          cargo run --example feeds query ./examples/feeds/example_queries/feed_links.ron
          cargo run --example feeds query ./examples/feeds/example_queries/game_reviews.ron

      - name: hackernews example
        if: steps.version.outputs.is_new_version == 'yes'
        run: |
          cd trustfall
          cargo run --example hackernews query ./examples/hackernews/example_queries/front_page_stories_with_links.ron 1
          cargo run --example hackernews query ./examples/hackernews/example_queries/latest_links_by_high_karma_users.ron 1

      - name: weather example
        if: steps.version.outputs.is_new_version == 'yes'
        run: |
          cd trustfall
          cargo run --example weather refresh
          cargo run --example weather query ./examples/weather/example_queries/boston_weather.ron
          cargo run --example weather query ./examples/weather/example_queries/high_winds.ron

      - name: trustfall_stubgen tests
        if: steps.version.outputs.is_new_version == 'yes'
        run: |
          cd trustfall_stubgen
          cargo test --all-features --all-targets

  attempt-publish-trustfall-filetests-macros:
    name: publish trustfall_filetests_macros
    if: github.ref == 'refs/heads/main'
    needs:
      - pre-publish-checks
    uses: ./.github/workflows/publish-rust.yml
    with:
      crate-name: trustfall_filetests_macros
    secrets:
      CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

  attempt-publish-trustfall-core:
    name: publish trustfall_core
    if: github.ref == 'refs/heads/main'
    needs:
      - pre-publish-checks
      - attempt-publish-trustfall-filetests-macros
    uses: ./.github/workflows/publish-rust.yml
    with:
      crate-name: trustfall_core
    secrets:
      CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

  attempt-publish-trustfall-derive:
    name: publish trustfall_derive
    if: github.ref == 'refs/heads/main'
    needs:
      - pre-publish-checks
      - attempt-publish-trustfall-core
    uses: ./.github/workflows/publish-rust.yml
    with:
      crate-name: trustfall_derive
    secrets:
      CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

  attempt-publish-trustfall:
    name: publish trustfall
    if: github.ref == 'refs/heads/main'
    needs:
      - pre-publish-checks
      - attempt-publish-trustfall-core
      - attempt-publish-trustfall-derive
    uses: ./.github/workflows/publish-rust.yml
    with:
      crate-name: trustfall
    secrets:
      CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

  attempt-publish-trustfall-stubgen:
    name: publish trustfall_stubgen
    if: github.ref == 'refs/heads/main'
    needs:
      - pre-publish-checks
      - attempt-publish-trustfall
    uses: ./.github/workflows/publish-rust.yml
    with:
      crate-name: trustfall_stubgen
    secrets:
      CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

  attempt-publish-pytrustfall:
    name: publish python trustfall
    if: github.ref == 'refs/heads/main'
    needs:
      - pre-publish-checks
      - attempt-publish-trustfall-core
    uses: ./.github/workflows/publish-python.yml