build.yml

name: Build
on:
  push:
    branches:
      - 'trunk'
      - 'prerelease'
      # The `**/*/` works around the fact that GitHub considers a leading `**/` as meaning "zero or more path components" where we want "one or more".
      - '**/*/branch-**'
  pull_request:
concurrency:
  # Cancel concurrent jobs on pull_request but not push, by including the run_id in the concurrency group for the latter.
  group: build-${{ github.event_name == 'push' && github.run_id || 'pr' }}-${{ github.ref }}
  cancel-in-progress: true

env:
  COMPOSER_ROOT_VERSION: "dev-trunk"

jobs:
  build:
    name: Build all projects
    runs-on: ubuntu-latest
    timeout-minutes: 30  # 2023-05-25: Build times have crept up to ~15–25+ minutes as we've added more projects, bump to 30.
    env:
      # Hard-code a specific directory to avoid paths in vendor/composer/installed.json changing every build.
      BUILD_BASE: /tmp/jetpack-build
      # This string is used as a unique identifier of test reminder comments on PRs.
      TEST_COMMENT_INDICATOR: "<!-- wpcom-reminder-comment -->"
    outputs:
      any_plugins: ${{ steps.plugins.outputs.any }}
      changed_projects: ${{ steps.changed.outputs.projects }}

    steps:
      - uses: actions/checkout@v4

      # For pull requests, list-changed-projects.sh needs the merge base.
      # But it doesn't have to be checked out.
      - name: Deepen to merge base
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/deepen-to-merge-base
        with:
          checkout: false

      - name: Setup tools
        uses: ./.github/actions/tool-setup

      - name: Pnpm install
        run: pnpm install

      - name: Detect changed projects
        id: changed
        run: |
          CHANGED="$(EXTRA=build .github/files/list-changed-projects.sh)"
          echo "projects=${CHANGED}" >> "$GITHUB_OUTPUT"

      - name: Check if a WordPress.com test reminder comment is needed.
        id: check-test-reminder-comment
        uses: actions/github-script@v7
        if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name }}
        env:
          CHANGED: ${{ steps.changed.outputs.projects }}
        with:
          script: |
            const checkTestReminderComment = require('.github/files/build-reminder-comment/check-test-reminder-comment.js')
            const data = await checkTestReminderComment( github, context, core );
            return data;

      # We need the tree (but not the blob) for packages that will have -alpha version numbers so the timestamp appending works right.
      - name: Deepen tree for packages
        env:
          CHANGED: ${{ steps.changed.outputs.projects }}
        run: |
          mapfile -t PROJECTS < <(jq -r 'to_entries[] | select( .value ) | .key' <<<"$CHANGED")
          if [[ ${#PROJECTS[@]} -gt 0 ]]; then
            depth=$( git rev-list --count --first-parent HEAD )
            [[ "$depth" -lt 1000 ]] && depth=1000
            BASE=$PWD
            REF=$(git rev-parse HEAD)
            for SLUG in $(pnpm jetpack dependencies list --add-dependencies --extra="build" --ignore-root "${PROJECTS[@]}"); do
              [[ "$SLUG" == packages/* ]] || continue
              cd "$BASE/projects/$SLUG/"
              CHANGES_DIR="$(jq -r '.extra.changelogger["changes-dir"] // "changelog"' composer.json)"
              [[ -d "$CHANGES_DIR" && -n "$(ls -- "$CHANGES_DIR")" ]] || continue
              echo "Checking depth for $SLUG"
              while git log --format='%h, %D,' -1 . | grep ', grafted,'; do
                depth=$((depth * 2))
                echo "::group::Deepen to $depth"
                echo "/usr/bin/git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=$depth --filter=blob:none origin $REF"
                /usr/bin/git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=$depth --filter=blob:none origin "$REF"
                echo "::endgroup::"
              done
            done
          fi

      - name: Build changed projects
        id: build
        env:
          CHANGED: ${{ steps.changed.outputs.projects }}
        run: |
          mapfile -t PROJECTS < <(jq -r 'to_entries[] | select( .value ) | .key' <<<"$CHANGED")
          if [[ ${#PROJECTS[@]} -eq 0 ]]; then
            echo "Nothing to build. Generating empty artifact."
            mkdir "$BUILD_BASE"
            touch "$BUILD_BASE/mirrors.txt"
          else
            pnpm jetpack build -v --no-pnpm-install --for-mirrors="$BUILD_BASE" "${PROJECTS[@]}"
          fi

      - name: Filter mirror list for release branch
        if: github.ref == 'refs/heads/prerelease' || contains( github.ref, '/branch-' )
        run: .github/files/filter-mirrors-for-release-branch.sh

      - name: Determine plugins to publish
        id: plugins
        run: |
          jq -r 'if .extra["mirror-repo"] and ( .extra["beta-plugin-slug"] // .extra["wp-plugin-slug"] ) then [ ( input_filename | sub( "/composer\\.json$"; "" ) ), .extra["mirror-repo"], .extra["beta-plugin-slug"] // .extra["wp-plugin-slug"] ] else empty end | @tsv' projects/plugins/*/composer.json | while IFS=$'\t' read -r SRC MIRROR SLUG; do
            if [[ -e "$BUILD_BASE/$MIRROR" ]] && grep -q --fixed-strings --line-regexp "$MIRROR" "$BUILD_BASE/mirrors.txt"; then
              printf '%s\t%s\t%s\n' "$SRC" "$MIRROR" "$SLUG"
            fi
          done > "$BUILD_BASE/plugins.tsv"
          if [[ -s "$BUILD_BASE/plugins.tsv" ]]; then
            cat "$BUILD_BASE/plugins.tsv"
            echo "any=true" >> "$GITHUB_OUTPUT"
          else
            echo "No plugins were built"
            echo "any=false" >> "$GITHUB_OUTPUT"
          fi

      # GitHub's artifact stuff doesn't preserve permissions or file case. Sigh.
      # This is the official workaround: https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
      # It'll also make it faster to upload and download though, so maybe it's a win anyway.
      - name: Create archive
        run: tar --owner=0 --group=0 --xz -cvvf build.tar.xz -C "$BUILD_BASE" --transform 's,^\.,build,' .

      - name: Store build as artifact
        uses: actions/upload-artifact@v4
        with:
          name: jetpack-build
          path: build.tar.xz
          # Only need to retain for a day since the beta builder slurps it up to distribute.
          retention-days: 1
          # Already compressed.
          compression-level: 0

      - name: Store plugins.tsv as artifact
        if: steps.plugins.outputs.any == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: plugins.tsv
          path: ${{ env.BUILD_BASE }}/plugins.tsv
          # We don't really care about this artifact, its presence is a flag to the post-build job.
          retention-days: 1

      - name: Update reminder with testing instructions
        id: update-reminder-comment
        uses: actions/github-script@v7
        if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name && fromJSON(steps.check-test-reminder-comment.outputs.result)['commentId'] != 0 }}
        env:
          BRANCH_NAME: ${{ github.head_ref }}
          DATA: ${{ steps.check-test-reminder-comment.outputs.result }}
        with:
          script: |
            const { BRANCH_NAME, TEST_COMMENT_INDICATOR } = process.env;
            const data = JSON.parse( process.env.DATA );
            const commands = data.projects.reduce( ( acc, cur ) => {
              return acc += `
              \`\`\`
              bin/jetpack-downloader test ${ cur } ${ BRANCH_NAME }
              \`\`\`
              `;
            }, '' );

            let jetpackMuWPcomLine = '';
            if ( data.projects.includes( 'jetpack-mu-wpcom-plugin' ) ) {
              jetpackMuWPcomLine = "  - For \`jetpack-mu-wpcom\` changes, also add \`define( 'JETPACK_MU_WPCOM_LOAD_VIA_BETA_PLUGIN', true );\` to your \`wp-config.php\` file.";
            }

            const commentBody = `${ TEST_COMMENT_INDICATOR }
            Are you an Automattician? Please test your changes on all WordPress.com environments to help mitigate accidental explosions.

            - To test on WoA, go to the Plugins menu on a WordPress.com Simple site. Click on the "Upload" button and follow the upgrade flow to be able to upload, install, and activate [the Jetpack Beta plugin](https://jetpack.com/download-jetpack-beta/). Once the plugin is active, go to Jetpack > Jetpack Beta, select your plugin, and enable the \`${ BRANCH_NAME }\` branch. 
            ${ jetpackMuWPcomLine }
            - To test on Simple, run the following command on your sandbox:
            ${ commands }

            *Interested in more tips and information?*

            - In your local development environment, use the \`jetpack rsync\` command to sync your changes to a WoA dev blog.
            - Read more about our development workflow here: PCYsg-eg0-p2
            - Figure out **when your changes will be shipped to customers** here: PCYsg-eg5-p2`;
            await github.rest.issues.updateComment( {
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: commentBody,
              comment_id: +data.commentId,
            } );

  jetpack_beta:
    name: Create artifact for Jetpack Beta plugin
    runs-on: ubuntu-latest
    needs: build
    if: needs.build.outputs.any_plugins == 'true'
    timeout-minutes: 10  # 2021-06-24: Successful runs should take just a few seconds now. But sometimes the upload is slow.
    steps:
      - uses: actions/checkout@v4
        with:
          path: monorepo

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: jetpack-build
      - name: Extract build archive
        run: tar --xz -xvvf build.tar.xz build

      - name: Prepare plugin zips
        id: prepare
        env:
          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: |
          mkdir work
          mkdir zips

          # Current version must compare greather than any previously used current version for this PR.
          # Assume GH run IDs are monotonic.
          VERSUFFIX="${GITHUB_RUN_ID}-g${SHA:0:8}"

          ANY_BUILT=false

          while IFS=$'\t' read -r SRC MIRROR SLUG; do
            echo "::group::$MIRROR (src=$SRC slug=$SLUG)"

            if [[ ! -e "build/$MIRROR" ]]; then
              echo "Plugin was not built, skipping."
              echo "::endgroup::"
              continue
            fi

            if ! grep -q --fixed-strings --line-regexp "$MIRROR" build/mirrors.txt; then
              echo "Plugin is not being mirrored in this build, skipping."
              echo "::endgroup::"
              continue
            fi

            # The Jetpack Beta Tester plugin needs the base directory name to be like "${SLUG}-dev", so copy it over.
            mv "build/$MIRROR" "work/${SLUG}-dev"

            # Copy testing docs that are not included in the mirror.
            if [[ -e "$SRC/to-test.md" ]]; then
              cp "$SRC/to-test.md" "work/${SLUG}-dev/"
            fi

            # Extract and update version.
            CURRENT_VERSION=$(monorepo/tools/plugin-version.sh "work/${SLUG}-dev/")-$VERSUFFIX
            echo "Using version $CURRENT_VERSION"
            echo "$CURRENT_VERSION" > "work/${SLUG}-dev/version.txt"
            # Don't use plugin-version.sh here, updating JETPACK__VERSION would clutter stats.
            sed -i -e 's/Version: .*$/Version: '"$CURRENT_VERSION"'/' "work/${SLUG}-dev"/*.php

            # Remove .github directory.
            rm -rf "work/${SLUG}-dev/.github"

            # Zip the plugin
            ( cd work && zip -9 -r "../zips/${SLUG}-dev.zip" "${SLUG}-dev" )

            ANY_BUILT=true

            echo "::endgroup::"
          done < build/plugins.tsv
          if ! $ANY_BUILT; then
            echo "No plugins were built"
          fi
          echo "any-built=$ANY_BUILT" >> "$GITHUB_OUTPUT"

      - name: Create plugins artifact
        uses: actions/upload-artifact@v4
        if: steps.prepare.outputs.any-built == 'true'
        with:
          name: plugins
          path: zips
          # Only need to retain for a day since the beta builder slurps it up to distribute.
          retention-days: 1
          # Already compressed.
          compression-level: 0

  update_mirrors:
    name: Push to mirror repos
    runs-on: ubuntu-latest
    needs: build

    if: github.event_name == 'push' && github.repository == 'Automattic/jetpack'

    # Not setting a job-level timeout because it would be kind of pointless with the blocking step. Set a step timeout for all other steps instead.
    steps:
      - uses: actions/checkout@v4
        with:
          path: monorepo
        timeout-minutes: 1  # 2021-01-18: Successful runs seem to take a few seconds

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: jetpack-build
        timeout-minutes: 2  # 2022-03-15: Successful runs normally take a few seconds, but on occasion they've been taking 60+ recently.
      - name: Extract build archive
        run: tar --xz -xvvf build.tar.xz build
        timeout-minutes: 1  # 2021-01-18: Successful runs seem to take a few seconds

      - name: Wait for prior instances of the workflow to finish
        uses: ./monorepo/.github/actions/turnstile

      - name: Push changed projects
        uses: ./monorepo/projects/github-actions/push-to-mirrors
        with:
          source-directory: ${{ github.workspace }}/monorepo
          token: ${{ secrets.API_TOKEN_GITHUB }}
          upstream-ref-since: '2024-04-10' # No point in checking 12 years of earlier commits from before we started adding "Upstream-Ref".
          username: matticbot
          working-directory: ${{ github.workspace }}/build
        timeout-minutes: 10  # 2024-04-11: Successful runs seem to take about a minute.