ci: Build Firefox with current neqo (#1834)

* ci: Build Firefox with current neqo * --application-choice browser * Allow qlog dupes * Typo * Get Firefox source from GitHub * Again * Again * Again * Again * Again * Debug * Again * Again * Again * Again * Again * Again * qlog * Fix * Again * Again * Again * Fingers crossed... * Try and prevent running out of disk space * Again * Again * Set CARGO_HOME and use sccache * actionlint * Use matrix * Again * Again * Upload binaries * Identify what to upload * Push * Naming * Again * Again * Again * Again * Again * Don't disable tests for debug builds * Work around https://bugzilla.mozilla.org/show_bug.cgi?id=1894031 * Prepare to comment on PR * Again * Again * Undo * Again * Again * Try and comment * cargo update * Again * Again * Again * Again * Again * Consolidate PR reporting * Fix actionlint --------- Signed-off-by: Lars Eggert <[email protected]>
mozilla · May 6, 2024 · 20a4058 · 20a4058
1 parent c854000
commit 20a4058
Show file tree

Hide file tree

Showing 6 changed files with 383 additions and 36 deletions.
diff --git a/.github/actions/maximize-build-space/action.yml b/.github/actions/maximize-build-space/action.yml
@@ -0,0 +1,206 @@
+# https://github.com/easimon/maximize-build-space/blob/fadc013e293a3453768b4ddb9db8c85104752807/action.yml
+
+name: 'Maximize build disk space'
+description: 'Maximize the available disk space for your build job'
+branding:
+  icon: 'crop'
+  color: 'orange'
+inputs:
+  root-reserve-mb:
+    description: 'Space to be left free on the root filesystem, in Megabytes.'
+    required: false
+    default: '1024'
+  temp-reserve-mb:
+    description: 'Space to be left free on the temp filesystem (/mnt), in Megabytes.'
+    required: false
+    default: '100'
+  swap-size-mb:
+    description: 'Swap space to create, in Megabytes.'
+    required: false
+    default: '4096'
+  overprovision-lvm:
+    description: |
+      Create the LVM disk images as sparse files, making the space required for the LVM image files *appear* unused on the
+      hosting volumes until actually allocated. Use with care, this can lead to surprising out-of-disk-space situations.
+      You should prefer adjusting root-reserve-mb/temp-reserve-mb over using this option.
+    required: false
+    default: 'false'
+  build-mount-path:
+    description: 'Absolute path to the mount point where the build space will be available, defaults to $GITHUB_WORKSPACE if unset.'
+    required: false
+  build-mount-path-ownership:
+    description: 'Ownership of the mount point path, defaults to standard "runner" user and group.'
+    required: false
+    default: 'runner:runner'
+  pv-loop-path:
+    description: 'Absolute file path for the LVM image created on the root filesystem, the default is usually fine.'
+    required: false
+    default: '/pv.img'
+  tmp-pv-loop-path:
+    description: 'Absolute file path for the LVM image created on the temp filesystem, the default is usually fine. Must reside on /mnt'
+    required: false
+    default: '/mnt/tmp-pv.img'
+  remove-dotnet:
+    description: 'Removes .NET runtime and libraries. (frees ~17 GB)'
+    required: false
+    default: 'false'
+  remove-android:
+    description: 'Removes Android SDKs and Tools. (frees ~11 GB)'
+    required: false
+    default: 'false'
+  remove-haskell:
+    description: 'Removes GHC (Haskell) artifacts. (frees ~2.7 GB)'
+    required: false
+    default: 'false'
+  remove-codeql:
+    description: 'Removes CodeQL Action Bundles. (frees ~5.4 GB)'
+    required: false
+    default: 'false'
+  remove-docker-images:
+    description: 'Removes cached Docker images. (frees ~3 GB)'
+    required: false
+    default: 'false'
+runs:
+  using: "composite"
+  steps:
+    - name: Disk space report before modification
+      shell: bash
+      run: |
+        echo "Memory and swap:"
+        sudo free
+        echo
+        sudo swapon --show
+        echo
+
+        echo "Available storage:"
+        sudo df -h
+        echo
+
+    - name: Maximize build disk space
+      shell: bash
+      run: |
+          set -euo pipefail
+
+          BUILD_MOUNT_PATH="${{ inputs.build-mount-path }}"
+          if [[ -z "${BUILD_MOUNT_PATH}" ]]; then
+            BUILD_MOUNT_PATH="${GITHUB_WORKSPACE}"
+          fi
+
+          echo "Arguments:"
+          echo
+          echo "  Root reserve:      ${{ inputs.root-reserve-mb }} MiB"
+          echo "  Temp reserve:      ${{ inputs.temp-reserve-mb }} MiB"
+          echo "  Swap space:        ${{ inputs.swap-size-mb }} MiB"
+          echo "  Overprovision LVM: ${{ inputs.overprovision-lvm }}"
+          echo "  Mount path:        ${BUILD_MOUNT_PATH}"
+          echo "  Root PV loop path: ${{ inputs.pv-loop-path }}"
+          echo "  Temp PV loop path: ${{ inputs.tmp-pv-loop-path }}"
+          echo -n "  Removing:     "
+          if [[ ${{ inputs.remove-dotnet }} == 'true' ]]; then
+            echo -n "dotnet "
+          fi
+          if [[ ${{ inputs.remove-android }} == 'true' ]]; then
+            echo -n "android "
+          fi
+          if [[ ${{ inputs.remove-haskell }} == 'true' ]]; then
+            echo -n "haskell "
+          fi
+          if [[ ${{ inputs.remove-codeql }} == 'true' ]]; then
+            echo -n "codeql "
+          fi
+          if [[ ${{ inputs.remove-docker-images }} == 'true' ]]; then
+            echo -n "docker "
+          fi
+          echo
+
+          # store owner of $GITHUB_WORKSPACE in case the action deletes it
+          WORKSPACE_OWNER="$(stat -c '%U:%G' "${GITHUB_WORKSPACE}")"
+
+          # ensure mount path exists before the action
+          sudo mkdir -p "${BUILD_MOUNT_PATH}"
+          sudo find "${BUILD_MOUNT_PATH}" -maxdepth 0 ! -empty -exec echo 'WARNING: directory [{}] is not empty, data loss might occur. Content:' \; -exec ls -al "{}" \;
+
+          echo "Removing unwanted software... "
+          if [[ ${{ inputs.remove-dotnet }} == 'true' ]]; then
+            sudo rm -rf /usr/share/dotnet
+          fi
+          if [[ ${{ inputs.remove-android }} == 'true' ]]; then
+            sudo rm -rf /usr/local/lib/android
+          fi
+          if [[ ${{ inputs.remove-haskell }} == 'true' ]]; then
+            sudo rm -rf /opt/ghc
+          fi
+          if [[ ${{ inputs.remove-codeql }} == 'true' ]]; then
+            sudo rm -rf /opt/hostedtoolcache/CodeQL
+          fi
+          if [[ ${{ inputs.remove-docker-images }} == 'true' ]]; then
+            sudo docker image prune --all --force
+          fi
+          echo "... done"
+
+          VG_NAME=buildvg
+
+          # github runners have an active swap file in /mnt/swapfile
+          # we want to reuse the temp disk, so first unmount swap and clean the temp disk
+          echo "Unmounting and removing swap file."
+          sudo swapoff -a
+          sudo rm -f /mnt/swapfile
+
+          echo "Creating LVM Volume."
+          echo "  Creating LVM PV on root fs."
+          # create loop pv image on root fs
+          ROOT_RESERVE_KB=$(expr ${{ inputs.root-reserve-mb }} \* 1024)
+          ROOT_FREE_KB=$(df --block-size=1024 --output=avail / | tail -1)
+          ROOT_LVM_SIZE_KB=$(expr $ROOT_FREE_KB - $ROOT_RESERVE_KB)
+          ROOT_LVM_SIZE_BYTES=$(expr $ROOT_LVM_SIZE_KB \* 1024)
+          sudo touch "${{ inputs.pv-loop-path }}" && sudo fallocate -z -l "${ROOT_LVM_SIZE_BYTES}" "${{ inputs.pv-loop-path }}"
+          export ROOT_LOOP_DEV=$(sudo losetup --find --show "${{ inputs.pv-loop-path }}")
+          sudo pvcreate -f "${ROOT_LOOP_DEV}"
+
+          # create pv on temp disk
+          echo "  Creating LVM PV on temp fs."
+          TMP_RESERVE_KB=$(expr ${{ inputs.temp-reserve-mb }} \* 1024)
+          TMP_FREE_KB=$(df --block-size=1024 --output=avail /mnt | tail -1)
+          TMP_LVM_SIZE_KB=$(expr $TMP_FREE_KB - $TMP_RESERVE_KB)
+          TMP_LVM_SIZE_BYTES=$(expr $TMP_LVM_SIZE_KB \* 1024)
+          sudo touch "${{ inputs.tmp-pv-loop-path }}" && sudo fallocate -z -l "${TMP_LVM_SIZE_BYTES}" "${{ inputs.tmp-pv-loop-path }}"
+          export TMP_LOOP_DEV=$(sudo losetup --find --show "${{ inputs.tmp-pv-loop-path }}")
+          sudo pvcreate -f "${TMP_LOOP_DEV}"
+
+          # create volume group from these pvs
+          sudo vgcreate "${VG_NAME}" "${TMP_LOOP_DEV}" "${ROOT_LOOP_DEV}"
+
+          echo "Recreating swap"
+          # create and activate swap
+          sudo lvcreate -L "${{ inputs.swap-size-mb }}M" -n swap "${VG_NAME}"
+          sudo mkswap "/dev/mapper/${VG_NAME}-swap"
+          sudo swapon "/dev/mapper/${VG_NAME}-swap"
+
+          echo "Creating build volume"
+          # create and mount build volume
+          sudo lvcreate -l 100%FREE -n buildlv "${VG_NAME}"
+          if [[ ${{ inputs.overprovision-lvm }} == 'true' ]]; then
+            sudo mkfs.ext4 -m0 "/dev/mapper/${VG_NAME}-buildlv"
+          else
+            sudo mkfs.ext4 -Enodiscard -m0 "/dev/mapper/${VG_NAME}-buildlv"
+          fi
+          sudo mount "/dev/mapper/${VG_NAME}-buildlv" "${BUILD_MOUNT_PATH}"
+          sudo chown -R "${{ inputs.build-mount-path-ownership }}" "${BUILD_MOUNT_PATH}"
+
+          # if build mount path is a parent of $GITHUB_WORKSPACE, and has been deleted, recreate it
+          if [[ ! -d "${GITHUB_WORKSPACE}" ]]; then
+            sudo mkdir -p "${GITHUB_WORKSPACE}"
+            sudo chown -R "${WORKSPACE_OWNER}" "${GITHUB_WORKSPACE}"
+          fi
+
+    - name: Disk space report after modification
+      shell: bash
+      run: |
+        echo "Memory and swap:"
+        sudo free
+        echo
+        sudo swapon --show
+        echo
+
+        echo "Available storage:"
+        sudo df -h
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -244,6 +244,6 @@ jobs:
       - name: Export PR comment data
         uses: ./.github/actions/pr-comment-data-export
         with:
-          name: bench
+          name: ${{ github.workflow }}
           contents: results.md
           log-url: ${{ steps.export.outputs.artifact-url }}
diff --git a/.github/workflows/firefox.yml b/.github/workflows/firefox.yml
@@ -0,0 +1,169 @@
+name: Firefox
+on:
+  push:
+    branches: ["main"]
+    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+  pull_request:
+    branches: ["main"]
+    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+env:
+    FIREFOX: Firefox
+
+jobs:
+  firefox:
+    name: Build Firefox
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, windows-latest]
+        type: [debug, release]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    env:
+      MOZBUILD_STATE_PATH: ${{ github.workspace }}/mozbuild
+      CARGO_HOME: ${{ github.workspace }}/cargo
+
+    steps:
+      # We need to check out Neqo first, because the maximize-build-space action
+      # is vendored in.
+      - name: Check out Neqo
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
+
+      - name: Maximize build space
+        if: runner.os == 'Linux'
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 2048
+          temp-reserve-mb: 2048
+          swap-size-mb: 4096
+          remove-dotnet: true
+          remove-android: true
+          remove-haskell: true
+          remove-docker-images: true
+
+      # The previous step blew it away, so we need to check it out again.
+      - name: Check out Neqo again
+        if: runner.os == 'Linux'
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
+
+      - name: Check out Firefox
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
+        with:
+          repository: mozilla/gecko-dev
+          path: mozilla-unified
+
+      - name: Install deps (Windows)
+        if: runner.os == 'Windows'
+        run: choco install -y mozillabuild
+
+      - name: Bootstrap Firefox
+        run: |
+          cd mozilla-unified
+          {
+            echo "mk_add_options MOZ_OBJDIR=../$FIREFOX"
+            echo "ac_add_options --with-ccache=sccache"
+            echo "ac_add_options --enable-application=browser"
+            # Work around https://bugzilla.mozilla.org/show_bug.cgi?id=1894031
+            if [ "${{ runner.os }}" != "Windows" ] || [ "${{ matrix.type}}" != "debug" ]; then
+              echo "ac_add_options --disable-tests"
+            fi
+            echo "ac_add_options --enable-${{ matrix.type }}"
+          } >> mozconfig
+          ./mach bootstrap --application-choice browser
+
+      - name: Plumb in Neqo
+        run: |
+          # Get qlog version used by neqo
+          cargo generate-lockfile
+          QLOG_VERSION=$(cargo pkgid qlog | cut -d@ -f2)
+          rm Cargo.lock
+          cd mozilla-unified
+          {
+            echo '[[audits.qlog]]'
+            echo 'who = "CI"'
+            echo 'criteria = "safe-to-deploy"'
+            echo "version = \"$QLOG_VERSION\""
+          } >> supply-chain/audits.toml
+          sed -i'' -e "s/qlog =.*/qlog = \"$QLOG_VERSION\"/" netwerk/socket/neqo_glue/Cargo.toml
+          {
+            echo '[patch."https://github.com/mozilla/neqo"]'
+            echo 'neqo-http3 = { path = "../neqo-http3" }'
+            echo 'neqo-transport = { path = "../neqo-transport" }'
+            echo 'neqo-common = { path = "../neqo-common" }'
+            echo 'neqo-qpack = { path = "../neqo-qpack" }'
+            echo 'neqo-crypto = { path = "../neqo-crypto" }'
+          } >> Cargo.toml
+          cargo update neqo-http3 neqo-transport neqo-common neqo-qpack neqo-crypto
+          ./mach vendor rust --ignore-modified
+
+      - name: Build Firefox
+        env:
+          NAME: ${{ runner.os == 'macOS' && 'Nightly' || 'bin' }}
+          TYPE: ${{ runner.os == 'macOS' && matrix.type == 'debug' && 'Debug' || '' }}
+          EXT: ${{ runner.os == 'macOS' && '.app' || '' }}
+        run: |
+          cd mozilla-unified
+          ./mach build && tar -cf "../$FIREFOX.tar" -C "../$FIREFOX/dist" "$NAME$TYPE$EXT"
+          exit 0
+
+      - name: Export binary
+        id: upload
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: ${{ runner.os }}-${{ env.FIREFOX }}-${{ matrix.type }}.tgz
+          path: ${{ env.FIREFOX }}.tar
+          compression-level: 9
+
+      - run: echo "${{ steps.upload.outputs.artifact-url }}" >> artifact
+
+      - name: Export artifact URL
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: artifact-${{ runner.os }}-${{ env.FIREFOX }}-${{ matrix.type }}
+          path: artifact
+          retention-days: 1
+
+  comment:
+    name: Comment on PR
+    if: always()
+    needs: firefox
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
+
+      - uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
+        with:
+          pattern: 'artifact-*'
+          path: artifacts
+
+      - run: |
+          {
+            echo "### Firefox builds for this PR"
+            echo "The following builds are available for testing. Crossed-out builds did not succeed."
+            for os in Linux macOS Windows; do
+              echo -n "* **$os**:"
+              for type in debug release; do
+                artifact="artifacts/artifact-$os-${{ env.FIREFOX }}-$type/artifact"
+                if [ -e "$artifact" ]; then
+                  echo -n " [${type^}]($(cat $artifact))"
+                else
+                  echo -n " ~~${type^}~~"
+                fi
+              done
+              echo
+            done
+          } > comment.md
+          cat comment.md > "$GITHUB_STEP_SUMMARY"
+
+      - uses: ./.github/actions/pr-comment-data-export
+        with:
+          name: ${{ github.workflow }}
+          contents: comment.md