Backports (stable-5.21) (#14581)

Up to LXD 6.2 + some smaller fixes.

.github/actions/disable-docker/action.yml

@@ -10,4 +10,4 @@ runs:
         set -eux
         sudo apt-get autopurge -y containerd.io moby-containerd docker docker-ce podman uidmap
         sudo ip link delete docker0
-        sudo nft flush ruleset
+        sudo nft flush ruleset || sudo iptables -I DOCKER-USER -j ACCEPT

.github/actions/download-minio/action.yml

@@ -0,0 +1,18 @@
+name: Download minio/mc
+description: Download minio/mc
+
+runs:
+  using: composite
+  steps:
+    - name: Download minio/mc
+      shell: bash
+      run: |
+        set -eux
+        mkdir -p "$(go env GOPATH)/bin"
+        # Download minio ready to include in dependencies for system tests.
+        curl -sSfL https://dl.min.io/server/minio/release/linux-amd64/minio --output "$(go env GOPATH)/bin/minio"
+        chmod +x "$(go env GOPATH)/bin/minio"
+
+        # Also grab the latest minio client to maintain compatibility with the server.
+        curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc --output "$(go env GOPATH)/bin/mc"
+        chmod +x "$(go env GOPATH)/bin/mc"

.github/actions/install-lxd-builddeps/action.yml

@@ -11,6 +11,8 @@ runs:
         sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
         sudo apt-get update
 
+        # mask services from lxc-utils (`lxc-*` tools are used in test/suites/lxc-to-lxd.sh)
+        # doing this masking before the package is installed means they won't even start
         sudo systemctl mask lxc.service lxc-net.service
 
         sudo apt-get install --no-install-recommends -y \
@@ -22,7 +24,7 @@ runs:
           libcap-dev \
           libdbus-1-dev \
           liblxc-dev \
-          lxc-templates \
+          lxc-utils \
           libseccomp-dev \
           libselinux-dev \
           libsqlite3-dev \

.github/actions/install-lxd-runtimedeps/action.yml

@@ -0,0 +1,49 @@
+name: Install LXD runtime dependencies
+description: Installs LXD runtime dependencies
+
+runs:
+  using: composite
+  steps:
+    - name: Installs LXD runtime dependencies
+      shell: bash
+      run: |
+        set -eux
+        sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
+        sudo apt-get update
+
+        # mask services from lxc-utils (`lxc-*` tools are used in test/suites/lxc-to-lxd.sh)
+        # doing this masking before the package is installed means they won't even start
+        sudo systemctl mask lxc.service lxc-net.service
+
+        sudo apt-get install --no-install-recommends -y \
+          curl \
+          git \
+          make \
+          acl \
+          attr \
+          bind9-dnsutils \
+          btrfs-progs \
+          busybox-static \
+          dnsmasq-base \
+          easy-rsa \
+          gettext \
+          jq \
+          lxc-utils \
+          lvm2 \
+          nftables \
+          quota \
+          rsync \
+          s3cmd \
+          socat \
+          sqlite3 \
+          squashfs-tools \
+          tar \
+          tcl \
+          thin-provisioning-tools \
+          uuid-runtime \
+          xfsprogs \
+          xz-utils \
+          zfsutils-linux
+
+        # reclaim some space
+        sudo apt-get clean

.github/actions/reclaim-disk-space/action.yml

@@ -9,12 +9,24 @@ runs:
       run: |
         set -eux
 
-        # Purge snaps.
-        sudo snap remove --purge $(snap list | awk '!/^Name|^core|^snapd/ {print $1}')
+        # Purge snaps, if any.
+        # The Canonical runners use aproxy for connectivity.
+        for s in $(snap list | awk '!/^(Name|core|snapd|aproxy)/ {print $1}'); do
+          sudo snap remove --purge "${s}" || true
+        done
 
         # This was inspired from https://github.com/easimon/maximize-build-space
         df -h /
 
+        # Remove leftover home directories
+        sudo rm -rf /home/linuxbrew /home/runneradmin
+
+        # Remove unneeded directories
+        sudo rm -rf /opt/google/chrome
+        sudo rm -rf /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/PyPy /opt/hostedtoolcache/Python
+        sudo rm -rf /opt/microsoft/msedge /opt/microsoft/msodbcsql* /opt/microsoft/powershell
+        sudo rm -rf /root/.sbt
+
         # dotnet
         sudo rm -rf /usr/share/dotnet
         # android

.github/actions/setup-microceph/action.yml

@@ -87,6 +87,7 @@ runs:
       run: |
           set -eux
 
+          sudo apt-get update
           sudo apt-get install --no-install-recommends -y ceph-common
           # reclaim some space
           sudo apt-get clean

.github/workflows/codeql.yml

@@ -12,6 +12,10 @@
 name: "CodeQL"
 
 on:
+  push:
+    branches:
+      - main
+      - stable-*
   pull_request:
     paths-ignore:
       - '.github/**'
@@ -47,45 +51,45 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'go' ]
+        language: ['go']
         # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
         # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
         # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88  # v3.27.5
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
 
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
 
 
-    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@f09c1c0a94de965c15400f5634aa42fac8fb8f88  # v3.27.5
+      # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-    # - run: |
-    #     echo "Run, Build Application using script"
-    #     ./location_of_script_within_repo/buildscript.sh
+      # - run: |
+      #     echo "Run, Build Application using script"
+      #     ./location_of_script_within_repo/buildscript.sh
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88  # v3.27.5
-      with:
-        category: "/language:${{matrix.language}}"
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/commits.yml

@@ -14,23 +14,23 @@ jobs:
     name: Branch target and CLA
     runs-on: ubuntu-latest
     steps:
-    - name: Check branch target
-      env:
-        TARGET: ${{ github.event.pull_request.base.ref }}
-        TITLE: ${{ github.event.pull_request.title }}
-      if: ${{ github.actor != 'dependabot[bot]' }}
-      run: |
-        set -eux
-        TARGET_FROM_PR_TITLE="$(echo "${TITLE}" | sed -n 's/.*(\(stable-[0-9]\+\.[0-9]\+\))$/\1/p')"
-        if [ -z "${TARGET_FROM_PR_TITLE}" ]; then
-          TARGET_FROM_PR_TITLE="main"
-        else
-          echo "Branch target overridden from PR title"
-        fi
-        [ "${TARGET}" = "${TARGET_FROM_PR_TITLE}" ] && exit 0
+      - name: Check branch target
+        env:
+          TARGET: ${{ github.event.pull_request.base.ref }}
+          TITLE: ${{ github.event.pull_request.title }}
+        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          set -eux
+          TARGET_FROM_PR_TITLE="$(echo "${TITLE}" | sed -n 's/.*(\(stable-[0-9]\+\.[0-9]\+\))$/\1/p')"
+          if [ -z "${TARGET_FROM_PR_TITLE}" ]; then
+            TARGET_FROM_PR_TITLE="main"
+          else
+            echo "Branch target overridden from PR title"
+          fi
+          [ "${TARGET}" = "${TARGET_FROM_PR_TITLE}" ] && exit 0
 
-        echo "Invalid branch target: ${TARGET} != ${TARGET_FROM_PR_TITLE}"
-        exit 1
+          echo "Invalid branch target: ${TARGET} != ${TARGET_FROM_PR_TITLE}"
+          exit 1
 
-    - name: Check if CLA signed
-      uses: canonical/has-signed-canonical-cla@046337b42822b7868ad62970988929c79f9c1d40  # 1.2.3
+      - name: Check if CLA signed
+        uses: canonical/has-signed-canonical-cla@046337b42822b7868ad62970988929c79f9c1d40 # 1.2.3

.github/workflows/security.yml

@@ -23,26 +23,44 @@ jobs:
     if: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && github.ref_name == 'main' && github.repository == 'canonical/lxd' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: main
 
       - name: Install Trivy
         uses: ./.github/actions/install-trivy
 
+      - name: Download Trivy DB
+        id: db_download
+        run: trivy fs --download-db-only --cache-dir /home/runner/vuln-cache
+        continue-on-error: true
+
+      - name: Use previously downloaded database
+        if: ${{ steps.db_download.outcome == 'failure' }}
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        with:
+          path: /home/runner/vuln-cache
+          key: download-failed # Use a non existing key to fallback to restore-keys
+          restore-keys: |
+            trivy-cache-
+
       - name: Run Trivy vulnerability scanner
         run: |
-          trivy fs --quiet --scanners vuln,secret,misconfig --format sarif --cache-dir /home/runner/vuln-cache \
-          --severity LOW,MEDIUM,HIGH,CRITICAL --output trivy-lxd-repo-scan-results.sarif .
+          trivy fs --skip-db-update \
+          --scanners vuln,secret,misconfig \
+          --format sarif \
+          --cache-dir /home/runner/vuln-cache \
+          --severity LOW,MEDIUM,HIGH,CRITICAL \
+          --output trivy-lxd-repo-scan-results.sarif .
 
       - name: Cache Trivy vulnerability database
-        uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
+        uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
         with:
           path: /home/runner/vuln-cache
           key: trivy-cache-${{ github.run_id }}
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88  # v3.27.5
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           sarif_file: "trivy-lxd-repo-scan-results.sarif"
           sha: ${{ github.sha }}
@@ -62,16 +80,18 @@ jobs:
           - "4.0"
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Trivy
         uses: ./.github/actions/install-trivy
 
       - name: Restore cached Trivy vulnerability database
-        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
         with:
           path: /home/runner/vuln-cache
-          key: trivy-cache-${{ github.run_id }}
+          key: download-failed # Use a non existing key to fallback to restore-keys
+          restore-keys: |
+            trivy-cache-
 
       - name: Download snap for scan
         run: |
@@ -80,8 +100,12 @@ jobs:
 
       - name: Run Trivy vulnerability scanner
         run: |
-          trivy rootfs --quiet --scanners vuln,secret,misconfig --format sarif --cache-dir /home/runner/vuln-cache \
-          --severity LOW,MEDIUM,HIGH,CRITICAL --output /home/runner/${{ matrix.version }}-stable.sarif squashfs-root
+          trivy rootfs --skip-db-update \
+          --scanners vuln,secret,misconfig \
+          --format sarif \
+          --cache-dir /home/runner/vuln-cache \
+          --severity LOW,MEDIUM,HIGH,CRITICAL \
+          --output /home/runner/${{ matrix.version }}-stable.sarif squashfs-root
 
       - name: Flag snap scanning alerts
         run: |
@@ -91,12 +115,12 @@ jobs:
 
       # Now we checkout to the branch related to the scanned snap to populate github.sha appropriately.
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ (matrix.version == 'latest' && 'main') || format('stable-{0}', matrix.version) }}
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88  # v3.27.5
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           sarif_file: /home/runner/${{ matrix.version }}-stable.sarif
           sha: ${{ github.sha }}

.github/workflows/tests-snap.yml

@@ -9,4 +9,4 @@ jobs:
   test-self-hosted-large-container:
     runs-on: [self-hosted, linux, X64, jammy, large]
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2