Merge branch 'main' into fix-unsquashfs-bug-data-queue-too-large

.github/ISSUE_TEMPLATE/bug.yaml

@@ -45,7 +45,7 @@ body:
         If possible, please paste your snapcraft.yaml contents. This
         will be automatically formatted into code, so no need for
         backticks.
-      render: shell
+      render: yaml
     validations:
       required: true
   - type: textarea

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
-- [ ] Have you followed the [guidelines for contributing](https://github.com/snapcore/snapcraft/blob/master/CONTRIBUTING.md)?
+- [ ] Have you followed the [guidelines for contributing](https://github.com/canonical/snapcraft/blob/main/CONTRIBUTING.md)?
 - [ ] Have you signed the [CLA](http://www.ubuntu.com/legal/contributors/)?
 - [ ] Have you successfully run `tox run -m lint`?
 - [ ] Have you successfully run `tox run -e test-py310`? (supported versions: `py39`, `py310`, `py311`, `py312`)

.github/renovate.json5

@@ -1,45 +1,94 @@
 {
   // Configuration file for RenovateBot: https://docs.renovatebot.com/configuration-options
-  extends: ["config:base"],
+  extends: ["config:recommended", ":semanticCommitTypeAll(build)"],
   labels: ["dependencies"],  // For convenient searching in GitHub
+  baseBranches: ["$default", "/^hotfix\\/.*/"],
   pip_requirements: {
-    fileMatch: ["^tox.ini$", "(^|/)requirements([\\w-]*)\\.txt$"]
+    fileMatch: ["^tox.ini$", "(^|/)requirements([\\w-]*)\\.txt$", "^.pre-commit-config.yaml$"]
   },
    packageRules: [
+    {
+      // Internal package minor patch updates get top priority, with auto-merging
+      groupName: "internal package minor releases",
+      matchPackagePatterns: ["^craft-.*"],
+      matchUpdateTypes: ["minor", "patch", "pin", "digest"],
+      prPriority: 10,
+      automerge: true,
+      minimumReleaseAge: "0 seconds",
+      schedule: ["at any time"],
+      matchBaseBranches: ["$default"],  // Only do minor releases on main
+    },
+    {
+      // Same as above, but for hotfix branches, only for patch, and without auto-merging.
+      groupName: "internal package patch releases (hotfix)",
+      matchPackagePatterns: ["^craft-.*"],
+      matchUpdateTypes: ["patch", "pin", "digest"],
+      prPriority: 10,
+      minimumReleaseAge: "0 seconds",
+      schedule: ["at any time"],
+      matchBaseBranches: ["/^hotfix\\/.*/"],  // All hotfix branches
+    },
     {
       // Automerge patches, pin changes and digest changes.
       // Also groups these changes together.
       groupName: "bugfixes",
-      excludePackagePrefixes: ["dev", "lint", "types"],
+      excludeDepPatterns: ["lint/.*", "types/.*"],
       matchUpdateTypes: ["patch", "pin", "digest"],
       prPriority: 3, // Patches should go first!
       automerge: true
     },
     {
       // Update all internal packages in one higher-priority PR
       groupName: "internal packages",
-      matchPackagePrefixes: ["craft-", "snap-"],
-      matchLanguages: ["python"],
-      prPriority: 2
+      matchDepPatterns: ["craft-.*", "snap-.*"],
+      matchCategories: ["python"],
+      prPriority: 2,
+      matchBaseBranches: ["$default"],  // Not for hotfix branches
     },
     {
-      // GitHub Actions are higher priority to update than most dependencies.
+      // GitHub Actions are higher priority to update than most dependencies since they don't tend to break things.
       groupName: "GitHub Actions",
       matchManagers: ["github-actions"],
       prPriority: 1,
       automerge: true,
     },
     // Everything not in one of these rules gets priority 0 and falls here.
+    {
+      //Do all pydantic-related updates together
+      groupName: "pydantic etc.",
+      matchPackagePatterns: ["^pydantic"],
+    },
     {
       // Minor changes can be grouped and automerged for dev dependencies, but are also deprioritised.
       groupName: "development dependencies (non-major)",
       groupSlug: "dev-dependencies",
-      matchPackagePrefixes: [
-        "dev",
-        "lint",
-        "types"
+      matchDepPatterns: [
+        "dev/.*",
+        "lint/.*",
+        "types/.*"
+      ],
+      matchPackagePatterns: [
+        // Brought from charmcraft. May not be complete.
+        // This helps group dependencies in requirements-dev.txt files.
+        "^(.*/)?autoflake$",
+        "^(.*/)?black$",
+        "^(.*/)?codespell$",
+        "^(.*/)?coverage$",
+        "^(.*/)?flake8$",
+        "^(.*/)?hypothesis$",
+        "^(.*/)?mypy$",
+        "^(.*/)?pycodestyle$",
+        "^(.*/)?docstyle$",
+        "^(.*/)?pyfakefs$",
+        "^(.*/)?pyflakes$",
+        "^(.*/)?pylint$",
+        "^(.*/)?pytest",
+        "^(.*/)?responses$",
+        "^(.*/)?ruff$",
+        "^(.*/)?twine$",
+        "^(.*/)?tox$",
+        "^(.*/)?types-",
       ],
-      excludePackagePatterns: ["ruff"],
       matchUpdateTypes: ["minor", "patch", "pin", "digest"],
       prPriority: -1,
       automerge: true
@@ -48,34 +97,39 @@
       // Documentation related updates
       groupName: "documentation dependencies",
       groupSlug: "doc-dependencies",
-      matchPackageNames: ["Sphinx"],
-      matchPackagePatterns: ["^[Ss]phinx.*$", "^furo$"],
-      matchPackagePrefixes: ["docs"],
+      matchPackageNames: ["Sphinx", "furo"],
+      matchPackagePatterns: ["[Ss]phinx.*$"],
+      matchDepPatterns: ["docs/.*"],
+      matchBaseBranches: ["$default"],  // Not for hotfix branches
     },
     {
       // Other major dependencies get deprioritised below minor dev dependencies.
       matchUpdateTypes: ["major"],
-      prPriority: -2
+      prPriority: -2,
+      matchBaseBranches: ["$default"],  // Not for hotfix branches
     },
     {
       // Major dev dependencies are stone last, but grouped.
       groupName: "development dependencies (major versions)",
       groupSlug: "dev-dependencies",
       matchDepTypes: ["devDependencies"],
       matchUpdateTypes: ["major"],
-      prPriority: -3
+      prPriority: -3,
+      matchBaseBranches: ["$default"],  // Not for hotfix branches
     },
     {
-      // Ruff is still unstable, so update it separately.
-      groupName: "ruff",
-      matchPackagePatterns: ["^(lint/)?ruff$"],
-      prPriority: -3
+      // Pyright makes regular breaking changes in patch releases, so we separate these
+      // and do them independently.
+      matchPackageNames: ["pyright", "types/pyright"],
+      prPriority: -4,
+      matchBaseBranches: ["$default"],  // Not for hotfix branches
     }
   ],
-  regexManagers: [
+  customManagers: [
     {
       // tox.ini can get updates too if we specify for each package.
       fileMatch: ["tox.ini"],
+      customType: "regex",
       depTypeTemplate: "devDependencies",
       matchStrings: [
         "# renovate: datasource=(?<datasource>\\S+)\n\\s+(?<depName>.*?)(\\[[\\w]*\\])*[=><]=?(?<currentValue>.*?)\n"
@@ -84,18 +138,22 @@
     {
       // .pre-commit-config.yaml version updates
       fileMatch: [".pre-commit-config.yaml"],
-      depTypeTemplate: "devDependencies",
+      customType: "regex",
+      datasourceTemplate: "pypi",
+      depTypeTemplate: "lint",
       matchStrings: [
-        "# renovate: datasource=(?<datasource>\\S+);\\s*depName=(?<depName>.*?)\n\s+rev: \"v?(?<currentValue>.*?)\""
+        "- repo: .*/<(?<depName>\\S+)\\s*\\n\\s*rev:\s+\"?v?(?<currentValue>\\S*)\"?",
       ]
     }
   ],
   timezone: "Etc/UTC",
-  automergeSchedule: ["every weekend"],
   schedule: ["every weekend"],
   prConcurrentLimit: 2, // No more than 2 open PRs at a time.
+  branchConcurrentLimit: 20, // No more than 20 open branches at a time.
   prCreation: "not-pending", // Wait until status checks have completed before raising the PR
   prNotPendingHours: 4, // ...unless the status checks have been running for 4+ hours.
   prHourlyLimit: 1, // No more than 1 PR per hour.
-  stabilityDays: 2 // Wait 2 days from release before updating.
-}
+  minimumReleaseAge: "2 days",
+  automergeStrategy: "squash", // Squash & rebase when auto-merging.
+  semanticCommitType: "build"  // use `build` as commit header type (i.e. `build(deps): <description>`)
+}

.github/workflows/check-renovate.yaml

@@ -0,0 +1,40 @@
+name: Renovate check
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/check-renovate.yaml"
+      - ".github/renovate.json5"
+
+  # Allows triggering the workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      enable_ssh_access:
+        type: boolean
+        description: 'Enable ssh access'
+        required: false
+        default: false
+
+jobs:
+  renovate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: Install renovate
+        run: npm install --global renovate
+      - name: Enable ssh access
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ inputs.enable_ssh_access }}
+        with:
+          limit-access-to-actor: true
+      - name: Check renovate config
+        run: renovate-config-validator .github/renovate.json5
+      - name: Renovate dry-run
+        run: renovate --dry-run --autodiscover
+        env:
+          RENOVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RENOVATE_USE_BASE_BRANCH_CONFIG: ${{ github.ref }}

.github/workflows/cla-check.yaml

@@ -3,7 +3,7 @@ on: [pull_request]
 
 jobs:
   cla-check:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Check if CLA signed
         uses: canonical/has-signed-canonical-cla@v1

.github/workflows/publish.yaml

@@ -5,6 +5,8 @@ on:
   push:
     branches:
       - "feature/**"
+  # allow manual re-publishing as branches expire after 30 days
+  workflow_dispatch:
 
 jobs:
   publish:
@@ -29,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - if: steps.decisions.outputs.PUBLISH == 'true'
-        uses: snapcore/action-build@v1
+        uses: canonical/action-build@v1
         name: Build Snapcraft Snap
         id: build
         with:
@@ -54,7 +56,7 @@ jobs:
 
       - name: Publish feature branch to edge/${{ steps.vars.outputs.branch }}
         if: steps.decisions.outputs.PUBLISH == 'true'
-        uses: snapcore/action-publish@v1
+        uses: canonical/action-publish@v1
         env:
           SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_STORE_CREDENTIALS }}
         with:

.github/workflows/release-drafter.yml

@@ -11,6 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Release Drafter
-        uses: release-drafter/release-drafter@v5.7.0
+        uses: release-drafter/release-drafter@v6.0.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/security-scan.yaml

@@ -0,0 +1,17 @@
+name: Security scan
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - hotfix/*
+      - work/secscan  # For development
+
+jobs:
+  python-scans:
+    name: Scan Python project
+    uses: canonical/starflow/.github/workflows/scan-python.yaml@main
+    with:
+      packages: python-apt-dev
+      osv-extra-args: '--config=source/osv-scanner.toml'
+      trivy-extra-args: '--severity HIGH,CRITICAL --ignore-unfixed --skip-dirs "tests/spread/**"'

.github/workflows/spread-scheduled.yaml

@@ -14,7 +14,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Build snap
-        uses: snapcore/action-build@v1
+        uses: canonical/action-build@v1
         id: snapcraft
       - name: Upload snap artifact
         uses: actions/upload-artifact@v4

.github/workflows/spread.yml

@@ -14,7 +14,7 @@ jobs:
 
       - name: Build snapcraft snap
         id: build-snapcraft
-        uses: snapcore/action-build@v1
+        uses: canonical/action-build@v1
 
       - name: Upload snapcraft snap
         uses: actions/upload-artifact@v4
@@ -110,10 +110,12 @@ jobs:
         name: Run spread
         env:
           SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
+          SNAPCRAFT_ASSERTION_KEY: "${{ secrets.SNAPCRAFT_ASSERTION_KEY }}"
           SNAPCRAFT_STORE_CREDENTIALS_STAGING: "${{ secrets.SNAPCRAFT_STORE_CREDENTIALS_STAGING }}"
           SNAPCRAFT_STORE_CREDENTIALS_STAGING_CANDID: "${{ secrets.SNAPCRAFT_STORE_CREDENTIALS_STAGING_CANDID }}"
           SNAPCRAFT_STORE_CREDENTIALS_STAGING_LEGACY: "${{ secrets.SNAPCRAFT_STORE_CREDENTIALS_STAGING_LEGACY }}"
-        run: spread google:ubuntu-22.04-64:tests/spread/general/store
+        run: |
+          spread google:ubuntu-22.04-64:tests/spread/store/
 
       - name: Discard spread workers
         if: always()

.github/workflows/tics.yaml

@@ -29,13 +29,15 @@ jobs:
           echo "::endgroup::"
 
       - name: Setup Tox environment
-        run: tox run-parallel --parallel auto --parallel-no-spinner --parallel-live --colored yes -e test-all-py310 --notest
+        run: tox --workdir /tmp/tox run-parallel --parallel auto --parallel-no-spinner --parallel-live --colored yes -e test-all-py310 --notest
 
       - name: Test with tox
-        run: tox run --skip-pkg-install --result-json results/tox-py310.json --colored yes -e test-all-py310
+        run: tox --workdir /tmp/tox run --skip-pkg-install --result-json results/tox-py310.json --colored yes -e test-all-py310
 
       - name: Run TICS analysis
         uses: tiobe/tics-github-action@v3
+        env:
+          PATH: "/tmp/tox/test-all-py310/bin:/snap/bin:/home/runner/.local/bin:/home/runner/.cargo/bin:/bin:/usr/bin:/usr/local/bin:"
         with:
           mode: qserver
           project: snapcraft

.github/workflows/tox.yaml

@@ -66,7 +66,7 @@ jobs:
       - name: Test with tox
         run: tox run --skip-pkg-install --result-json results/tox-ubuntu-22.04.json --colored yes -e test-py310,test-legacy-py310
       - name: Upload code coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           directory: ./results/
           files: coverage*.xml

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "tests/spread/tools/snapd-testing-tools"]
 	path = tests/spread/tools/snapd-testing-tools
-	url = https://github.com/snapcore/snapd-testing-tools.git
+	url = https://github.com/canonical/snapd-testing-tools.git
 [submodule "docs/sphinx-resources"]
 	path = docs/sphinx-resources
 	url = https://github.com/canonical/sphinx-docs-starter-pack.git

.readthedocs.yaml

@@ -16,19 +16,17 @@ build:
     - libapt-pkg-dev
   jobs:
     post_checkout:
-      - git fetch --tags --depth 1 # Also fetch tags
-      - git describe               # Useful for debugging
+      - git fetch --tags --unshallow  # Also fetch tags
+      - git describe                  # Make sure we get a proper version
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
-  builder: html
+  builder: dirhtml
   fail_on_warning: true
 
 python:
   install:
-    - requirements: docs/requirements.txt
+    - requirements: requirements-docs.txt
     - method: pip
       path: .
-      extra_requirements:
-        - docs