Snapdeng 34131 migrate nightly tests from cron to snapd

.github/workflows/nightly-spread.yaml

@@ -0,0 +1,69 @@
+name: Nightly spread executions
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+    - cron: '0 3 * * *'
+    - cron: '0 6 * * *'
+
+jobs:
+
+  spread-nightly:
+    if: github.event.schedule == '0 2 * * *'
+    uses: ./.github/workflows/spread-tests.yaml
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: google
+      backend: google
+      systems: 'ALL'
+      tasks: 'tests/nightly/...'
+      rules: ''
+      use-master-snap: true
+
+  spread-test-build-from-current:
+    if: github.event.schedule == '0 6 * * *'
+    uses: ./.github/workflows/spread-tests.yaml
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: ${{ matrix.group }}
+      backend: ${{ matrix.backend }}
+      systems: ${{ matrix.systems }}
+      tasks: 'tests/...'
+      rules: ''
+      use-master-snap: true
+      spread-snapd-deb-from-repo: false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - group: google
+            backend: google
+            systems: 'ubuntu-18.04-64 ubuntu-20.04-64 ubuntu-22.04-64 ubuntu-24.04-64'
+          - group: debian-not-req
+            backend: google-distro-1
+            systems: 'debian-11-64 debian-12-64 debian-sid-64'
+
+  spread-test-experimental:
+    if: github.event.schedule == '0 3 * * *'
+    uses: ./.github/workflows/spread-tests.yaml
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: 'google'
+      backend: 'google'
+      systems: 'ubuntu-18.04-64 ubuntu-20.04-64 ubuntu-21.10-64 ubuntu-core-20-64'
+      tasks: 'tests/...'
+      rules: ''
+      use-master-snap: true
+      spread-experimental-features: gate-auto-refresh-hook
+
+  spread-test-openstack:
+    if: github.event.schedule == '0 3 * * *'
+    uses: ./.github/workflows/spread-tests.yaml
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: 'openstack'
+      backend: 'openstack'
+      systems: 'ALL'
+      tasks: 'tests/main/...'
+      rules: ''
+      use-master-snap: true

.github/workflows/spread-tests.yaml

@@ -10,11 +10,11 @@ on:
         required: true
         type: string
       backend:
-        description: 'The spread backend to use (for possible values, check spread.yaml)'
+        description: 'The spread backend to use (for possible values, check spread.yaml). This cannot be empty'
         required: true
         type: string
       systems:
-        description: 'The spread system(s) to use (for possible values, check spread.yaml). If more than one, separate them with a space'
+        description: 'The spread system(s) to use (for possible values, check spread.yaml). If more than one, separate them with a space. To run all, write ALL. To run none, write NONE'
         required: true
         type: string
       tasks:
@@ -25,12 +25,42 @@ on:
         description: 'The rule .yaml file to use (found under tests/lib/spread/rules) for test discovery'
         required: true
         type: string
+      use-master-snap:
+        description: 'If true, will use the snapd snap built on master'
+        required: false
+        type: boolean
+        default: false
+      spread-snapd-deb-from-repo:
+        description: 'If true, will use the snapd package from the repository when possible'
+        required: false
+        type: boolean
+        default: true
+      spread-experimental-features:
+        description: 'Experimental snapd features to enable'
+        required: false
+        type: string
 
 
 jobs:
   run-spread:
+    env:
+      SPREAD_SNAPD_DEB_FROM_REPO: ${{ inputs.spread-snapd-deb-from-repo }}
+      SPREAD_EXPERIMENTAL_FEATURES: ${{ inputs.spread-experimental-features }}
+
     runs-on: ${{ fromJSON(inputs.runs-on) }}
     steps:
+
+    - name: Verify inputs
+      run: |
+          if [ "${{ inputs.backend }}" == "" ]; then
+            echo "You must specify a backend."
+            exit 1
+          fi
+          if [ "${{ inputs.systems }}" == "" ]; then
+            echo "You must specify a value for systems. If you want to run all, specify ALL. If you want to run none, specify NONE."
+            exit 1
+          fi
+
     - name: Cleanup job workspace
       id: cleanup-job-workspace
       run: |
@@ -101,32 +131,41 @@ jobs:
     - name: Setup run tests variable
       if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
       run: |
+          get_new_tasks() {
+            local prefix=$1
+            local changes_param=$2
+            # The tests are just filtered when the change is a PR
+            # When 'Run Nested' label is added in a PR, all the nested tests have to be executed
+            TASKS_TO_RUN=""
+            if [ -z "${{ github.event.number }}" ] || [ "$RUN_NESTED" = 'true' ]; then
+                for TASKS in ${{ inputs.tasks }}; do
+                    TASKS_TO_RUN="$TASKS_TO_RUN $prefix:$TASKS"
+                done
+            else
+                NEW_TASKS="$(./tests/lib/external/snapd-testing-tools/utils/spread-filter -r ./tests/lib/spread/rules/${{ inputs.rules }}.yaml -p "$prefix" $changes_param)"
+                TASKS_TO_RUN="$NEW_TASKS"
+            fi
+            echo "$TASKS_TO_RUN"
+          }
+
+          CHANGES_PARAM=""
+          for CHANGE in $CHANGED_FILES; do
+              CHANGES_PARAM="$CHANGES_PARAM -c $CHANGE"
+          done
           RUN_TESTS=""
-          SUGGESTED_TESTS=""
           # Save previous failed test results in FAILED_TESTS env var
           if [ -n "$FAILED_TESTS" ]; then
               RUN_TESTS="$FAILED_TESTS"
-          else
-              CHANGES_PARAM=""
-              for CHANGE in $CHANGED_FILES; do
-                  CHANGES_PARAM="$CHANGES_PARAM -c $CHANGE"
-              done
+          elif [ "${{ inputs.systems }}" == "ALL" ]; then
+              RUN_TESTS=$(get_new_tasks "${{ inputs.backend }}" "$CHANGES_PARAM")
+          elif [ "${{ inputs.systems }}" != "NONE" ]; then
               for SYSTEM in ${{ inputs.systems }}; do
-                  # Configure parameters to run tests based on current changes
-                  # The tests are just filtered when the change is a PR
-                  # When 'Run Nested' label is added in a PR, all the nested tests have to be executed
-                  if [ -z "${{ github.event.number }}" ] || [ "$RUN_NESTED" = 'true' ]; then
-                      for TESTS in ${{ inputs.tasks }}; do
-                          RUN_TESTS="$RUN_TESTS ${{ inputs.backend }}:$SYSTEM:$TESTS"
-                      done
-                  else
-                      NEW_TESTS="$(./tests/lib/external/snapd-testing-tools/utils/spread-filter -r ./tests/lib/spread/rules/${{ inputs.rules }}.yaml -p "${{ inputs.backend }}:$SYSTEM" $CHANGES_PARAM)"
-                      if [ -z "$RUN_TESTS" ]; then
-                          RUN_TESTS="$NEW_TESTS"
-                      else
-                          RUN_TESTS="$RUN_TESTS $NEW_TESTS"
-                      fi
-                  fi
+                NEW_TASKS="$(get_new_tasks "${{ inputs.backend }}:$SYSTEM" "$CHANGES_PARAM")"
+                if [ -z "$RUN_TESTS" ]; then
+                    RUN_TESTS="$NEW_TASKS"
+                else
+                    RUN_TESTS="$RUN_TESTS $NEW_TASKS"
+                fi
               done
           fi
           echo RUN_TESTS="$RUN_TESTS"  >> $GITHUB_ENV
@@ -149,7 +188,7 @@ jobs:
 
     - name: Download built snap (amd64)
       uses: actions/download-artifact@v4
-      if: "!contains(inputs.group, '-arm64') && !endsWith(inputs.group, '-fips')"
+      if: "${{ !inputs.use-master-snap && !contains(inputs.group, '-arm64') && !endsWith(inputs.group, '-fips') }}"
       with:
         name: snap-files-amd64-default-test
         # eg. snapd_1337.2.65.1+git97.gd35b459_amd64.snap
@@ -168,14 +207,15 @@ jobs:
     - name: Download built FIPS snap (amd64)
       uses: actions/download-artifact@v4
       # eg. ubuntu-fips
-      if: "!contains(inputs.group, '-arm64') && endsWith(inputs.group, '-fips')"
+      if: "${{ !inputs.use-master-snap && !contains(inputs.group, '-arm64') && endsWith(inputs.group, '-fips') }}"
       with:
         name: snap-files-amd64-FIPS-test
         # eg. snapd_1337.2.65.1+git97.gd35b459-fips_amd64.snap
         pattern: snapd_1337.*-fips_*.snap
         path: "${{ github.workspace }}/built-snap"
 
     - name: Rename imported snap
+      if: "${{ !inputs.use-master-snap }}"
       run: |
         for snap in built-snap/snapd_1337.*.snap; do
           mv -v "${snap}" "${snap}.keep"
@@ -196,29 +236,35 @@ jobs:
           fi
 
           export SPREAD_USE_PREBUILT_SNAPD_SNAP=true
+
+          if [ "${{ inputs.use-master-snap }}" = true ]; then
+            export SPREAD_USE_SNAPD_SNAP_URL=https://storage.googleapis.com/snapd-spread-tests/snapd-tests/snaps/snapd_master_amd64.snap
+          fi
 
-          if [[ "${{ inputs.systems }}" =~ amazon-linux-2023 ]]; then
-              # Amazon Linux 2023 has no xdelta, however we cannot disable
-              # xdelta on a per-target basis as it's used in the repack section
-              # of spread.yaml, which is shared by all targets, so all systems
-              # in this batch will not use delta for transferring project data
-              echo "Disabling xdelta support"
-              export NO_DELTA=1
+          # This could be the case when either there are not systems for a group or
+          # the list of tests to run is empty
+          if [ -z "$RUN_TESTS" ]; then
+            echo "No tests to run, exiting..."
+            exit 0
           fi
 
           # Add openstack backend definition to spread.yaml
           if [ "${{ inputs.backend }}" = openstack ]; then
               ./tests/lib/spread/add-backend tests/lib/spread/backend.openstack.yaml spread.yaml
           fi
 
-          # This could be the case when either there are not systems for a group or
-          # the list of tests to run is empty
-          if [ -z "$RUN_TESTS" ]; then
-            echo "No tests to run, exiting..."
-            exit 0
+          spread_list=$($SPREAD -list $RUN_TESTS 2>&1)
+
+          if [[ "$spread_list" =~ amazon-linux-2023 ]]; then
+              # Amazon Linux 2023 has no xdelta, however we cannot disable
+              # xdelta on a per-target basis as it's used in the repack section
+              # of spread.yaml, which is shared by all targets, so all systems
+              # in this batch will not use delta for transferring project data
+              echo "Disabling xdelta support"
+              export NO_DELTA=1
           fi
 
-          if "$SPREAD" -list $RUN_TESTS 2>&1 | grep -q "nothing matches provider filter"; then
+          if "$spread_list" | grep -q "nothing matches provider filter"; then
             echo "No tests to run, exiting..."
             exit 0
           fi
@@ -234,7 +280,7 @@ jobs:
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: spread-logs-${{ inputs.systems }}
+        name: "spread-logs-${{ inputs.group }}-${{ inputs.systems }}"
         path: "spread-logs/*.log"
         if-no-files-found: ignore
 

.github/workflows/test.yaml

@@ -282,7 +282,7 @@ jobs:
             rules: 'main'
           - group: ubuntu-no-lts
             backend: google
-            systems: ''
+            systems: 'NONE'
             tasks: 'tests/...'
             rules: 'main'
           - group: ubuntu-daily