Parallelize CI build

CI was building packages one after another in a single VM. This is slow, has the risk of one package affecting another and makes it harder to figure out which job failed. This change adds a preparation job which builds the image and gets the list of packages and changes build job into matrix depending on the prepare job.
debian-cryptoanarchy · Dec 18, 2023 · 159d223 · 159d223
1 parent e09a6bf
commit 159d223
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 29 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,8 +18,41 @@ env:
   EXECUTE_CMD: "$PODMAN_CMD exec -u $USER_NAME -i $CONTAINER_NAME"
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+            sources: ${{ steps.list-sources.outputs.sources }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-cadr-image
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
+          lookup-only: true
+      - name: Build CADR Running Environment Image
+        if: steps.cache-cadr-image.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get upgrade podman
+          $PODMAN_CMD build -t $IMAGE_NAME .
+          $PODMAN_CMD save $IMAGE_NAME > CADR_image.tar
+      - name: Install jq
+        run: sudo apt-get install -y jq
+      - name: List sources
+        id: list-sources
+        run: echo "::set-output name=sources::$(ls pkg_specs/*.sss | sed -e 's:^pkg_specs/::' -e 's/\.sss$//' | jq -R -s -c 'split("\n")[:-1]')"
   build:
+    needs: prepare
+    if: ${{ success() }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        source: ${{ fromJson(needs.prepare.outputs.sources) }}
     steps:
       - uses: actions/checkout@v2
       - name: Setup CI Build Script
@@ -30,37 +63,36 @@ jobs:
           sudo apt-get update
           sudo chown -R $USER_NAME $BUILD_DIR
           cd $BUILD_DIR
-          MKCMD="make BUILD_DIR=${BUILD_DIR}/build"
+          MKCMD="make SOURCES=\$1 BUILD_DIR=${BUILD_DIR}/build"
           \$MKCMD build-dep
           \$MKCMD all
           EOF
           chmod +x $CI_SCRIPT
-      - name: Build CADR Running Environment Image
-        run: |
-          sudo apt-get update
-          sudo apt-get upgrade podman
-          $PODMAN_CMD build -t $IMAGE_NAME .
-          $PODMAN_CMD save $IMAGE_NAME > CADR_image.tar
-      - name: Upload the Running Environment Image to Artifact
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
+      - name: Fetch CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache/restore@v3
+        env:
+          cache-name: cache-cadr-image
         with:
-          name: CADR_image
+          # npm cache files are stored in `~/.npm` on Linux/macOS
           path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
+      - name: Load Running Environment Image
+        run: |
+          $PODMAN_CMD load < CADR_image.tar
+          mkdir build
       - name: Spawn Podman Container to Prepare Running Environment
         run: |
           eval $SPAWN_CONTAINER
       - name: Build CADR
         run: |
-          eval $EXECUTE_CMD ${BUILD_DIR}/$CI_SCRIPT
+          eval $EXECUTE_CMD bash -c "${BUILD_DIR}/$CI_SCRIPT \"${{ matrix.source }}\""
       - name: Upload Debian Packages Just Built to Artifact
         uses: actions/upload-artifact@v2
-        if: ${{ always() }}
         with:
-          name: CADR_debs
+          name: ${{ matrix.source }}
           path: build/*.deb
       - name: Check SHA256
-        if: ${{ always() }}
         run: |
           sudo chown -R $USER build
           cd build
@@ -71,17 +103,15 @@ jobs:
           done
       - name: Upload SHA256 of Debian Packages Just Built to Artifact
         uses: actions/upload-artifact@v2
-        if: ${{ always() }}
         with:
           name: CADR_debs_sha256sum
           path: build/*.deb.sha256sum
       - name: Fix the Dir Permission for Post checkout
-        if: ${{ always() }}
         run: |
           sudo chown -R $USER $PWD
 
   test:
-    needs: build
+    needs: [build, prepare]
     if: ${{ success() }}
     runs-on: ubuntu-latest
     strategy:
@@ -110,20 +140,24 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get upgrade podman
-      - name: Download Pre-built Container Image
-        uses: actions/download-artifact@v3
+      - name: Fetch CADR image cache
+        id: cache-cadr-image
+        uses: actions/cache/restore@v3
+        env:
+          cache-name: cache-cadr-image
         with:
-          name: CADR_image
-          path: .
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: CADR_image.tar
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('Dockerfile', 'debcrafter-version', 'tests/data/microsoft_apt.list', 'tests/data/microsoft_key.gpg') }}
       - name: Load Running Environment Image
         run: |
           $PODMAN_CMD load < CADR_image.tar
           mkdir build
       - name: Download Pre-built Debian Packages
-        uses: actions/download-artifact@v3
+        uses: thecodenebula/download-artifact@e9e49e9bbce8ff2b901957ee034714cab099644a
         with:
-          name: CADR_debs
-          path: build
+          names: ${{ fromJson(needs.prepare.outputs.sources) }}
+          paths: build
       - name: Test CADR Basic
         run: |
           eval $SPAWN_CONTAINER
@@ -134,6 +168,5 @@ jobs:
           eval $SPAWN_CONTAINER
           eval $EXECUTE_CMD bash -c "\"$TEST_PREFIX SPLIT_STRATEGY=upgrade test-here-upgrade-${{ matrix.package }}\""
       - name: Fix the Dir Permission for Post checkout
-        if: ${{ always() }}
         run: |
           sudo chown -R $USER $PWD
diff --git a/Dockerfile b/Dockerfile
@@ -8,6 +8,7 @@ RUN echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf.d/90assumeyes
 
 COPY tests/data/microsoft_key.gpg /tmp/
 COPY tests/data/microsoft_apt.list /tmp/
+COPY debcrafter-version /tmp/
 
 RUN apt-get update && apt-get dist-upgrade && \
     apt-get install apt-utils ca-certificates && \
@@ -18,7 +19,7 @@ RUN apt-get update && apt-get dist-upgrade && \
     mv /tmp/microsoft_apt.list /etc/apt/sources.list.d/microsoft.list && \
     apt-key add < /tmp/microsoft_key.gpg && \
     apt-get update && \
-    cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter && \
+    cargo install --root /usr/local --locked --git https://github.com/Kixunil/debcrafter --rev "`cat /tmp/debcrafter-version`" && \
     cargo install --root /usr/local --locked cfg_me && \
     apt-get autoremove && apt-get clean && \
     rm -rf /root/.cargo \

diff --git a/build_template.mustache b/build_template.mustache
@@ -112,13 +112,14 @@ $(BUILD_DIR)/verify-signature-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{
 	{{/verify_commit}}
 	touch $@
 
-$(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{{{file_name}}}{{/unpack}}{{#clone_url}}$(BUILD_DIR)/fetch-{{{source_name}}}.stamp{{/clone_url}} $({{{pkg_name_upper}}}_FILTERED_SHASUMS)
+$(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp: {{#unpack}}$(BUILD_DIR)/{{{file_name}}}{{/unpack}}{{#clone_url}}$(BUILD_DIR)/fetch-{{{source_name}}}.stamp{{/clone_url}}
 	{{#unpack}}
 	cd $(BUILD_DIR) && sha256sum -c $(SOURCE_DIR)/build_rules/{{{source_name}}}.pin
 	{{/unpack}}
 	{{#clone_url}}
 	test "`cat $(SOURCE_DIR)/build_rules/{{{source_name}}}.pin`" = "`cd "$({{{pkg_name_upper}}}_BUILD_DIR)" && git rev-parse HEAD`"
 	{{/clone_url}}
+	touch $@
 
 {{#unpinned}}
 $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-signature-{{{source_name}}}.stamp
@@ -127,7 +128,7 @@ $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-signature-{{{so
 $(BUILD_DIR)/verify-{{{source_name}}}.stamp: $(BUILD_DIR)/verify-pin-{{{source_name}}}.stamp
 {{/unpinned}}
 {{#unpack}}
-	tar -C $(BUILD_DIR) -x{{#compression}}{{{compression}}}{{/compression}}{{^compression}}z{{/compression}}mf $<
+	tar -C $(BUILD_DIR) -x{{#compression}}{{{compression}}}{{/compression}}{{^compression}}z{{/compression}}mf $(BUILD_DIR)/{{{file_name}}}
 	{{#rename}}
 	mv -T {{{rename}}} $({{{pkg_name_upper}}}_BUILD_DIR)
 	{{/rename}}

diff --git a/debcrafter-version b/debcrafter-version
@@ -0,0 +1 @@
+b42f2fad79f23a03108ac69ae3e0e6987f64a60f