Tweak build linux image rules

- reuse mk/external.mk to verify integrity of the source - let build-system-image Makefile target can be reused in build-artifact CI - the download, extract and verify function in mk/external.mk are adjusted since git CLI command is added to pull buildroot and Linux. Note that the '*/.git/*' of a git cloned repository has to be filtered out during SHA value verification because the metadata is difference every time when cloning.
sysprog21 · Nov 11, 2024 · 64abe47 · 64abe47
1 parent 3432824
commit 64abe47
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 23 deletions.
diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml
@@ -32,8 +32,8 @@ jobs:
         uses: tj-actions/changed-files@v45
         with:
           files: |
-            .ci/buildroot_version.txt
-            .ci/linux_version.txt
+            # FIXME: detect only Linux image version changes
+            mk/external.mk
       - name: Set alias
         id: has_changed_files
         run: |
@@ -67,6 +67,7 @@ jobs:
           sudo apt-get install -q -y build-essential git
       - name: Build Linux image
         run: |
+          make build-linux-image
           make artifact ENABLE_PREBUILT=0 ENABLE_SYSTEM=1
           mkdir -p /tmp/rv32emu-linux-image-prebuilt
           mv build/Image /tmp/rv32emu-linux-image-prebuilt

diff --git a/mk/artifact.mk b/mk/artifact.mk
@@ -85,7 +85,6 @@ endif
 	fi
 else
 ifeq ($(call has, SYSTEM), 1)
-	$(Q)./tools/build-linux-image.sh
 	$(Q)($(SHA1SUM) $(BIN_DIR)/Image) >> $(BIN_DIR)/sha1sum-linux-image
 	$(Q)($(SHA1SUM) $(BIN_DIR)/rootfs.cpio) >> $(BIN_DIR)/sha1sum-linux-image
 else

diff --git a/mk/external.mk b/mk/external.mk
@@ -21,21 +21,32 @@ define epilogue
 endef
 
 # $(1): compressed source URL
+# if URL contains git command, then use git
 define download
-    $(eval _ := $(shell wget -q --show-progress --continue "$(strip $(1))"))
+    $(eval _ :=  \
+	$(if $(findstring git,$(1)), \
+            ($(eval _ := $(shell $(1)))), \
+            ($(eval _ := $(shell wget -q --show-progress --continue "$(strip $(1))"))) \
+    ))
 endef
 
 # $(1): destination directory
 # $(2): compressed source(.zip or.gz)
+# For Buildroot and Linux kernel, no need to extract
 define extract
     $(eval COMPRESSED_SUFFIX := $(suffix $(2)))
     $(eval COMPRESSED_IS_ZIP := $(filter $(COMPRESSED_SUFFIX),.zip))
-    $(eval _ :=  \
-        $(if $(COMPRESSED_IS_ZIP), \
-            ($(eval EXTRACTOR := unzip -d $(1) $(2))), \
-            ($(eval EXTRACTOR := tar -xf $(2) -C $(1))) \
-    ))
-    $(eval _ := $(shell $(EXTRACTOR)))
+    $(eval _ := \
+        $(if $(findstring git,$(2)), \
+	    (# git is used, do nothing),
+            ($(eval _ :=  \
+                $(if $(COMPRESSED_IS_ZIP), \
+                    ($(eval EXTRACTOR := unzip -d $(1) $(2))), \
+                    ($(eval EXTRACTOR := tar -xf $(2) -C $(1))) \
+            ))
+            $(eval _ := $(shell $(EXTRACTOR))))
+        ) \
+    )
 endef
 
 # $(1): correct SHA1 value
@@ -51,7 +62,7 @@ define verify
         $(if $(filter 1,$(COMPRESSED_IS_DIR)), \
             ($(eval VERIFIER :=  \
                 echo $(1) > $(SHA1_FILE1) \
-                | find $(2) -type f -print0 \
+                | find $(2) -type f -not -path '*/.git/*' -print0 \
                 | sort -z \
                 | xargs -0 $(SHA1SUM) \
                 | sort \
@@ -91,6 +102,18 @@ TIMIDITY_DATA_URL = http://www.libsdl.org/projects/mixer/timidity/timidity.tar.g
 TIMIDITY_DATA = $(OUT)/timidity
 TIMIDITY_DATA_SHA1 = cf6217a5d824b717ec4a07e15e6c129a4657ca25
 
+# Buildroot
+BUILDROOT_VERSION = 2024.05.2
+BUILDROOT_DATA = /tmp/buildroot
+BUILDROOT_DATA_URL = git clone https://github.com/buildroot/buildroot $(BUILDROOT_DATA) -b $(BUILDROOT_VERSION) --depth=1
+BUILDROOT_DATA_SHA1 = e6677825313cdb47f6e62e4dc09a22ae30699b69
+
+# Linux kernel
+LINUX_VERSION = linux-6.1.y
+LINUX_DATA = /tmp/linux
+LINUX_DATA_URL = git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git $(LINUX_DATA) -b $(LINUX_VERSION) --depth=1
+LINUX_DATA_SHA1 = 30e95c62e8264aba948bb52d75af57d435bb6b8e
+
 define download-extract-verify
 $($(T)_DATA):
 	$(Q)$$(call prologue,$$@)
@@ -100,5 +123,5 @@ $($(T)_DATA):
 	$(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA1_FILE1),$(SHA1_FILE2))
 endef
 
-EXTERNAL_DATA = DOOM QUAKE TIMIDITY
+EXTERNAL_DATA = DOOM QUAKE TIMIDITY BUILDROOT LINUX
 $(foreach T,$(EXTERNAL_DATA),$(eval $(download-extract-verify)))
diff --git a/mk/tools.mk b/mk/tools.mk
@@ -26,5 +26,7 @@ $(HIST_BIN): $(HIST_OBJS)
 TOOLS_BIN += $(HIST_BIN)
 
 # Build Linux image
-build-linux-img:
-	tools/build-linux-image.sh
+LINUX_IMAGE_SRC = $(BUILDROOT_DATA) $(LINUX_DATA)
+build-linux-image: $(LINUX_IMAGE_SRC)
+	$(Q)./tools/build-linux-image.sh
+	$(Q)$(PRINTF) "Build done."
diff --git a/tools/build-linux-image.sh b/tools/build-linux-image.sh
@@ -17,36 +17,36 @@ function OK
     printf " [ ${PASS_COLOR} OK ${NO_COLOR} ]\n"
 }
 
+SRC_DIR=/tmp
+
 PARALLEL="-j$(nproc)"
 
 function do_buildroot
 {
-    ASSERT git clone https://github.com/buildroot/buildroot -b 2024.05.2 --depth=1
-    cp -f assets/system/configs/buildroot.config buildroot/.config
-    cp -f assets/system/configs/busybox.config buildroot/busybox.config
+    cp -f assets/system/configs/buildroot.config $SRC_DIR/buildroot/.config
+    cp -f assets/system/configs/busybox.config $SRC_DIR/buildroot/busybox.config
     # Otherwise, the error below raises:
     #   You seem to have the current working directory in your
     #   LD_LIBRARY_PATH environment variable. This doesn't work.
     unset LD_LIBRARY_PATH
-    pushd buildroot
+    pushd $SRC_DIR/buildroot
     ASSERT make olddefconfig
     ASSERT make $PARALLEL
     popd
-    cp -f buildroot/output/images/rootfs.cpio ./build/
+    cp -f $SRC_DIR/buildroot/output/images/rootfs.cpio ./build/
 }
 
 function do_linux
 {
-    ASSERT git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git -b linux-6.1.y --depth=1
-    cp -f assets/system/configs/linux.config linux/.config
-    export PATH="$PWD/buildroot/output/host/bin:$PATH"
+    cp -f assets/system/configs/linux.config $SRC_DIR/linux/.config
+    export PATH="$SRC_DIR/buildroot/output/host/bin:$PATH"
     export CROSS_COMPILE=riscv32-buildroot-linux-gnu-
     export ARCH=riscv
     pushd linux
     ASSERT make olddefconfig
     ASSERT make $PARALLEL
     popd
-    cp -f linux/arch/riscv/boot/Image ./build/
+    cp -f $SRC_DIR/linux/arch/riscv/boot/Image ./build/
 }
 
 do_buildroot && OK