feat!: add support for building rootfs.ext2

cartesi · Dec 13, 2023 · b21f097 · b21f097
1 parent 0177e8e
commit b21f097
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 162 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1,6 @@
 machine-emulator-tools-*.ext2
-example/
+rootfs*
+.clang-format
+.github
+.git
+*.md
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -7,9 +7,24 @@ jobs:
   tools:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
+      - name: Install libarchive-tools
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends build-essential autoconf automake libarchive-dev libarchive-tools
+
+      # Building from source cause the provided debian package is for Debian Bookworm
+      - name: Download, build and install xgenext2fs
+        run: |
+          wget https://github.com/cartesi/genext2fs/archive/refs/tags/v1.5.3.tar.gz
+          echo "af62ec69916ffc15fa5921b19630c9935e69a0d8b1bf529d7f26cefdc23a5120  v1.5.3.tar.gz" | sha256sum -c -
+          tar -xzf v1.5.3.tar.gz
+          cd genext2fs-1.5.3
+          ./autogen.sh
+          ./configure
+          make
+          sudo make install
+          rm -rf genext2fs-1.5.3 v1.5.3.tar.gz
 
       - name: Login to GHCR
         uses: docker/login-action@v2
@@ -24,74 +39,65 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
       - name: Export makefile variables
         run: make env >> $GITHUB_ENV
 
-      - name: Download and check dependencies
-        run: make checksum
+      - name: Create debian package control file
+        run: make control
 
       - name: Buildx setup
         uses: docker/setup-buildx-action@v2
 
-      - name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ secrets.DOCKER_ORGANIZATION }}/machine-emulator-tools
-          tags: |
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Build [${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}]
+      - name: Build [${{ env.TOOLS_DEB }}]
         id: docker_build
         uses: docker/build-push-action@v4
         with:
           context: .
           builder: ${{ steps.buildx.outputs.name }}
-          tags: ${{ steps.docker_meta.outputs.tags }}
+          tags: ${{ env.TOOLS_IMAGE }}
           push: false
           load: true
           build-args: |
-            MACHINE_EMULATOR_TOOLS_TAR_GZ=${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}
-            MACHINE_EMULATOR_TOOLS_DEB=${{ env.MACHINE_EMULATOR_TOOLS_DEB }}
-            MACHINE_EMULATOR_TOOLS_VERSION=${{ env.MACHINE_EMULATOR_TOOLS_VERSION }}
+            TOOLS_DEB=${{ env.TOOLS_DEB }}
             LINUX_SOURCES_VERSION=${{ env.LINUX_SOURCES_VERSION }}
-            LINUX_SOURCES_FILEPATH=${{ env.LINUX_SOURCES_FILEPATH }}
-            RNDADDENTROPY_VERSION=${{ env.RNDADDENTROPY_VERSION }}
-            RNDADDENTROPY_FILEPATH=${{ env.RNDADDENTROPY_FILEPATH }}
+            LINUX_SOURCES_URLPATH=${{ env.LINUX_SOURCES_URLPATH }}
           cache-from: type=gha,scope=regular,mode=max
           cache-to: type=gha,scope=regular
 
       - name: Retrieve artifacts
-        run: make copy MACHINE_EMULATOR_TOOLS_IMAGE=`echo "${{ steps.docker_meta.outputs.tags }}" | head -1 | cut -d "," -f 1 | xargs`
+        run: make copy
+
+      - name: Build rootfs
+        run: make fs
 
-      - name: Upload [${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}]
+      - name: Upload [${{ env.TOOLS_DEB }}]
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}
-          path: ${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}
+          path: ${{ env.TOOLS_DEB }}
 
-      - name: Upload [${{ env.MACHINE_EMULATOR_TOOLS_DEB }}]
+      - name: Upload [${{ env.TOOLS_ROOTFS }}]
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.MACHINE_EMULATOR_TOOLS_DEB }}
-          path: ${{ env.MACHINE_EMULATOR_TOOLS_DEB }}
+          path: ${{ env.TOOLS_ROOTFS }}
 
       - name: Checksum artifacts
         run: |
-          sha512sum ${{ env.MACHINE_EMULATOR_TOOLS_DEB }} > ${{ env.MACHINE_EMULATOR_TOOLS_DEB }}.sha512
-          sha512sum ${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }} > ${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}.sha512
+          sha512sum ${{ env.TOOLS_DEB }} > ${{ env.TOOLS_DEB }}.sha512
+          sha512sum ${{ env.TOOLS_ROOTFS }} > ${{ env.TOOLS_ROOTFS }}.sha512
 
       - uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/v')
         with:
           prerelease: true
           files: |
-            ${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}
-            ${{ env.MACHINE_EMULATOR_TOOLS_TAR_GZ }}.sha512
-            ${{ env.MACHINE_EMULATOR_TOOLS_DEB }}
-            ${{ env.MACHINE_EMULATOR_TOOLS_DEB }}.sha512
+            ${{ env.TOOLS_DEB }}
+            ${{ env.TOOLS_DEB }}.sha512
+            ${{ env.TOOLS_ROOTFS }}
+            ${{ env.TOOLS_ROOTFS }}.sha512
 
   build:
     runs-on: ubuntu-22.04

diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,7 @@
 control
 machine-emulator-tools-*.tar.gz
 machine-emulator-tools-*.deb
-dep/
+rootfs.*
 
 # Prerequisites
 *.d

diff --git a/Dockerfile b/Dockerfile
@@ -21,9 +21,9 @@ ARG BUILD_BASE=/opt/cartesi
 
 # apt
 # ------------------------------------------------------------------------------
-RUN DEBIAN_FRONTEND=noninteractive apt update && \
-    apt upgrade -y && \
-    apt install -y --no-install-recommends \
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
       build-essential \
       ca-certificates \
       git \
@@ -73,7 +73,7 @@ RUN cd ${BUILD_BASE}/tools/linux/rollup/http/rollup-http-server && \
 # pack tools (deb)
 # ------------------------------------------------------------------------------
 FROM c-builder as packer
-ARG MACHINE_EMULATOR_TOOLS_DEB=machine-emulator-tools.deb
+ARG TOOLS_DEB=machine-emulator-tools.deb
 ARG STAGING_BASE=${BUILD_BASE}/_install
 ARG STAGING_DEBIAN=${STAGING_BASE}/DEBIAN
 ARG STAGING_BIN=${STAGING_BASE}/opt/cartesi/bin
@@ -90,4 +90,4 @@ COPY --from=rust-builder ${BUILD_BASE}/tools/linux/rollup/http/rollup-http-serve
 COPY skel/ ${STAGING_BASE}/
 COPY control ${STAGING_DEBIAN}/control
 
-RUN dpkg-deb -Zxz --root-owner-group --build ${STAGING_BASE} ${BUILD_BASE}/${MACHINE_EMULATOR_TOOLS_DEB}
+RUN dpkg-deb -Zxz --root-owner-group --build ${STAGING_BASE} ${BUILD_BASE}/${TOOLS_DEB}
diff --git a/Makefile b/Makefile
@@ -20,50 +20,65 @@ PATCH := 0
 LABEL :=
 VERSION := $(MAJOR).$(MINOR).$(PATCH)$(LABEL)
 
-MACHINE_EMULATOR_TOOLS_VERSION ?= v$(VERSION)
-MACHINE_EMULATOR_TOOLS_DEB     := machine-emulator-tools-$(MACHINE_EMULATOR_TOOLS_VERSION).deb
-MACHINE_EMULATOR_TOOLS_IMAGE   := cartesi/machine-emulator-tools:$(MACHINE_EMULATOR_TOOLS_VERSION)
+TOOLS_DEB     := machine-emulator-tools-v$(VERSION).deb
+TOOLS_IMAGE   := cartesi/machine-emulator-tools:$(VERSION)
+TOOLS_ROOTFS  := rootfs-tools-v$(VERSION).ext2
 
 LINUX_SOURCES_VERSION  ?= 6.5.9-ctsi-1
 LINUX_SOURCES_URLPATH  := https://github.com/cartesi/linux/archive/refs/tags/v$(LINUX_SOURCES_VERSION).tar.gz
 
-all: $(MACHINE_EMULATOR_TOOLS_DEB)
+all: $(TOOLS_DEB)
 
-build: Dockerfile control
-	docker buildx build --platform=linux/riscv64 --load \
-		--build-arg MACHINE_EMULATOR_TOOLS_DEB=$(MACHINE_EMULATOR_TOOLS_DEB) \
-		--build-arg LINUX_SOURCES_VERSION=$(LINUX_SOURCES_VERSION) \
-		--build-arg LINUX_SOURCES_URLPATH=$(LINUX_SOURCES_URLPATH) \
-		-t $(MACHINE_EMULATOR_TOOLS_IMAGE) \
-		-f $< \
-		.
+build: control
+	@if ! (docker image inspect "$(TOOLS_IMAGE)" >/dev/null 2>&1) || [[ "$(force)" == "true" ]]; then \
+		docker buildx build --platform=linux/riscv64 --load \
+			--build-arg TOOLS_DEB=$(TOOLS_DEB) \
+			--build-arg LINUX_SOURCES_VERSION=$(LINUX_SOURCES_VERSION) \
+			--build-arg LINUX_SOURCES_URLPATH=$(LINUX_SOURCES_URLPATH) \
+			-t $(TOOLS_IMAGE) \
+			-f $< \
+			. ; \
+	fi
+	@$(MAKE) copy
 
 copy:
-	ID=`docker create --platform=linux/riscv64 $(MACHINE_EMULATOR_TOOLS_IMAGE)` && \
-	   docker cp $$ID:/opt/cartesi/$(MACHINE_EMULATOR_TOOLS_DEB)    . && \
+	@ID=`docker create --platform=linux/riscv64 $(TOOLS_IMAGE)` && \
+	   docker cp $$ID:/opt/cartesi/$(TOOLS_DEB)    . && \
 	   docker rm $$ID
 
+$(TOOLS_DEB) deb: build
+
 control: Makefile control.template
-	sed 's|ARG_VERSION|$(VERSION)|g' control.template > control
+	@sed 's|ARG_VERSION|$(VERSION)|g' control.template > control
 
-$(MACHINE_EMULATOR_TOOLS_DEB): build copy
 
-fs: $(MACHINE_EMULATOR_TOOLS_DEB)
+$(TOOLS_ROOTFS) fs: $(TOOLS_DEB)
+	@docker buildx build --platform=linux/riscv64 \
+	  --build-arg TOOLS=$(TOOLS_DEB) \
+	  --output type=tar,dest=rootfs.tar \
+	  --file fs.Dockerfile \
+	  . && \
+	bsdtar -cf rootfs.gnutar --format=gnutar @rootfs.tar && \
+	xgenext2fs -fzB 4096 -b 25600 -i 4096 -a rootfs.gnutar -L rootfs $(TOOLS_ROOTFS) && \
+	rm -f rootfs.gnutar rootfs.tar
 
 env:
-	@echo MACHINE_EMULATOR_TOOLS_DEB=$(MACHINE_EMULATOR_TOOLS_DEB)
-	@echo MACHINE_EMULATOR_TOOLS_VERSION=$(MACHINE_EMULATOR_TOOLS_VERSION)
+	@echo TOOLS_DEB=$(TOOLS_DEB)
+	@echo TOOLS_ROOTFS=$(TOOLS_ROOTFS)
+	@echo TOOLS_IMAGE=$(TOOLS_IMAGE)
 	@echo LINUX_SOURCES_VERSION=$(LINUX_SOURCES_VERSION)
 	@echo LINUX_SOURCES_URLPATH=$(LINUX_SOURCES_URLPATH)
 
 setup:
-	docker run --privileged --rm  linuxkit/binfmt:bebbae0c1100ebf7bf2ad4dfb9dfd719cf0ef132
+	@docker run --privileged --rm  linuxkit/binfmt:bebbae0c1100ebf7bf2ad4dfb9dfd719cf0ef132
 
 setup-required:
 	@echo 'riscv64 buildx setup required:' `docker buildx ls | grep -q riscv64 && echo no || echo yes`
 
 help:
 	@echo 'available commands:'
+	@echo '  deb             - build machine-emulator-tools .deb package'
+	@echo '  fs              - build rootfs.ext2'
 	@echo '  setup           - setup riscv64 buildx'
 	@echo '  setup-required  - check if riscv64 buildx setup is required'
 	@echo '  help            - list makefile commands'
@@ -72,6 +87,6 @@ help:
 	@echo '  distclean       - clean and remove dependencies'
 
 clean:
-	rm -f $(MACHINE_EMULATOR_TOOLS_DEB) control
+	rm -f $(TOOLS_DEB) control rootfs*
 
-.PHONY: build copy fs env setup setup-required help distclean
+.PHONY: build fs deb env setup setup-required help distclean
diff --git a/example/README.md b/example/README.md
diff --git a/example/build b/example/build
diff --git a/example/img-pack b/example/img-pack
diff --git a/example/ubuntu-22.04.dockerfile b/example/ubuntu-22.04.dockerfile
diff --git a/fs.Dockerfile b/fs.Dockerfile
@@ -0,0 +1,14 @@
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04
+ARG TOOLS_DEB=machine-emulator-tools-v0.14.0.deb
+ADD ${TOOLS_DEB} /tmp/
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      busybox-static=1:1.30.1-7ubuntu3 \
+      ca-certificates=20230311ubuntu0.22.04.1 \
+      curl=7.81.0-1ubuntu1.14 \
+      /tmp/${TOOLS_DEB} \
+      python3 \
+      python3-requests && \
+    rm -rf /tmp/${TOOLS_DEB} && \
+    rm -rf /var/lib/apt/lists/*
+RUN useradd --create-home --user-group dapp