refactor!: simplify build system

cartesi · Jan 15, 2025 · c9ad9b0 · c9ad9b0
1 parent 3c1b337
commit c9ad9b0
Show file tree

Hide file tree

Showing 16 changed files with 135 additions and 232 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,6 +1,5 @@
 machine-emulator-tools-*.ext2
 rootfs*
-libcmt*
 .clang-format
 .github
 .git

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -63,9 +63,8 @@ jobs:
           load: true
           build-args: |
             TOOLS_TARGZ=${{ env.TOOLS_TARGZ }}
-            IMAGE_KERNEL_VERSION=${{ env.IMAGE_KERNEL_VERSION }}
-            LINUX_VERSION=${{ env.LINUX_VERSION }}
             LINUX_HEADERS_URLPATH=${{ env.LINUX_HEADERS_URLPATH }}
+            LINUX_HEADERS_SHA256=${{ env.LINUX_HEADERS_SHA256 }}
           cache-from: type=gha,scope=regular,mode=max
           cache-to: type=gha,scope=regular
 

diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,12 @@ rootfs.*
 package.json
 *.bin
 
+sys-utils/hex/hex
+sys-utils/ioctl-echo-loop/ioctl-echo-loop
+sys-utils/rollup/rollup
+sys-utils/xhalt/xhalt
+sys-utils/yield/yield
+
 # Prerequisites
 *.d
 

diff --git a/Dockerfile b/Dockerfile
@@ -15,18 +15,13 @@
 #
 
 FROM ubuntu:24.04 AS tools-env
-ARG IMAGE_KERNEL_VERSION=v0.20.0
-ARG LINUX_VERSION=6.5.13-ctsi-1
-ARG LINUX_HEADERS_URLPATH=https://github.com/cartesi/image-kernel/releases/download/${IMAGE_KERNEL_VERSION}/linux-libc-dev-riscv64-cross-${LINUX_VERSION}-${IMAGE_KERNEL_VERSION}.deb
-ARG BUILD_BASE=/opt/cartesi
 
 # Install dependencies
-# ------------------------------------------------------------------------------
-ENV LINUX_HEADERS_FILEPATH=/tmp/linux-libc-dev-riscv64-cross-${LINUX_VERSION}-${IMAGE_KERNEL_VERSION}.deb
-
+ARG LINUX_HEADERS_URLPATH=
+ARG LINUX_HEADERS_SHA256=
+ADD ${LINUX_HEADERS_URLPATH} /tmp/linux-libc-dev-riscv64-cross.deb
 RUN <<EOF
 set -e
-
 export DEBIAN_FRONTEND=noninteractive
 apt-get update
 apt-get upgrade -y
@@ -37,7 +32,6 @@ apt-get install -y --no-install-recommends \
         make \
         ca-certificates \
         git \
-        wget \
         libclang-dev \
         pkg-config \
         dpkg-cross \
@@ -46,87 +40,26 @@ apt-get install -y --no-install-recommends \
         gcc-riscv64-linux-gnu \
         g++-riscv64-linux-gnu
 
-wget -O ${LINUX_HEADERS_FILEPATH} ${LINUX_HEADERS_URLPATH}
-echo "2723435e8b45d8fb7a79e9344f6dc517b3dbc08e03ac17baab311300ec475c08  ${LINUX_HEADERS_FILEPATH}" | sha256sum --check
-apt-get install -y --no-install-recommends --allow-downgrades ${LINUX_HEADERS_FILEPATH}
-
-adduser developer -u 499 --gecos ",,," --disabled-password
-mkdir -p ${BUILD_BASE}/tools && chown -R developer:developer ${BUILD_BASE}/tools
-rm -rf /var/lib/apt/lists/* ${LINUX_HEADERS_FILEPATH}
+echo "${LINUX_HEADERS_SHA256}  /tmp/linux-libc-dev-riscv64-cross.deb" | sha256sum --check
+apt-get install -y --no-install-recommends --allow-downgrades /tmp/linux-libc-dev-riscv64-cross.deb
 EOF
 
 ENV TOOLCHAIN_PREFIX="riscv64-linux-gnu-"
+ENV RUSTUP_HOME=/opt/rust
+ENV PATH="/opt/rust/toolchains/1.77-x86_64-unknown-linux-gnu/bin:${PATH}"
 
-FROM tools-env AS builder
-COPY --chown=developer:developer sys-utils/ ${BUILD_BASE}/tools/sys-utils/
-
-# build C/C++ tools
-# ------------------------------------------------------------------------------
-FROM builder AS c-builder
-ARG BUILD_BASE=/opt/cartesi
-
-USER developer
-RUN make -C ${BUILD_BASE}/tools/sys-utils -j$(nproc) all
-RUN make -C ${BUILD_BASE}/tools/sys-utils install \
-        DESTDIR=${BUILD_BASE}/tools/sys-utils_staging PREFIX=/usr
-
-# build rust tools
-# ------------------------------------------------------------------------------
-FROM c-builder AS rust-env
-ENV PATH="/home/developer/.cargo/bin:${PATH}"
-
-USER developer
+# Install rust
 RUN rustup default 1.77 && rustup target add riscv64gc-unknown-linux-gnu
 
-FROM rust-env AS rust-builder
-COPY --chown=developer:developer rollup-http/rollup-init ${BUILD_BASE}/tools/rollup-http/rollup-init
-COPY --chown=developer:developer rollup-http/rollup-http-client ${BUILD_BASE}/tools/rollup-http/rollup-http-client
-COPY --chown=developer:developer rollup-http/.cargo ${BUILD_BASE}/tools/rollup-http/.cargo
-
-# build rollup-http-server dependencies
-FROM rust-builder AS http-server-dep-builder
-COPY --chown=developer:developer rollup-http/rollup-http-server/Cargo.toml rollup-http/rollup-http-server/Cargo.lock ${BUILD_BASE}/tools/rollup-http/rollup-http-server/
-RUN cd ${BUILD_BASE}/tools/rollup-http/rollup-http-server && \
-    mkdir src/ && \
-    echo "fn main() {}" > src/main.rs && \
-    echo "pub fn dummy() {}" > src/lib.rs && \
-    cargo build --target riscv64gc-unknown-linux-gnu --release
-
-# build rollup-http-server
-FROM http-server-dep-builder AS http-server-builder
-COPY --chown=developer:developer rollup-http/rollup-http-server/build.rs ${BUILD_BASE}/tools/rollup-http/rollup-http-server/
-COPY --chown=developer:developer rollup-http/rollup-http-server/src ${BUILD_BASE}/tools/rollup-http/rollup-http-server/src
-RUN cd ${BUILD_BASE}/tools/rollup-http/rollup-http-server && touch build.rs src/* && \
-    cargo build --target riscv64gc-unknown-linux-gnu --release
-
-# build echo-dapp dependencies
-FROM rust-builder AS echo-dapp-dep-builder
-COPY --chown=developer:developer rollup-http/echo-dapp/Cargo.toml rollup-http/echo-dapp/Cargo.lock ${BUILD_BASE}/tools/rollup-http/echo-dapp/
-RUN cd ${BUILD_BASE}/tools/rollup-http/echo-dapp && \
-    mkdir src/ && echo "fn main() {}" > src/main.rs && \
-    cargo build --target riscv64gc-unknown-linux-gnu --release
-
-# build echo-dapp
-FROM echo-dapp-dep-builder AS echo-dapp-builder
-COPY --chown=developer:developer rollup-http/echo-dapp/src ${BUILD_BASE}/tools/rollup-http/echo-dapp/src
-RUN cd ${BUILD_BASE}/tools/rollup-http/echo-dapp && touch src/* && \
-    cargo build --target riscv64gc-unknown-linux-gnu --release
-
-# pack tools (deb)
+# build
 # ------------------------------------------------------------------------------
-FROM tools-env AS packer
-ARG TOOLS_TARGZ=machine-emulator-tools.tar.gz
-ARG STAGING_BASE=${BUILD_BASE}/_install
-
-COPY package.json ${STAGING_SHARE}/package.json
-COPY copyright ${STAGING_BASE}/usr/share/doc/machine-emulator-tools/copyright
-
-RUN mkdir -p ${STAGING_BASE}/usr/bin ${STAGING_BASE}/usr/bin ${STAGING_BASE}/etc && \
-    echo "cartesi-machine" > ${staging_base}/etc/hostname
+FROM tools-env AS builder
 
-COPY --from=c-builder ${BUILD_BASE}/tools/sys-utils_staging ${STAGING_BASE}
-COPY --from=rust-builder ${BUILD_BASE}/tools/rollup-http/rollup-init/rollup-init ${STAGING_BASE}/usr/bin
-COPY --from=http-server-builder ${BUILD_BASE}/tools/rollup-http/rollup-http-server/target/riscv64gc-unknown-linux-gnu/release/rollup-http-server ${STAGING_BASE}/usr/bin
-COPY --from=echo-dapp-builder ${BUILD_BASE}/tools/rollup-http/echo-dapp/target/riscv64gc-unknown-linux-gnu/release/echo-dapp ${STAGING_BASE}/usr/bin
+COPY . /work
+WORKDIR /work
 
-RUN cd ${STAGING_BASE} && tar -czf ${BUILD_BASE}/${TOOLS_TARGZ} *
+RUN make -j$(nproc) -C sys-utils
+RUN make -j$(nproc) -C rollup-http
+RUN make install DESTDIR=$(pwd)/_install PREFIX=/usr
+ARG TOOLS_TARGZ=machine-emulator-tools.tar.gz
+RUN cd _install && tar -czf /work/${TOOLS_TARGZ} *
diff --git a/Makefile b/Makefile
@@ -28,33 +28,41 @@ TOOLS_ROOTFS_IMAGE := cartesi/rootfs-tools:$(VERSION)
 IMAGE_KERNEL_VERSION ?= v0.20.0
 LINUX_VERSION ?= 6.5.13-ctsi-1
 LINUX_HEADERS_URLPATH := https://github.com/cartesi/image-kernel/releases/download/${IMAGE_KERNEL_VERSION}/linux-libc-dev-riscv64-cross-${LINUX_VERSION}-${IMAGE_KERNEL_VERSION}.deb
+LINUX_HEADERS_SHA256 := 2723435e8b45d8fb7a79e9344f6dc517b3dbc08e03ac17baab311300ec475c08
 
-all: fs
+PREFIX ?= /usr/local
+DESTDIR ?= .
+ARCH = $(shell uname -m)
 
-build: package.json
+ifeq ($(ARCH),riscv64)
+all: sys-utils rollup-http package.json ## Build all tools from a riscv64 environment
+else
+all: targz fs
+endif
+
+build: targz fs ## Build targz and fs (cross compiling with Docker)
+
+targz: ## Build tools .tar.gz (cross compiling with Docker)
 	@docker buildx build --load \
 		--build-arg TOOLS_TARGZ=$(TOOLS_TARGZ) \
-		--build-arg IMAGE_KERNEL_VERSION=$(IMAGE_KERNEL_VERSION) \
-		--build-arg LINUX_VERSION=$(LINUX_VERSION) \
 		--build-arg LINUX_HEADERS_URLPATH=$(LINUX_HEADERS_URLPATH) \
+		--build-arg LINUX_HEADERS_SHA256=$(LINUX_HEADERS_SHA256) \
 		-t $(TOOLS_IMAGE) \
 		-f Dockerfile \
 		. ;
 	$(MAKE) copy
 
 copy:
 	@ID=`docker create $(TOOLS_IMAGE)` && \
-	   docker cp $$ID:/opt/cartesi/$(TOOLS_TARGZ) . && \
+	   docker cp $$ID:/work/$(TOOLS_TARGZ) . && \
 	   docker rm $$ID
 
-$(TOOLS_TARGZ) targz: build
-
 package.json: Makefile package.json.in
 	@sed 's|ARG_VERSION|$(VERSION)|g' package.json.in > package.json
 
-fs: $(TOOLS_ROOTFS)
+fs: $(TOOLS_ROOTFS) ## Build tools rootfs image (cross compiling with Docker)
 
-$(TOOLS_ROOTFS): $(TOOLS_TARGZ)
+$(TOOLS_ROOTFS): $(TOOLS_TARGZ) fs/Dockerfile
 	@docker buildx build --platform linux/riscv64 \
 	  --build-arg TOOLS_TARGZ=$(TOOLS_TARGZ) \
 	  --output type=tar,dest=rootfs.tar \
@@ -63,15 +71,15 @@ $(TOOLS_ROOTFS): $(TOOLS_TARGZ)
 	xgenext2fs -fzB 4096 -i 4096 -r +4096 -a rootfs.tar -L rootfs $(TOOLS_ROOTFS) && \
 	rm -f rootfs.tar
 
-fs-license:
+fs-license: ## Build tools rootfs image HTML with license information
 	@docker buildx build --load --platform linux/riscv64 \
 	  --build-arg TOOLS_TARGZ=$(TOOLS_TARGZ) \
 	  -t $(TOOLS_ROOTFS_IMAGE) \
 	  --file fs/Dockerfile \
 	  .
 	TMPFILE=$$(mktemp) && (cd fs/third-party/repo-info/; ./scan-local.sh $(TOOLS_ROOTFS_IMAGE) linux/riscv64) | tee $$TMPFILE && pandoc -s -f markdown -t html5 -o $(TOOLS_ROOTFS).html $$TMPFILE && rm -f $$TMPFILE
 
-env:
+env: ## Print useful Makefile information
 	@echo VERSION=$(VERSION)
 	@echo TOOLS_TARGZ=$(TOOLS_TARGZ)
 	@echo TOOLS_ROOTFS=$(TOOLS_ROOTFS)
@@ -80,66 +88,65 @@ env:
 	@echo IMAGE_KERNEL_VERSION=$(IMAGE_KERNEL_VERSION)
 	@echo LINUX_VERSION=$(LINUX_VERSION)
 	@echo LINUX_HEADERS_URLPATH=$(LINUX_HEADERS_URLPATH)
+	@echo LINUX_HEADERS_SHA256=$(LINUX_HEADERS_SHA256)
 
-test:
+test: ## Test tools using mock builds
 	make -C sys-utils/libcmt/ test
 	cd rollup-http/rollup-http-server && \
 	MOCK_BUILD=true cargo test -- --show-output --test-threads=1
 
-setup:
+setup: ## Setup riscv64 buildx
 	@docker run --privileged --rm  linuxkit/binfmt:bebbae0c1100ebf7bf2ad4dfb9dfd719cf0ef132
 
-setup-required:
+setup-required: ## Check if riscv64 buildx setup is required
 	@echo 'riscv64 buildx setup required:' `docker buildx ls | grep -q riscv64 && echo no || echo yes`
 
-build-tools-env:
-	docker build --target tools-env -t $(TOOLS_IMAGE)-tools-env -f Dockerfile .
-
-build-rust-builder-env:
-	docker build --target rust-builder -t $(TOOLS_IMAGE)-rust-builder -f Dockerfile .
+image: ## Build tools cross compilation Docker image
+	docker build \
+		--build-arg LINUX_HEADERS_URLPATH=$(LINUX_HEADERS_URLPATH) \
+		--build-arg LINUX_HEADERS_SHA256=$(LINUX_HEADERS_SHA256) \
+		--target tools-env -t $(TOOLS_IMAGE)-tools-env -f Dockerfile .
 
-tools-env:
+shell: ## Spawn a cross compilation shell with tools Docker image
 	@docker run --hostname rust-builder -it --rm \
 		-e USER=$$(id -u -n) \
 		-e GROUP=$$(id -g -n) \
 		-e UID=$$(id -u) \
 		-e GID=$$(id -g) \
-		-v `pwd`:/opt/cartesi/machine-emulator-tools \
-		-w /opt/cartesi/machine-emulator-tools \
+		-u $$(id -u):$$(id -g) \
+		-v `pwd`:/work/machine-emulator-tools \
+		-w /work/machine-emulator-tools \
 		$(TOOLS_IMAGE)-tools-env /bin/bash
 
+sys-utils: ## Compile system utilities tools
+	@$(MAKE) -C $@
 
-rust-builder-env:
-	@docker run --hostname rust-builder -it --rm \
-		-e USER=$$(id -u -n) \
-		-e GROUP=$$(id -g -n) \
-		-e UID=$$(id -u) \
-		-e GID=$$(id -g) \
-		-v `pwd`:/opt/cartesi/machine-emulator-tools \
-		-w /opt/cartesi/machine-emulator-tools \
-		$(TOOLS_IMAGE)-rust-builder /bin/bash
+rollup-http: ## Compile rollup http tools
+	@$(MAKE) -C $@
 
-clean-image:
+install-share: package.json
+	install -Dm644 package.json $(DESTDIR)$(PREFIX)/share/package.json
+
+install: install-share ## Install all tools into ${DESTDIR}/${PREFIX}
+	@$(MAKE) -C sys-utils/libcmt install
+	@$(MAKE) -C sys-utils install
+	@$(MAKE) -C rollup-http install
+
+clean-image: ## Clean docker images
 	@(docker rmi $(TOOLS_IMAGE) > /dev/null 2>&1 || true)
 
-clean:
-	@rm -f $(TOOLS_TARGZ) rootfs*
+clean: ## Clean built files
+	@rm -f $(TOOLS_TARGZ) rootfs*.ext2
 	@$(MAKE) -C sys-utils clean
+	@$(MAKE) -C rollup-http clean
 
-distclean: clean clean-image
-
-sys-utils:
-	@$(MAKE) -C sys-utils
+distclean: clean clean-image ## Clean built files and docker images
 
-help:
-	@echo 'available commands:'
-	@echo '  targz           - build machine-emulator-tools .tar.gz'
-	@echo '  fs              - build rootfs.ext2'
-	@echo '  fs-license      - build rootfs.ext2.html with licence information'
-	@echo '  setup           - setup riscv64 buildx'
-	@echo '  setup-required  - check if riscv64 buildx setup is required'
-	@echo '  help            - list makefile commands'
-	@echo '  env             - print useful Makefile variables as a KEY=VALUE list'
-	@echo '  clean           - remove the generated artifacts'
+help: ## Show this help
+	@sed \
+		-e '/^[a-zA-Z0-9_\-]*:.*##/!d' \
+		-e 's/:.*##\s*/:/' \
+		-e 's/^\(.\+\):\(.*\)/$(shell tput setaf 6)\1$(shell tput sgr0):\2/' \
+		$(MAKEFILE_LIST) | column -c2 -t -s :
 
-.PHONY: build fs fs-license targz env setup setup-required help distclean
+.PHONY: all build targz fs fs-license env test setup setup-required image shell sys-utils rollup-http install clean-image clean distclean help
diff --git a/README.md b/README.md
@@ -1,10 +1,10 @@
 # Cartesi Machine Emulator Tools
 
-The Cartesi Machine Emulator Tools is a repository that contains a set of tools developed for the RISC-V Linux OS. It provides a RISC-V Debian package and a root filesystem for Ubuntu 22.04.
+The Cartesi Machine Emulator Tools is a repository that contains a set of tools developed for the RISC-V Linux OS. It provides a RISC-V Debian package and a root filesystem for Ubuntu 24.04.
 
 ## Getting Started
 
-Users looking to create cartesi-machine applications can use the provided Debian package and root filesystem directly from one of the prepackaged [releases](https://github.com/cartesi/machine-emulator-tools/releases), without needing to build this repository themselves.
+Users looking to create cartesi-machine applications can use the provided Debian package and root filesystem directly from one of the prepackaged releases, without needing to build this repository themselves.
 
 ### Requirements
 
@@ -20,26 +20,29 @@ To set up riscv64 buildx, use the following command:
 $ make setup
 ```
 
-### Building
+### Building (from riscv64 environment)
 
-Invoking make will build all tools and create the `machine-emulator-tools-$VERSION.tar.gz` archive along with the `rootfs-tools-$VERSION.ext2` root filesystem artifacts.
+Assuming you are inside a riscv64 environment, you can build tools and install in the system with:
 
-```bash
-$ make
+```sh
+make
+make install PREFIX=/usr/local
 ```
 
-#### Makefile targets
+### Building (cross compiling)
 
-The following commands are available as `make` targets:
+In case you need to patch and develop tools, you can cross compile using Docker with:
+
+```sh
+make build
+```
+
+This creates the `machine-emulator-tools-$VERSION.tar.gz` archive and the `rootfs-tools-$VERSION.ext2` root filesystem artifacts.
+Both should only be use for testing and development purposes, not directly in dapps.
+
+#### Makefile targets
 
-- **all**: Build Debian package and rootfs (Default)
-- **targz**: Build machine-emulator-tools.tar.gz
-- **fs**: Build rootfs.ext2
-- **setup**: Setup riscv64 buildx
-- **setup-required**: Check if riscv64 buildx setup is required
-- **help**: List Makefile commands
-- **env**: Print useful Makefile variables as a KEY=VALUE list
-- **clean**: Remove the generated artifacts
+Check all targets available with `make help`.
 
 ## Contributing