chore: change build image to debian/alpine (#14755)

databendlabs · Feb 27, 2024 · 83f294c · 83f294c
1 parent eb46c30
commit 83f294c
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 125 deletions.
diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml
@@ -27,6 +27,13 @@ runs:
   using: "composite"
   steps:
     - name: Setup Build Tool
+      if: endsWith(inputs.target, '-linux-gnu')
+      uses: ./.github/actions/setup_build_tool
+      with:
+        target: base
+        bypass_env_vars: RUSTFLAGS,RUST_LOG
+    - name: Setup Build Tool
+      if: endsWith(inputs.target, '-linux-musl')
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}

diff --git a/.github/actions/build_linux_sanitizer/action.yml b/.github/actions/build_linux_sanitizer/action.yml
@@ -16,6 +16,13 @@ runs:
   using: "composite"
   steps:
     - name: Setup Build Tool
+      if: endsWith(inputs.target, '-linux-gnu')
+      uses: ./.github/actions/setup_build_tool
+      with:
+        target: base
+        bypass_env_vars: RUSTFLAGS,RUST_LOG
+    - name: Setup Build Tool
+      if: endsWith(inputs.target, '-linux-musl')
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}

diff --git a/.github/workflows/build_tool.yml b/.github/workflows/build_tool.yml
@@ -13,48 +13,13 @@ permissions:
   contents: read
 
 jobs:
-  base:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: ./.github/actions/setup_docker
-        id: login
-        with:
-          repo: build-tool
-          ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
-          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
-          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Get rust toolchain version
-        id: toolchain
-        run: |
-          version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
-          echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
-
-      - name: Build and publish databend build base image
-        uses: docker/build-push-action@v5
-        with:
-          pull: true
-          push: true
-          tags: |
-            ${{ steps.login.outputs.dockerhub_repo }}:base
-            ${{ steps.login.outputs.dockerhub_repo }}:base-${{ steps.toolchain.outputs.TOOLCHAIN }}
-          context: ./scripts/setup
-          file: ./docker/build-tool/base/Dockerfile
-          platforms: linux/amd64,linux/arm64
-
   build:
     runs-on: ubuntu-latest
-    needs: base
     strategy:
       matrix:
-        arch:
-          - x86_64
-          - aarch64
-        libc:
-          - gnu
-          - musl
+        os:
+          - debian
+          - alpine
     steps:
       - uses: actions/checkout@v4
 
@@ -69,7 +34,6 @@ jobs:
       - name: Get rust toolchain version
         id: toolchain
         run: |
-          # version=$(taplo get -f scripts/setup/rust-toolchain.toml '.toolchain.channel')
           version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
           echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
 
@@ -79,19 +43,15 @@ jobs:
           pull: true
           push: true
           tags: |
-            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}
-            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}-${{ steps.toolchain.outputs.TOOLCHAIN }}
-            ${{ steps.login.outputs.ecr_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}
-            ${{ steps.login.outputs.ecr_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}-${{ steps.toolchain.outputs.TOOLCHAIN }}
-          context: ./docker/build-tool/${{ matrix.libc }}/
-          file: ./docker/build-tool/${{ matrix.libc }}/Dockerfile
-          build-args: |
-            ARCH=${{ matrix.arch }}
-            RUST_TOOLCHAIN=${{ steps.toolchain.outputs.TOOLCHAIN }}
+            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.os }}
+            ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.os }}-${{ steps.toolchain.outputs.TOOLCHAIN }}
+          context: ./scripts/setup
+          file: ./docker/build-tool/${{ matrix.os }}/Dockerfile
+          platforms: linux/amd64,linux/arm64
 
   dev:
     runs-on: ubuntu-latest
-    needs: base
+    needs: build
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -73,12 +73,11 @@ jobs:
           artifacts: all
 
   build_address_sanitizer:
-    runs-on: [ self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}" ]
+    runs-on: [ self-hosted, ARM64, Linux, 8c16g, "${{ inputs.runner_provider }}" ]
     strategy:
       fail-fast: false
       matrix:
         include:
-          # - { arch: x86_64, libc: gnu }
           - { arch: aarch64, libc: gnu }
     steps:
       - uses: actions/checkout@v4
@@ -97,9 +96,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # - { arch: x86_64, libc: musl }
           - { arch: aarch64, libc: musl }
-          # - { arch: aarch64, libc: gnu }
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/docker/build-tool/alpine/Dockerfile b/docker/build-tool/alpine/Dockerfile
@@ -0,0 +1,30 @@
+FROM alpine:3.19
+
+ENV PATH=/root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+ENV LANGUAGE en_US
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+RUN apk --update add --no-cache bash
+
+COPY rust-toolchain.toml /build/rust-toolchain.toml
+COPY dev_setup.sh /build/scripts/setup/dev_setup.sh
+RUN chmod +x /build/scripts/setup/dev_setup.sh && \
+    /build/scripts/setup/dev_setup.sh -yb && \
+    rm -rf /root/.cargo/git && \
+    rm -rf /root/.cargo/registry && \
+    mkdir -p /opt/rust && \
+    mv /root/.cargo /opt/rust/cargo && \
+    mv /root/.rustup /opt/rust/rustup
+
+RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v2.4.0/mold-2.4.0-$(uname -m)-linux.tar.gz && \
+    tar --strip-components=1 -xzf /tmp/mold.tar.gz -C /usr/local && \
+    rm -rf /tmp/mold.tar.gz && \
+    ln -sf /usr/local/bin/mold /usr/bin/$(uname -m)-linux-gnu-ld
+
+ENV RUSTUP_HOME /opt/rust/rustup
+ENV CARGO_HOME /opt/rust/cargo
+ENV PATH /opt/rust/cargo/bin:/opt/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+VOLUME [ "/workspace", "/opt/rust/cargo/registry", "/opt/rust/cargo/git" ]
diff --git a/docker/build-tool/base/Dockerfile → docker/build-tool/debian/Dockerfile b/docker/build-tool/base/Dockerfile → docker/build-tool/debian/Dockerfile
diff --git a/docker/build-tool/gnu/Dockerfile b/docker/build-tool/gnu/Dockerfile
diff --git a/docker/build-tool/musl/Dockerfile b/docker/build-tool/musl/Dockerfile
diff --git a/docker/build-tool/musl/linker.sh b/docker/build-tool/musl/linker.sh
diff --git a/scripts/setup/dev_setup.sh b/scripts/setup/dev_setup.sh
@@ -120,6 +120,9 @@ function install_ziglang {
 	brew)
 		install_pkg zig "$PACKAGE_MANAGER"
 		;;
+	apk)
+		echo "TODO: install ziglang for alpine"
+		;;
 	*)
 		echo "Unable to install ziglang with package manager: $PACKAGE_MANAGER"
 		exit 1
@@ -201,7 +204,7 @@ function install_protobuf {
 	echo "==> installing protobuf compiler..."
 
 	case "$PACKAGE_MANAGER" in
-	brew)
+	brew | apk)
 		install_pkg protobuf "$PACKAGE_MANAGER"
 		;;
 	*)
@@ -343,7 +346,6 @@ function install_sqlite3 {
 		;;
 	apk)
 		install_pkg sqlite-dev "$PACKAGE_MANAGER"
-		install_pkg sqlite "$PACKAGE_MANAGER"
 		;;
 	yum | dnf)
 		install_pkg sqlite-devel "$PACKAGE_MANAGER"