Switch the nightly build to producing a multiarch docker image that supports both x86_64 and arm64

.github/workflows/nightly.yml

@@ -9,50 +9,156 @@ on:
     branches:
       - 'main'
 
+env:
+  REGISTRY_IMAGE: clux/muslrust
+
 jobs:
-  docker:
+  build:
     name: 'Nightly Build'
     runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [linux/amd64, linux/arm64]
+        include:
+          - platform: linux/amd64
+            dockerfile: Dockerfile.x86_64
+            arch: amd64
+            target_dir: x86_64-unknown-linux-musl
+          - platform: linux/arm64
+            dockerfile: Dockerfile.arm64
+            arch: arm64
+            target_dir: aarch64-unknown-linux-musl
     steps:
     - uses: 'actions/checkout@v2'
     - uses: extractions/setup-just@v1
 
     - name: Login to DockerHub
-      if: github.event_name != 'pull_request'
       uses: docker/login-action@v1
       with:
         username: clux
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+    - name: Prepare
+      run: |
+        platform=${{ matrix.platform }}
+        echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY_IMAGE }}
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
     - name: Build nightly image
-      uses: docker/build-push-action@v2
+      id: build
+      uses: docker/build-push-action@v3
       with:
         context: .
+        platforms: ${{ matrix.platform }}
+        labels: ${{ steps.meta.outputs.labels }}
+        file:  ${{ matrix.dockerfile }}
         push: false
+        load: true
+        tags: rustmusl-temp
         build-args: |
           CHANNEL=nightly
-        tags: clux/muslrust:temp
 
-    - name: Compute tags
+    - name: Run tests
+      shell: bash
+      run: |
+        docker image ls
+        docker buildx build --platform ${{ matrix.platform }} --output type=docker -t test-runner - < Dockerfile.test-runner
+        docker image ls
+        TARGET_DIR=${{ matrix.target_dir }} PLATFORM=${{ matrix.platform }} just test
+
+    # The date/channel/version are expected to be the same on both architectures and are needed for the merge step.
+    # We store them here since it makes the merge step a bit easier - it doesn't need to figure out which of the
+    # architectures it can run (to extract the rust version). The problem is that it appears we can't run images
+    # that were built by docker buildx (the build-push-action step) locally. They get pushed to dockerhub but are
+    # only identifiable by their digest and it appears docker does not let us select an image that way.
+    # Not the most elegant, but it works.
+    - name: Store tag info
       shell: bash
       run: |
-        docker run clux/muslrust:temp rustc --version
-        RUST_VER="$(docker run clux/muslrust:temp rustc --version | grep -oE "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]")"
+        mkdir -p /tmp/tags
         RUST_DATE="$(date +"%Y-%m-%d")"
         RUST_CHANNEL=nightly
-        echo "TAG1=clux/muslrust:${RUST_CHANNEL}" >> $GITHUB_ENV
-        echo "TAG2=clux/muslrust:${RUST_CHANNEL}-${RUST_DATE}" >> $GITHUB_ENV
-        echo "TAG3=clux/muslrust:${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}" >> $GITHUB_ENV
+        RUST_VER="$(docker run --platform ${{ matrix.platform }} rustmusl-temp rustc --version | grep -oE "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]")"
 
-    - name: Run tests
+        echo $RUST_DATE > /tmp/tags/rust-date
+        echo $RUST_CHANNEL > /tmp/tags/rust-channel
+        echo $RUST_VER > /tmp/tags/rust-ver
+
+    - name: Tag and push
       shell: bash
-      run: just test
+      run: |
+        RUST_DATE=$(cat /tmp/tags/rust-date)
+        RUST_CHANNEL=$(cat /tmp/tags/rust-channel)
+        RUST_VER=$(cat /tmp/tags/rust-ver)
+
+        TAG_NAME="${{ matrix.arch }}-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}"
 
-    - name: Push image under computed tags
-      uses: docker/build-push-action@v2
+        docker tag rustmusl-temp ${{ env.REGISTRY_IMAGE }}:$TAG_NAME
+        docker push ${{ env.REGISTRY_IMAGE }}:$TAG_NAME
+
+    - name: Upload tags
+      uses: actions/upload-artifact@v4
       with:
-        context: .
-        build-args: |
-          CHANNEL=nightly
-        push: ${{ github.event_name != 'pull_request' }}
-        tags: clux/muslrust:latest,${{ env.TAG1 }},${{ env.TAG2 }},${{ env.TAG3 }}
+        name: tags
+        path: /tmp/tags
+        if-no-files-found: error
+        retention-days: 1
+        overwrite: true
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+
+      -
+        name: Download tags
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/tags
+          name: tags
+
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: clux
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Create manifest list and push multi-platform images
+        run: |
+          RUST_DATE=$(cat /tmp/tags/rust-date)
+          RUST_CHANNEL=$(cat /tmp/tags/rust-channel)
+          RUST_VER=$(cat /tmp/tags/rust-ver)
+
+          for tag in latest ${RUST_CHANNEL} ${RUST_CHANNEL}-${RUST_DATE} ${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}; do
+            docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:$tag \
+              ${{ env.REGISTRY_IMAGE }}:amd64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE} \
+              ${{ env.REGISTRY_IMAGE }}:arm64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE}
+            done
+
+      -
+        name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest

Dockerfile.arm64

@@ -0,0 +1,157 @@
+FROM ubuntu:jammy
+LABEL maintainer="Eirik Albrigtsen <[email protected]>"
+
+# Required packages:
+# - musl-dev, musl-tools - the musl toolchain
+# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs
+# - ca-certificates - openssl + curl + peer verification of downloads
+# - xutils-dev - for openssl makedepend
+# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process
+# - git - cargo builds in user projects
+# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only)
+# - file - needed by rustup.sh install
+# - automake autoconf libtool - support crates building C deps as part cargo build
+# NB: does not include cmake atm
+RUN apt-get update && apt-get install -y \
+  musl-dev \
+  musl-tools \
+  file \
+  git \
+  openssh-client \
+  make \
+  cmake \
+  g++ \
+  curl \
+  pkgconf \
+  ca-certificates \
+  xutils-dev \
+  libssl-dev \
+  libpq-dev \
+  automake \
+  autoconf \
+  libtool \
+  libprotobuf-dev \
+  unzip \
+  --no-install-recommends && \
+  rm -rf /var/lib/apt/lists/*
+
+# Install rust using rustup
+ARG CHANNEL
+ENV RUSTUP_VER="1.26.0" \
+    RUST_ARCH="aarch64-unknown-linux-gnu" \
+    CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+
+RUN curl "https://static.rust-lang.org/rustup/archive/${RUSTUP_VER}/${RUST_ARCH}/rustup-init" -o rustup-init && \
+    chmod +x rustup-init && \
+    ./rustup-init -y --default-toolchain ${CHANNEL} --profile minimal --no-modify-path && \
+    rm rustup-init && \
+    ~/.cargo/bin/rustup target add aarch64-unknown-linux-musl
+
+# Allow non-root access to cargo
+RUN chmod a+X /root
+
+# Convenience list of versions and variables for compilation later on
+# This helps continuing manually if anything breaks.
+ENV SSL_VER="1.1.1w" \
+    CURL_VER="8.6.0" \
+    ZLIB_VER="1.3.1" \
+    PQ_VER="11.12" \
+    SQLITE_VER="3450100" \
+    PROTOBUF_VER="25.2" \
+    CC=musl-gcc \
+    PREFIX=/musl \
+    PATH=/usr/local/bin:/root/.cargo/bin:$PATH \
+    PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \
+    LD_LIBRARY_PATH=$PREFIX
+
+# Install a more recent release of protoc (protobuf-compiler in jammy is 4 years old and misses some features)
+RUN cd /tmp && \
+    curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-aarch_64.zip -o protoc.zip && \
+    unzip protoc.zip && \
+    cp bin/protoc /usr/bin/protoc && \
+    rm -rf *
+
+# Set up a prefix for musl build libraries, make the linker's job of finding them easier
+# Primarily for the benefit of postgres.
+# Lastly, link some linux-headers for openssl 1.1 (not used herein)
+RUN mkdir $PREFIX && \
+    echo "$PREFIX/lib" >> /etc/ld-musl-aarch64.path && \
+    ln -s /usr/include/aarch64-linux-gnu/asm /usr/include/aarch64-linux-musl/asm && \
+    ln -s /usr/include/asm-generic /usr/include/aarch64-linux-musl/asm-generic && \
+    ln -s /usr/include/linux /usr/include/aarch64-linux-musl/linux
+
+# Build zlib (used in openssl and pq)
+RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \
+    cd zlib-$ZLIB_VER && \
+    CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \
+    make -j$(nproc) && make install && \
+    cd .. && rm -rf zlib-$ZLIB_VER
+
+# Build openssl (used in curl and pq)
+# Would like to use zlib here, but can't seem to get it to work properly
+# TODO: fix so that it works
+RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz && \
+    cd openssl-$SSL_VER && \
+    CFLAGS="-mno-outline-atomics" ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-aarch64 && \
+    env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \
+    make -j$(nproc) && make all install_sw && \
+    cd .. && rm -rf openssl-$SSL_VER
+
+# Build curl (needs with-zlib and all this stuff to allow https)
+# curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what
+RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \
+    cd curl-$CURL_VER && \
+    CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+      --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \
+      --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \
+      --with-openssl --without-libpsl && \
+    make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \
+    cd .. && rm -rf curl-$CURL_VER
+
+# Build libpq
+RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \
+    cd postgresql-$PQ_VER && \
+    CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+    --without-readline \
+    --with-openssl \
+    --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \
+    cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \
+    cd ../../bin/pg_config && make -j $(nproc) && make install && \
+    cd .. && rm -rf postgresql-$PQ_VER
+
+# Build libsqlite3 using same configuration as the alpine linux main/sqlite package
+RUN curl -sSL https://www.sqlite.org/2024/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \
+    cd sqlite-autoconf-$SQLITE_VER && \
+    CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \
+    CC="musl-gcc -fPIC -pie" \
+    ./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --enable-dynamic-extensions --disable-shared && \
+    make && make install && \
+    cd .. && rm -rf sqlite-autoconf-$SQLITE_VER
+
+# SSL cert directories get overridden by --prefix and --openssldir
+# and they do not match the typical host configurations.
+# The SSL_CERT_* vars fix this, but only when inside this container
+# musl-compiled binary must point SSL at the correct certs (muslrust/issues/5) elsewhere
+# Postgres bindings need vars so that diesel_codegen.so uses the GNU deps at build time
+# but finally links with the static libpq.a at the end.
+# It needs the non-musl pg_config to set this up with libpq-dev (depending on libssl-dev)
+# See https://github.com/sgrif/pq-sys/pull/18
+ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \
+    RUSTUP_HOME=/root/.rustup \
+    CARGO_BUILD_TARGET=aarch64-unknown-linux-musl \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld -Ctarget-feature=+crt-static" \
+    PKG_CONFIG_ALLOW_CROSS=true \
+    PKG_CONFIG_ALL_STATIC=true \
+    PQ_LIB_STATIC_AARCH64_UNKNOWN_LINUX_MUSL=true \
+    PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \
+    PG_CONFIG_AARCH64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \
+    OPENSSL_STATIC=true \
+    OPENSSL_DIR=$PREFIX \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
+    SSL_CERT_DIR=/etc/ssl/certs \
+    LIBZ_SYS_STATIC=1 \
+    DEBIAN_FRONTEND=noninteractive \
+    TZ=Etc/UTC
+
+# Allow ditching the -w /volume flag to docker run
+WORKDIR /volume

Dockerfile.test-runner

@@ -0,0 +1,5 @@
+FROM ubuntu:jammy
+
+RUN apt-get update && apt-get install -y \
+  ca-certificates \
+  file

Dockerfile → Dockerfile.x86_64

@@ -94,7 +94,7 @@ RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz &&
     cd openssl-$SSL_VER && \
     ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \
     env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \
-    make -j$(nproc) && make install && \
+    make -j$(nproc) && make all install_sw && \
     cd .. && rm -rf openssl-$SSL_VER
 
 # Build curl (needs with-zlib and all this stuff to allow https)

test.sh

@@ -2,18 +2,31 @@
 set -ex
 
 docker_build() {
+  echo "Target dir: $TARGET_DIR"
+  echo "Platform: $PLATFORM"
+
   # NB: add -vv to cargo build when debugging
   local -r crate="$1"crate
   docker run --rm \
     -v "$PWD/test/${crate}:/volume" \
     -v cargo-cache:/root/.cargo/registry \
     -e RUST_BACKTRACE=1 \
-    clux/muslrust:temp \
+    -e AR=ar \
+    --platform $PLATFORM \
+    rustmusl-temp \
     cargo build
+
   cd "test/${crate}"
-  ./target/x86_64-unknown-linux-musl/debug/"${crate}"
-  ldd "target/x86_64-unknown-linux-musl/debug/${crate}" 2>&1 | grep -qE "not a dynamic|statically linked" && \
-    echo "${crate} is a static executable"
+
+  # Ideally we would use `ldd` but due to a qemu bug we can't :(
+  # See https://github.com/multiarch/qemu-user-static/issues/172
+  # Instead we use `file`.
+  docker run --rm \
+    -v "$PWD:/volume" \
+    -e RUST_BACKTRACE=1 \
+    --platform $PLATFORM \
+    test-runner \
+    bash -c "cd volume; ./target/$TARGET_DIR/debug/${crate} && file ./target/$TARGET_DIR/debug/${crate} && file /volume/target/$TARGET_DIR/debug/${crate} 2>&1 | grep -qE 'static-pie linked|statically linked' && echo ${crate} is a static executable"
 }
 
 # Helper to check how ekidd/rust-musl-builder does it