From 7783809597d20ff178e93a2b339a5cd21840bb12 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 16 Jan 2024 16:16:30 -0800 Subject: [PATCH] Adjust tarball creation to be reproducible --- Dockerfile-builder.template | 82 ++++++++++++++++++++++-------- build.sh | 11 +++- latest-1/glibc/Dockerfile.builder | 80 +++++++++++++++++++++-------- latest-1/musl/Dockerfile.builder | 80 +++++++++++++++++++++-------- latest-1/uclibc/Dockerfile.builder | 78 ++++++++++++++++++++-------- latest/glibc/Dockerfile.builder | 80 +++++++++++++++++++++-------- latest/musl/Dockerfile.builder | 80 +++++++++++++++++++++-------- latest/uclibc/Dockerfile.builder | 78 ++++++++++++++++++++-------- 8 files changed, 418 insertions(+), 151 deletions(-) diff --git a/Dockerfile-builder.template b/Dockerfile-builder.template index 3205698c..6f72c08a 100644 --- a/Dockerfile-builder.template +++ b/Dockerfile-builder.template @@ -13,6 +13,8 @@ RUN set -eux; \ musl-dev \ patch \ tzdata \ +# busybox's tar ironically does not maintain mtime of directories correctly (which we need for SOURCE_DATE_EPOCH / reproducibility) + tar \ ; {{ ) else ( -}} FROM debian:bookworm-slim @@ -234,14 +236,37 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ @@ -249,7 +274,7 @@ RUN set -eux; \ # CONFIG_LAST_SUPPORTED_WCHAR: see https://github.com/docker-library/busybox/issues/13 (UTF-8 input) CONFIG_LAST_SUPPORTED_WCHAR=0 \ {{ if env.variant == "glibc" then ( -}} -# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... :cry: +# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... 😭 {{ ) else ( -}} CONFIG_STATIC=y \ {{ ) end -}} @@ -361,7 +386,10 @@ RUN set -eux; \ {{ ) else "" end -}} chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -404,25 +432,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -430,11 +464,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: diff --git a/build.sh b/build.sh index 8943ff00..2ae14cb2 100755 --- a/build.sh +++ b/build.sh @@ -13,7 +13,16 @@ for dir; do ( set -x docker build -t "$base-builder" -f "$dir/Dockerfile.builder" "$dir" - docker run --rm "$base-builder" tar cC rootfs . | xz -T0 -z9 > "$dir/busybox.tar.xz" + docker run --rm "$base-builder" \ + tar \ + --create \ + --directory rootfs \ + --numeric-owner \ + --transform 's,^./,,' \ + --sort name \ + . \ + | xz -T0 -z9 > "$dir/busybox.tar.xz" + sha256sum "$dir/busybox.tar.xz" docker build -t "$base-test" "$dir" docker run --rm "$base-test" sh -xec 'true' diff --git a/latest-1/glibc/Dockerfile.builder b/latest-1/glibc/Dockerfile.builder index 87a3158a..e4986772 100644 --- a/latest-1/glibc/Dockerfile.builder +++ b/latest-1/glibc/Dockerfile.builder @@ -34,21 +34,44 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ CONFIG_FEATURE_AR_LONG_FILENAMES=y \ # CONFIG_LAST_SUPPORTED_WCHAR: see https://github.com/docker-library/busybox/issues/13 (UTF-8 input) CONFIG_LAST_SUPPORTED_WCHAR=0 \ -# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... :cry: +# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... 😭 '; \ \ unsetConfs=' \ @@ -130,7 +153,10 @@ RUN set -eux; \ done; \ chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -171,25 +197,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -197,11 +229,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: diff --git a/latest-1/musl/Dockerfile.builder b/latest-1/musl/Dockerfile.builder index 1fb32242..6a9c2979 100644 --- a/latest-1/musl/Dockerfile.builder +++ b/latest-1/musl/Dockerfile.builder @@ -18,6 +18,8 @@ RUN set -eux; \ musl-dev \ patch \ tzdata \ +# busybox's tar ironically does not maintain mtime of directories correctly (which we need for SOURCE_DATE_EPOCH / reproducibility) + tar \ ; # pub 1024D/ACC9965B 2006-12-12 @@ -36,14 +38,37 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ @@ -110,7 +135,10 @@ RUN set -eux; \ gcc -o rootfs/bin/getconf -static -Os /usr/src/getconf.c; \ chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -151,25 +179,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -177,11 +211,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: diff --git a/latest-1/uclibc/Dockerfile.builder b/latest-1/uclibc/Dockerfile.builder index 3fa33dc5..21227a57 100644 --- a/latest-1/uclibc/Dockerfile.builder +++ b/latest-1/uclibc/Dockerfile.builder @@ -220,14 +220,37 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ @@ -285,7 +308,10 @@ RUN set -eux; \ ln -vL ../buildroot/output/target/usr/bin/getconf rootfs/bin/; \ chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -313,25 +339,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -339,11 +371,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: diff --git a/latest/glibc/Dockerfile.builder b/latest/glibc/Dockerfile.builder index 32fc5a3c..a496ba16 100644 --- a/latest/glibc/Dockerfile.builder +++ b/latest/glibc/Dockerfile.builder @@ -34,21 +34,44 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ CONFIG_FEATURE_AR_LONG_FILENAMES=y \ # CONFIG_LAST_SUPPORTED_WCHAR: see https://github.com/docker-library/busybox/issues/13 (UTF-8 input) CONFIG_LAST_SUPPORTED_WCHAR=0 \ -# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... :cry: +# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... 😭 '; \ \ unsetConfs=' \ @@ -128,7 +151,10 @@ RUN set -eux; \ done; \ chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -169,25 +195,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -195,11 +227,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: diff --git a/latest/musl/Dockerfile.builder b/latest/musl/Dockerfile.builder index c98f14f9..22b489d6 100644 --- a/latest/musl/Dockerfile.builder +++ b/latest/musl/Dockerfile.builder @@ -18,6 +18,8 @@ RUN set -eux; \ musl-dev \ patch \ tzdata \ +# busybox's tar ironically does not maintain mtime of directories correctly (which we need for SOURCE_DATE_EPOCH / reproducibility) + tar \ ; # pub 1024D/ACC9965B 2006-12-12 @@ -36,14 +38,37 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ @@ -108,7 +133,10 @@ RUN set -eux; \ gcc -o rootfs/bin/getconf -static -Os /usr/src/getconf.c; \ chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -149,25 +177,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -175,11 +209,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: diff --git a/latest/uclibc/Dockerfile.builder b/latest/uclibc/Dockerfile.builder index 4b5b8a45..14e65219 100644 --- a/latest/uclibc/Dockerfile.builder +++ b/latest/uclibc/Dockerfile.builder @@ -220,14 +220,37 @@ RUN set -eux; \ curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \ echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \ gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \ - mkdir -p /usr/src/busybox; \ - tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \ - rm busybox.tar.bz2* +# Alpine... 😅 + mkdir -p /usr/src; \ + tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \ + mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \ + rm busybox.tar.bz2*; \ + \ +# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later + SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822 WORKDIR /usr/src/busybox +# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise) +RUN set -eux; \ + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \ + echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \ + chmod +x clamp.sh; \ + mkdir rootfs; \ + ./clamp.sh + RUN set -eux; \ \ +# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that + SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \ + export SOURCE_DATE_EPOCH; \ +# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly) + \ setConfs=' \ CONFIG_AR=y \ CONFIG_FEATURE_AR_CREATE=y \ @@ -283,7 +306,10 @@ RUN set -eux; \ ln -vL ../buildroot/output/target/usr/bin/getconf rootfs/bin/; \ chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \ \ - chroot rootfs /bin/busybox --install /bin +# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full") + chroot rootfs /bin/busybox --install /bin; \ + \ + ./clamp.sh # install a few extra files from buildroot (/etc/passwd, etc) RUN set -eux; \ @@ -311,25 +337,31 @@ RUN set -eux; \ } \ printf "chmod %s %s\n", $3, $1; \ } \ - ' ../buildroot/system/device_table.txt | sh -eux + ' ../buildroot/system/device_table.txt | sh -eux; \ + \ + ./clamp.sh # create missing home directories and ensure /usr/bin/env exists RUN set -eux; \ - cd rootfs; \ - for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ - user="${userHome%%=*}"; \ - home="${userHome#*=}"; \ - home="./${home#/}"; \ - if [ ! -d "$home" ]; then \ - mkdir -p "$home"; \ - chown "$user" "$home"; \ - chmod 755 "$home"; \ + ( \ + cd rootfs; \ + for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \ + user="${userHome%%=*}"; \ + home="${userHome#*=}"; \ + home="./${home#/}"; \ + if [ ! -d "$home" ]; then \ + mkdir -p "$home"; \ + chown "$user" "$home"; \ + chmod 755 "$home"; \ + fi; \ + done; \ + if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ + mkdir -p usr/bin; \ + ln -s ../../bin/env usr/bin/; \ fi; \ - done; \ - if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \ - mkdir -p usr/bin; \ - ln -s ../../bin/env usr/bin/; \ - fi + ); \ + \ + ./clamp.sh # test and make sure it works RUN chroot rootfs /usr/bin/env sh -xec 'true' @@ -337,11 +369,15 @@ RUN chroot rootfs /usr/bin/env sh -xec 'true' # ensure correct timezone (UTC) RUN set -eux; \ ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \ - [ "$(chroot rootfs date +%Z)" = 'UTC' ] + [ "$(chroot rootfs date +%Z)" = 'UTC' ]; \ + \ + ./clamp.sh # test and make sure DNS works too RUN cp -L /etc/resolv.conf rootfs/etc/; \ chroot rootfs /bin/sh -xec 'nslookup google.com'; \ - rm rootfs/etc/resolv.conf + rm rootfs/etc/resolv.conf; \ + \ + ./clamp.sh # vim:set ft=dockerfile: