From 537a6a2706d20d16ce5bc7bc15278a3237f08941 Mon Sep 17 00:00:00 2001 From: Rustie Lin Date: Mon, 18 Nov 2024 14:20:31 -0800 Subject: [PATCH] [docker] use cloudfront debian mirrors for bullseye (#15303) * [builder] Point docker builds to aws hosted cloudfront mirror This should improve build time and reliability Test Plan: build images * [docker] use cloudfront debian mirrors for bullseye * [docker] debian-base install * [docker] cleanup stuff into debian-base --------- Co-authored-by: Perry Randall (cherry picked from commit 7cf76370091bcc470ae4342e49f60e58f6e4eabd) --- docker/builder/builder.Dockerfile | 24 ++++++----- docker/builder/debian-base.Dockerfile | 15 ++++++- docker/builder/faucet.Dockerfile | 8 +--- docker/builder/forge.Dockerfile | 16 ++++---- docker/builder/indexer-grpc.Dockerfile | 12 ------ .../builder/keyless-pepper-service.Dockerfile | 12 ------ .../builder/nft-metadata-crawler.Dockerfile | 12 ------ docker/builder/node-checker.Dockerfile | 13 +----- docker/builder/telemetry-service.Dockerfile | 14 +------ docker/builder/tools.Dockerfile | 7 +--- docker/builder/validator-testing.Dockerfile | 40 ++++++++----------- docker/builder/validator.Dockerfile | 9 +---- 12 files changed, 57 insertions(+), 125 deletions(-) diff --git a/docker/builder/builder.Dockerfile b/docker/builder/builder.Dockerfile index 66ada6142bf653..ae619444fc71e3 100644 --- a/docker/builder/builder.Dockerfile +++ b/docker/builder/builder.Dockerfile @@ -3,21 +3,23 @@ FROM rust as rust-base WORKDIR /aptos + RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sed -i 's|http://deb.debian.org/debian|http://cloudfront.debian.net/debian|g' /etc/apt/sources.list && \ apt update && apt-get --no-install-recommends install -y \ - cmake \ - curl \ - clang \ - git \ - pkg-config \ - libssl-dev \ - libpq-dev \ - libdw-dev \ - binutils \ - lld \ - libudev-dev + binutils \ + clang \ + cmake \ + curl \ + git \ + libdw-dev \ + libpq-dev \ + libssl-dev \ + libudev-dev \ + lld \ + pkg-config ### Build Rust code ### FROM rust-base as builder-base diff --git a/docker/builder/debian-base.Dockerfile b/docker/builder/debian-base.Dockerfile index e27cb5c34263d9..4bdaa397c68f00 100644 --- a/docker/builder/debian-base.Dockerfile +++ b/docker/builder/debian-base.Dockerfile @@ -6,6 +6,19 @@ ARG TARGETARCH RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sed -i 's|http://deb.debian.org/debian|http://cloudfront.debian.net/debian|g' /etc/apt/sources.list && \ + apt-get update && apt-get --no-install-recommends --allow-downgrades -y install \ + ca-certificates \ + curl \ + iproute2 \ + libpq-dev \ + libssl1.1 \ + netcat \ + net-tools \ + tcpdump + # Add Tini to make sure the binaries receive proper SIGTERM signals when Docker is shut down ADD --chmod=755 https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TARGETARCH /tini -ENTRYPOINT ["/tini", "--"] \ No newline at end of file +ENTRYPOINT ["/tini", "--"] diff --git a/docker/builder/faucet.Dockerfile b/docker/builder/faucet.Dockerfile index afa7c35c866d74..23ee5fc6f2f669 100644 --- a/docker/builder/faucet.Dockerfile +++ b/docker/builder/faucet.Dockerfile @@ -3,14 +3,8 @@ FROM debian-base AS faucet RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sed -i 's|http://security.debian.org/debian-security|https://cloudfront.debian.net/debian-security|g' /etc/apt/sources.list && \ apt-get update && apt-get --no-install-recommends install -y \ - libssl1.1 \ - ca-certificates \ - nano \ - net-tools \ - tcpdump \ - iproute2 \ - netcat \ procps RUN mkdir -p /aptos/client/data/wallet/ diff --git a/docker/builder/forge.Dockerfile b/docker/builder/forge.Dockerfile index b5ee2fc3bd8cdf..70b2a62b473c48 100644 --- a/docker/builder/forge.Dockerfile +++ b/docker/builder/forge.Dockerfile @@ -3,16 +3,14 @@ FROM debian-base as forge RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ - openssh-client \ - wget \ - busybox \ - git \ - unzip \ - awscli + awscli \ + busybox \ + git \ + openssh-client \ + unzip \ + wget WORKDIR /aptos diff --git a/docker/builder/indexer-grpc.Dockerfile b/docker/builder/indexer-grpc.Dockerfile index 2f791810846620..867f30a60b493a 100644 --- a/docker/builder/indexer-grpc.Dockerfile +++ b/docker/builder/indexer-grpc.Dockerfile @@ -2,18 +2,6 @@ FROM debian-base AS indexer-grpc -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ - net-tools \ - tcpdump \ - iproute2 \ - netcat \ - libpq-dev \ - curl - COPY --link --from=indexer-builder /aptos/dist/aptos-indexer-grpc-cache-worker /usr/local/bin/aptos-indexer-grpc-cache-worker COPY --link --from=indexer-builder /aptos/dist/aptos-indexer-grpc-file-store /usr/local/bin/aptos-indexer-grpc-file-store COPY --link --from=indexer-builder /aptos/dist/aptos-indexer-grpc-data-service /usr/local/bin/aptos-indexer-grpc-data-service diff --git a/docker/builder/keyless-pepper-service.Dockerfile b/docker/builder/keyless-pepper-service.Dockerfile index fde68bca54f4ba..1ecadbce7bf87e 100644 --- a/docker/builder/keyless-pepper-service.Dockerfile +++ b/docker/builder/keyless-pepper-service.Dockerfile @@ -1,17 +1,5 @@ FROM debian-base AS keyless-pepper-service -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ - net-tools \ - tcpdump \ - iproute2 \ - netcat \ - libpq-dev \ - curl - COPY --link --from=tools-builder /aptos/dist/aptos-keyless-pepper-service /usr/local/bin/aptos-keyless-pepper-service EXPOSE 8000 diff --git a/docker/builder/nft-metadata-crawler.Dockerfile b/docker/builder/nft-metadata-crawler.Dockerfile index 1b1d6998740ad7..ddb5a1722153fe 100644 --- a/docker/builder/nft-metadata-crawler.Dockerfile +++ b/docker/builder/nft-metadata-crawler.Dockerfile @@ -4,18 +4,6 @@ FROM indexer-builder FROM debian-base AS nft-metadata-crawler -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ - net-tools \ - tcpdump \ - iproute2 \ - netcat \ - libpq-dev \ - curl - COPY --link --from=indexer-builder /aptos/dist/aptos-nft-metadata-crawler /usr/local/bin/aptos-nft-metadata-crawler # The health check port diff --git a/docker/builder/node-checker.Dockerfile b/docker/builder/node-checker.Dockerfile index d297f81eb39544..29941eac85c545 100644 --- a/docker/builder/node-checker.Dockerfile +++ b/docker/builder/node-checker.Dockerfile @@ -2,17 +2,6 @@ FROM debian-base AS node-checker -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ - net-tools \ - tcpdump \ - iproute2 \ - netcat \ - libpq-dev - COPY --link --from=tools-builder /aptos/dist/aptos-node-checker /usr/local/bin/aptos-node-checker ENV RUST_LOG_FORMAT=json @@ -25,4 +14,4 @@ ENV GIT_TAG ${GIT_TAG} ARG GIT_BRANCH ENV GIT_BRANCH ${GIT_BRANCH} ARG GIT_SHA -ENV GIT_SHA ${GIT_SHA} \ No newline at end of file +ENV GIT_SHA ${GIT_SHA} diff --git a/docker/builder/telemetry-service.Dockerfile b/docker/builder/telemetry-service.Dockerfile index 92e589d3c32447..8de9deffa10178 100644 --- a/docker/builder/telemetry-service.Dockerfile +++ b/docker/builder/telemetry-service.Dockerfile @@ -1,17 +1,5 @@ FROM debian-base AS telemetry-service -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ - net-tools \ - tcpdump \ - iproute2 \ - netcat \ - libpq-dev \ - curl - COPY --link --from=tools-builder /aptos/dist/aptos-telemetry-service /usr/local/bin/aptos-telemetry-service EXPOSE 8000 @@ -23,4 +11,4 @@ ENV GIT_TAG ${GIT_TAG} ARG GIT_BRANCH ENV GIT_BRANCH ${GIT_BRANCH} ARG GIT_SHA -ENV GIT_SHA ${GIT_SHA} \ No newline at end of file +ENV GIT_SHA ${GIT_SHA} diff --git a/docker/builder/tools.Dockerfile b/docker/builder/tools.Dockerfile index 9abfa556222475..16f2c1a3f74a3d 100644 --- a/docker/builder/tools.Dockerfile +++ b/docker/builder/tools.Dockerfile @@ -1,9 +1,6 @@ ### Tools Image ### FROM debian-base AS tools -RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list && \ - echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences.d/bullseye - RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get --no-install-recommends --allow-downgrades -y \ @@ -13,9 +10,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ perl-base=5.32.1-4+deb11u4 \ libtinfo6=6.2+20201114-2+deb11u2 \ git \ - libssl1.1 \ - ca-certificates \ - socat \ + socat \ python3-botocore/bullseye \ awscli/bullseye \ gnupg2 \ diff --git a/docker/builder/validator-testing.Dockerfile b/docker/builder/validator-testing.Dockerfile index 0171363e1d44c2..cd6b499c7f06c8 100644 --- a/docker/builder/validator-testing.Dockerfile +++ b/docker/builder/validator-testing.Dockerfile @@ -5,29 +5,23 @@ FROM debian-base as validator-testing-base RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get install -y --no-install-recommends \ - libssl1.1 \ - ca-certificates \ - # Needed to run debugging tools like perf - linux-perf \ - sudo \ - procps \ - gdb \ - curl \ - # postgres client lib required for indexer - libpq-dev \ - # Extra goodies for debugging - less \ - git \ - vim \ - nano \ - libjemalloc-dev \ - binutils \ - graphviz \ - ghostscript \ - strace \ - htop \ - sysstat \ - valgrind + # Needed to run debugging tools like perf + gdb \ + linux-perf \ + procps \ + sudo \ + # Extra goodies for debugging + binutils \ + ghostscript \ + git \ + graphviz \ + htop \ + less \ + libjemalloc-dev \ + strace \ + sysstat \ + valgrind \ + vim FROM node-builder diff --git a/docker/builder/validator.Dockerfile b/docker/builder/validator.Dockerfile index db356cf63ed713..905e0aee3c9eba 100644 --- a/docker/builder/validator.Dockerfile +++ b/docker/builder/validator.Dockerfile @@ -7,18 +7,13 @@ FROM tools-builder FROM debian-base AS validator RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get install --no-install-recommends -y \ - libssl1.1 \ - ca-certificates \ # Needed to run debugging tools like perf linux-perf \ sudo \ procps \ - gdb \ - curl \ - # postgres client lib required for indexer - libpq-dev + gdb ### Because build machine perf might not match run machine perf, we have to symlink ### Even if version slightly off, still mostly works