From aa4f96978154e514945a3be0b48ce6f2be27708e Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Sun, 8 Oct 2023 14:34:13 +0800 Subject: [PATCH 1/5] Fix docker container test instruction --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eae77f4..032b6b3 100644 --- a/Makefile +++ b/Makefile @@ -34,9 +34,13 @@ sync-ckb: if [ -d ckb ]; then git -C ckb pull; else git clone --depth 1 https://github.com/nervosnetwork/ckb.git; fi test-bionic: sync-ckb - docker --rm -it -w /ckb -v "$$(pwd)/ckb:/ckb" ${DOCKERHUB_REPO}:bionic-${IMAGE_VERSION} make prod + docker run --rm -it -w /ckb -v "$$(pwd)/ckb:/ckb" \ +-e OPENSSL_STATIC=1 -e OPENSSL_LIB_DIR=/usr/local/lib64 -e OPENSSL_INCLUDE_DIR=/usr/local/include \ +${DOCKERHUB_REPO}:bionic-${IMAGE_VERSION} make prod test-centos-7: sync-ckb - docker --rm -it -w /ckb -v "$$(pwd)/ckb:/ckb" ${DOCKERHUB_REPO}:centos-7-${IMAGE_VERSION} make prod + docker run --rm -it -w /ckb -v "$$(pwd)/ckb:/ckb" \ +-e OPENSSL_STATIC=1 -e OPENSSL_LIB_DIR=/usr/local/lib64 -e OPENSSL_INCLUDE_DIR=/usr/local/include \ +${DOCKERHUB_REPO}:centos-7-${IMAGE_VERSION} make prod .PHONY: test-all test-bionic test-centos-7 sync-ckb From f647f431ab0f6d0d24d93a365d18abddbff67ed3 Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Sun, 8 Oct 2023 14:35:18 +0800 Subject: [PATCH 2/5] Add OPENSSL_VERSION and OPENSSL_SHA256 to gen-docker-files script --- Makefile | 4 +++- gen-dockerfiles | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 032b6b3..aecb573 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ DOCKERHUB_REPO := nervos/ckb-docker-builder GHCR_REPO := ghcr.io/nervosnetwork/ckb-docker-builder -IMAGE_VERSION := rust-$(shell sed -n "s/RUST_VERSION = '\(.*\)'$$/\1/p" gen-dockerfiles) +RUST_VERSION := rust-$(shell sed -n "s/RUST_VERSION\s*=\s*'\(.*\)'$$/\1/p" gen-dockerfiles) +OPENSSL_VERSION := openssl-$(shell sed -n "s/OPENSSL_VERSION\s*=\s*'\(.*\)'$$/\1/p" gen-dockerfiles) +IMAGE_VERSION := ${RUST_VERSION}-${OPENSSL_VERSION} bionic/Dockerfile: gen-dockerfiles templates/bionic.Dockerfile python3 gen-dockerfiles diff --git a/gen-dockerfiles b/gen-dockerfiles index 6515253..f175344 100755 --- a/gen-dockerfiles +++ b/gen-dockerfiles @@ -3,8 +3,9 @@ import os from urllib import request -RUST_VERSION = '1.71.1' -RUSTUP_VERSION = '1.26.0' +RUST_VERSION = '1.71.1' +RUSTUP_VERSION = '1.26.0' +OPENSSL_VERSION = '3.1.3' RUST_ARCH = 'x86_64-unknown-linux-gnu' @@ -21,6 +22,11 @@ def fetch_rustup_hash(): with request.urlopen(url) as f: return f.read().decode('utf-8').split()[0] +def fetch_openssl_hash(): + url = f'https://www.openssl.org/source/openssl-{OPENSSL_VERSION}.tar.gz.sha256' + with request.urlopen(url) as f: + return f.read().decode('utf-8').split()[0] + def load_template(dist): with open(f'templates/{dist}.Dockerfile', 'r') as f: @@ -35,20 +41,23 @@ def save_dockerfile(dist, contents): f.write(contents) -def generate_dockerfile(dist, rustup_sha256): +def generate_dockerfile(dist, rustup_sha256, openssl_sha256): template = load_template(dist) rendered = template \ .replace('%%RUST_VERSION%%', RUST_VERSION) \ .replace('%%RUSTUP_VERSION%%', RUSTUP_VERSION) \ .replace('%%RUSTUP_SHA256%%', rustup_sha256) \ - .replace('%%RUST_ARCH%%', RUST_ARCH) + .replace('%%RUST_ARCH%%', RUST_ARCH) \ + .replace('%%OPENSSL_VERSION%%', OPENSSL_VERSION) \ + .replace('%%OPENSSL_SHA256%%', openssl_sha256) save_dockerfile(dist, rendered) def main(): rustup_sha256 = fetch_rustup_hash() + openssl_sha256 = fetch_openssl_hash() for dist in DISTRIBUTIONS: - generate_dockerfile(dist, rustup_sha256) + generate_dockerfile(dist, rustup_sha256, openssl_sha256) if __name__ == '__main__': From c28de614b96339c868ae1f4260d8d83eb5a1833f Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Sun, 8 Oct 2023 14:35:49 +0800 Subject: [PATCH 3/5] Build openssl from source --- templates/bionic.Dockerfile | 18 ++++++++++++++++-- templates/centos-7.Dockerfile | 18 +++++++++--------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/templates/bionic.Dockerfile b/templates/bionic.Dockerfile index 04cf1ef..a23359a 100644 --- a/templates/bionic.Dockerfile +++ b/templates/bionic.Dockerfile @@ -8,7 +8,6 @@ RUN set -eux; \ g++ \ libc6-dev \ wget \ - libssl-dev \ git \ pkg-config \ libclang-dev \ @@ -20,7 +19,22 @@ ENV RUSTUP_HOME=/usr/local/rustup \ PATH=/usr/local/cargo/bin:$PATH \ RUSTUP_VERSION=%%RUSTUP_VERSION%% \ RUSTUP_SHA256=%%RUSTUP_SHA256%% \ - RUST_ARCH=%%RUST_ARCH%% + RUST_ARCH=%%RUST_ARCH%% \ + OPENSSL_VERSION=%%OPENSSL_VERSION%% \ + OPENSSL_SHA256=%%OPENSSL_SHA256%% + +RUN set -eux; \ + url="https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"; \ + wget --no-check-certificate "$url"; \ + echo "${OPENSSL_SHA256} *openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -; \ + tar -xzf "openssl-${OPENSSL_VERSION}.tar.gz"; \ + cd openssl-${OPENSSL_VERSION}; \ + ./config no-shared no-zlib -fPIC -DOPENSSL_NO_SECURE_MEMORY; \ + make; \ + make install; \ + cd ..; \ + rm -rf openssl-${OPENSSL_VERSION} openssl-${OPENSSL_VERSION}.tar.gz + RUN set -eux; \ url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${RUST_ARCH}/rustup-init"; \ diff --git a/templates/centos-7.Dockerfile b/templates/centos-7.Dockerfile index 35fec35..63977cf 100644 --- a/templates/centos-7.Dockerfile +++ b/templates/centos-7.Dockerfile @@ -4,12 +4,18 @@ ENV PATH=/root/.cargo/bin:$PATH RUN set -eux; \ yum install -y centos-release-scl; \ - yum install -y git curl make devtoolset-7 llvm-toolset-7 perl-core pcre-devel wget zlib-devel; \ + yum install -y git curl make devtoolset-8 llvm-toolset-7 perl-core pcre-devel wget zlib-devel; \ yum clean all; \ rm -rf /var/cache/yum -ENV OPENSSL_VERSION=1.1.1q \ - OPENSSL_SHA256=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUSTUP_VERSION=%%RUSTUP_VERSION%% \ + RUSTUP_SHA256=%%RUSTUP_SHA256%% \ + RUST_ARCH=%%RUST_ARCH%% \ + OPENSSL_VERSION=%%OPENSSL_VERSION%% \ + OPENSSL_SHA256=%%OPENSSL_SHA256%% RUN set -eux; \ url="https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"; \ @@ -23,12 +29,6 @@ RUN set -eux; \ cd ..; \ rm -rf openssl-${OPENSSL_VERSION} openssl-${OPENSSL_VERSION}.tar.gz -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH \ - RUSTUP_VERSION=%%RUSTUP_VERSION%% \ - RUSTUP_SHA256=%%RUSTUP_SHA256%% \ - RUST_ARCH=%%RUST_ARCH%% RUN set -eux; \ url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${RUST_ARCH}/rustup-init"; \ From c5d3acade9762de1fa603d4e3438ed198e617d50 Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Sun, 8 Oct 2023 14:36:19 +0800 Subject: [PATCH 4/5] Execute `gen-dockerfiles` script --- bionic/Dockerfile | 18 ++++++++++++++++-- centos-7/Dockerfile | 18 +++++++++--------- centos-7/entrypoint.sh | 2 +- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/bionic/Dockerfile b/bionic/Dockerfile index 1e2b08a..1db8140 100644 --- a/bionic/Dockerfile +++ b/bionic/Dockerfile @@ -8,7 +8,6 @@ RUN set -eux; \ g++ \ libc6-dev \ wget \ - libssl-dev \ git \ pkg-config \ libclang-dev \ @@ -20,7 +19,22 @@ ENV RUSTUP_HOME=/usr/local/rustup \ PATH=/usr/local/cargo/bin:$PATH \ RUSTUP_VERSION=1.26.0 \ RUSTUP_SHA256=0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db \ - RUST_ARCH=x86_64-unknown-linux-gnu + RUST_ARCH=x86_64-unknown-linux-gnu \ + OPENSSL_VERSION=3.1.3 \ + OPENSSL_SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 + +RUN set -eux; \ + url="https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"; \ + wget --no-check-certificate "$url"; \ + echo "${OPENSSL_SHA256} *openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -; \ + tar -xzf "openssl-${OPENSSL_VERSION}.tar.gz"; \ + cd openssl-${OPENSSL_VERSION}; \ + ./config no-shared no-zlib -fPIC -DOPENSSL_NO_SECURE_MEMORY; \ + make; \ + make install; \ + cd ..; \ + rm -rf openssl-${OPENSSL_VERSION} openssl-${OPENSSL_VERSION}.tar.gz + RUN set -eux; \ url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${RUST_ARCH}/rustup-init"; \ diff --git a/centos-7/Dockerfile b/centos-7/Dockerfile index 29f85cb..ce9b2a7 100644 --- a/centos-7/Dockerfile +++ b/centos-7/Dockerfile @@ -4,12 +4,18 @@ ENV PATH=/root/.cargo/bin:$PATH RUN set -eux; \ yum install -y centos-release-scl; \ - yum install -y git curl make devtoolset-7 llvm-toolset-7 perl-core pcre-devel wget zlib-devel; \ + yum install -y git curl make devtoolset-8 llvm-toolset-7 perl-core pcre-devel wget zlib-devel; \ yum clean all; \ rm -rf /var/cache/yum -ENV OPENSSL_VERSION=1.1.1q \ - OPENSSL_SHA256=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUSTUP_VERSION=1.26.0 \ + RUSTUP_SHA256=0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db \ + RUST_ARCH=x86_64-unknown-linux-gnu \ + OPENSSL_VERSION=3.1.3 \ + OPENSSL_SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 RUN set -eux; \ url="https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"; \ @@ -23,12 +29,6 @@ RUN set -eux; \ cd ..; \ rm -rf openssl-${OPENSSL_VERSION} openssl-${OPENSSL_VERSION}.tar.gz -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH \ - RUSTUP_VERSION=1.26.0 \ - RUSTUP_SHA256=0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db \ - RUST_ARCH=x86_64-unknown-linux-gnu RUN set -eux; \ url="https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${RUST_ARCH}/rustup-init"; \ diff --git a/centos-7/entrypoint.sh b/centos-7/entrypoint.sh index 91735d3..0f0cca0 100755 --- a/centos-7/entrypoint.sh +++ b/centos-7/entrypoint.sh @@ -2,4 +2,4 @@ export OPENSSL_LIB_DIR=/usr/local/lib64 OPENSSL_INCLUDE_DIR=/usr/local/include -scl enable devtoolset-7 llvm-toolset-7 "$*" +scl enable devtoolset-8 llvm-toolset-7 "$*" From bc3bda05df7b85e5335c21cca9dbae8adf6741b4 Mon Sep 17 00:00:00 2001 From: Eval EXEC Date: Sun, 8 Oct 2023 14:38:03 +0800 Subject: [PATCH 5/5] Update README.md: we need to take care of openssl version --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7f42834..a23637c 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Environment for building [ckb](https://github.com/nervosnetwork/ckb#readme). ## How to Upgrade Rust -- Update rustup version if needed in [`gen-dockerfiles`]. +- Update rustup and openssl version if needed in [`gen-dockerfiles`]. - Update rust version in [`gen-dockerfiles`]. - Run the script [`gen-dockerfiles`]. -- Commit, tag `rust-${RUST_VERSION}` such as `rust-1.51.0`. +- Commit, tag `rust-${RUST_VERSION}` such as `rust-1.71.0-openssl-3.1.3`. [`gen-dockerfiles`]: gen-dockerfiles