From bbda0d40a59e533605a03913fad04de2d5e1e230 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 25 Mar 2024 17:53:25 +0000 Subject: [PATCH] HPCC-31426 CentOS 7 + Python 3.8 Add custom build to support CentOS 7 + Python 3.8 Mount local vcpkg cache to avoid rebuilding packages Signed-off-by: Gordon Smith --- .github/workflows/build-assets.yml | 4 ++ CMakeLists.txt | 8 ++-- cmake_modules/options.cmake | 2 + dockerfiles/vcpkg/build.sh | 40 +++++++++++++------ .../vcpkg/centos-7-rh-python38.dockerfile | 13 ++++++ 5 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 dockerfiles/vcpkg/centos-7-rh-python38.dockerfile diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index e8188c6b51a..c7f3c01211d 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -128,6 +128,10 @@ jobs: name: LN cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" ln: true + - os: centos-7-rh-python38 + name: LN Python 3.8 + cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic -DCUSTOM_LABEL=_rh_python38" + ln: true fail-fast: false steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 663d17c0648..2a01154a91c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,11 +340,11 @@ if(TOP_LEVEL_PROJECT) message("-- distro uses ${packageManagement}, revision is ${packageRevisionArch}") if("${packageManagement}" STREQUAL "DEB") - set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}${packageRevisionArch}${packageStrippedLabel}${packageContainerizedLabel}") + set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}${packageRevisionArch}${packageStrippedLabel}${packageContainerizedLabel}${CUSTOM_LABEL}") elseif("${packageManagement}" STREQUAL "RPM") - set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}.${packageRevisionArch}${packageStrippedLabel}${packageContainerizedLabel}") + set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}.${packageRevisionArch}${packageStrippedLabel}${packageContainerizedLabel}${CUSTOM_LABEL}") else() - set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}_${stagever}${CPACK_SYSTEM_NAME}${packageStrippedLabel}${packageContainerizedLabel}") + set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}_${stagever}${CPACK_SYSTEM_NAME}${packageStrippedLabel}${packageContainerizedLabel}${CUSTOM_LABEL}") endif() endif () @@ -500,7 +500,7 @@ if(TOP_LEVEL_PROJECT) set(CPACK_GENERATOR "productbuild") endif () if(APPLE OR WIN32) - set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${version}-${stagever}${CPACK_SYSTEM_NAME}${packageStrippedLabel}${packageContainerizedLabel}") + set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${version}-${stagever}${CPACK_SYSTEM_NAME}${packageStrippedLabel}${packageContainerizedLabel}${CUSTOM_LABEL}") endif() file(WRITE "${PROJECT_BINARY_DIR}/welcome.txt" "HPCC Systems® - Client Tools\r" diff --git a/cmake_modules/options.cmake b/cmake_modules/options.cmake index 0d1dbc1f677..2d7d3e91a72 100644 --- a/cmake_modules/options.cmake +++ b/cmake_modules/options.cmake @@ -70,6 +70,8 @@ option(INSTALL_VCPKG_CATALOG "Install vcpkg-catalog.txt" ON) option(PORTALURL "Set url to hpccsystems portal download page") option(PROFILING "Set to true if planning to profile so stacks are informative" OFF) +set(CUSTOM_LABEL "" CACHE STRING "Appends a custom label to the final package name") + if ( NOT PORTALURL ) set( PORTALURL "http://hpccsystems.com/download" ) endif() diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh index f314770edc2..074e61bc62b 100755 --- a/dockerfiles/vcpkg/build.sh +++ b/dockerfiles/vcpkg/build.sh @@ -12,6 +12,7 @@ cd vcpkg VCPKG_REF=$(git rev-parse --short=8 HEAD) cd .. GITHUB_BRANCH=$(git log -50 --pretty=format:"%D" | tr ',' '\n' | grep 'upstream/' | awk 'NR==1 {sub("upstream/", ""); print}' | xargs) +GITHUB_BRANCH=${GITHUB_BRANCH:-master} DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}" DOCKER_PASSWORD="${DOCKER_PASSWORD:-none}" @@ -26,7 +27,9 @@ echo "DOCKER_PASSWORD: $DOCKER_PASSWORD" docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD -CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON" +CMAKE_ALL_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DINCLUDE_PLUGINS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" +CMAKE_OPENBLAS_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF -DECLBLAS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" +CMAKE_PLATFORM_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DPLATFORM=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" function doBuild() { docker pull "hpccsystems/platform-build-base-$1:$VCPKG_REF" || true @@ -46,7 +49,7 @@ function doBuild() { # docker push hpccsystems/platform-build-$1:$GITHUB_BRANCH CMAKE_OPTIONS_EXTRA="" - if [ "$1" == "centos-7" ]; then + if [ "$1" == "centos-7*" ]; then CMAKE_OPTIONS_EXTRA="-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" elif [ "$1" == "amazonlinux" ]; then CMAKE_OPTIONS_EXTRA="-DVCPKG_TARGET_TRIPLET=x64-amazonlinux-dynamic" @@ -54,22 +57,35 @@ function doBuild() { mkdir -p $HOME/.ccache docker run --rm \ --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached \ + --mount source="$(realpath ~)/.cache/vcpkg",target=/root/.cache/vcpkg,type=bind,consistency=cached \ --mount source="$HOME/.ccache",target=/root/.ccache,type=bind,consistency=cached \ hpccsystems/platform-build-$1:$VCPKG_REF \ - "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS} ${CMAKE_OPTIONS_EXTRA} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && \ - cmake --build /hpcc-dev/HPCC-Platform/build-$1 --target install --parallel $(nproc) && \ - /etc/init.d/hpcc-init start" + "rm -rf /hpcc-dev/HPCC-Platform/build-$1/CMakeCache.txt /hpcc-dev/HPCC-Platform/build-$1/CMakeFiles && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_ALL_OPTIONS} ${CMAKE_OPTIONS_EXTRA} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && \ + cmake --build /hpcc-dev/HPCC-Platform/build-$1 --parallel && \ + echo 'Done'" - sudo chown -R $(id -u):$(id -g) ./build-$1 +# sudo chown -R $(id -u):$(id -g) ./build-$1 # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash } -# doBuild ubuntu-23.10 -# doBuild ubuntu-20.04 -# doBuild amazonlinux -# doBuild ubuntu-22.04 -# doBuild centos-8 -doBuild centos-7 +trap 'kill $(jobs -p)' EXIT + +# ./vcpkg/bootstrap-vcpkg.sh +mkdir -p ./vcpkg-logs + +if [ "$1" != "" ]; then + doBuild $1 & +else + # doBuild ubuntu-24.04 &> vcpkg-logs/ubuntu-24.04.log & + doBuild ubuntu-22.04 &> vcpkg-logs/ubuntu-22.04.log & + doBuild ubuntu-20.04 &> vcpkg-logs/ubuntu-20.04.log & + # doBuild rockylinux-8 &> vcpkg-logs/rockylinux-8.log & + doBuild centos-8 &> vcpkg-logs/centos-8.log & + doBuild amazonlinux &> vcpkg-logs/amazonlinux.log & + doBuild centos-7-rh-python38 &> vcpkg-logs/centos-7-rh-python38.log & + doBuild centos-7 &> vcpkg-logs/centos-7.log & +fi wait diff --git a/dockerfiles/vcpkg/centos-7-rh-python38.dockerfile b/dockerfiles/vcpkg/centos-7-rh-python38.dockerfile new file mode 100644 index 00000000000..8752d764d67 --- /dev/null +++ b/dockerfiles/vcpkg/centos-7-rh-python38.dockerfile @@ -0,0 +1,13 @@ +ARG VCPKG_REF=latest +FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF + +RUN yum remove -y python3 python3-devel && \ + yum install -y rh-python38 rh-python38-python-devel && \ + yum clean all + +RUN echo "source /opt/rh/rh-python38/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["/bin/bash"]