From 3fbcc43a7d1b6c5a26ee58a3223c6d73622ad06f Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 25 Mar 2024 17:53:25 +0000 Subject: [PATCH] HPCC-31524 Update package dependencies Added missing noble + amzn2 files Updated detection code to support rocky linux Signed-off-by: Gordon Smith --- cmake_modules/dependencies/amzn2.cmake | 2 ++ cmake_modules/dependencies/noble.cmake | 2 ++ cmake_modules/getpackagerevisionarch.sh | 2 +- dockerfiles/vcpkg/build.sh | 46 +++++++++++++++++-------- 4 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 cmake_modules/dependencies/amzn2.cmake create mode 100644 cmake_modules/dependencies/noble.cmake diff --git a/cmake_modules/dependencies/amzn2.cmake b/cmake_modules/dependencies/amzn2.cmake new file mode 100644 index 00000000000..329026728f7 --- /dev/null +++ b/cmake_modules/dependencies/amzn2.cmake @@ -0,0 +1,2 @@ +# Amazon Linux 2 +SET_DEPENDENCIES ( CPACK_RPM_PACKAGE_REQUIRES m4 libtool gcc-c++ openssh-server openssh-clients expect rsync zip psmisc curl jq ) diff --git a/cmake_modules/dependencies/noble.cmake b/cmake_modules/dependencies/noble.cmake new file mode 100644 index 00000000000..30bce954f41 --- /dev/null +++ b/cmake_modules/dependencies/noble.cmake @@ -0,0 +1,2 @@ +# Ubuntu 22.04 +SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS g++ openssh-client openssh-server expect rsync python3 psmisc curl jq ) diff --git a/cmake_modules/getpackagerevisionarch.sh b/cmake_modules/getpackagerevisionarch.sh index 708c06f387d..63e378246b9 100755 --- a/cmake_modules/getpackagerevisionarch.sh +++ b/cmake_modules/getpackagerevisionarch.sh @@ -85,7 +85,7 @@ elif [ -e /etc/redhat-release ]; then OS_GROUP=`/bin/rpm -q --qf "%{NAME}" --whatprovides /etc/redhat-release | sed 's/-release.*//' | tr '[A-Z]' '[a-z]'` REDHAT_VERSION=`/bin/rpm -q --qf "%{VERSION}" --whatprovides /etc/redhat-release | cut -f1 -d"."` case "$OS_GROUP" in - "centos"* | "fedora") + "centos"* | "fedora" | "rocky") if [ ${NOARCH} -eq 0 ]; then OUTPUT="el${REDHAT_VERSION}.${ARCH}" else diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh index 68b33a8cd45..c15e8a2f2f3 100755 --- a/dockerfiles/vcpkg/build.sh +++ b/dockerfiles/vcpkg/build.sh @@ -12,6 +12,7 @@ cd vcpkg VCPKG_REF=$(git rev-parse --short=8 HEAD) cd .. GITHUB_BRANCH=$(git log -50 --pretty=format:"%D" | tr ',' '\n' | grep 'upstream/' | awk 'NR==1 {sub("upstream/", ""); print}' | xargs) +GITHUB_BRANCH=${GITHUB_BRANCH:-master} DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}" DOCKER_PASSWORD="${DOCKER_PASSWORD:-none}" @@ -26,12 +27,14 @@ echo "DOCKER_PASSWORD: $DOCKER_PASSWORD" docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD -CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON" +CMAKE_ALL_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DINCLUDE_PLUGINS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" +CMAKE_OPENBLAS_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF -DECLBLAS=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" +CMAKE_PLATFORM_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON -DPLATFORM=ON -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" function doBuild() { - docker pull "hpccsystems/platform-build-base-$1:$VCPKG_REF" || true - docker pull "hpccsystems/platform-build-$1:$VCPKG_REF" || true - docker pull "hpccsystems/platform-build-$1:$GITHUB_BRANCH" || true + # docker pull "hpccsystems/platform-build-base-$1:$VCPKG_REF" || true + # docker pull "hpccsystems/platform-build-$1:$VCPKG_REF" || true + # docker pull "hpccsystems/platform-build-$1:$GITHUB_BRANCH" || true docker build --progress plain --rm -f "$SCRIPT_DIR/$1.dockerfile" \ --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \ @@ -46,7 +49,7 @@ function doBuild() { # docker push hpccsystems/platform-build-$1:$GITHUB_BRANCH CMAKE_OPTIONS_EXTRA="" - if [ "$1" == "centos-7" ]; then + if [ "$1" == "centos-7*" ]; then CMAKE_OPTIONS_EXTRA="-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" elif [ "$1" == "amazonlinux" ]; then CMAKE_OPTIONS_EXTRA="-DVCPKG_TARGET_TRIPLET=x64-amazonlinux-dynamic" @@ -54,22 +57,35 @@ function doBuild() { mkdir -p $HOME/.ccache docker run --rm \ --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached \ + --mount source="$(realpath ~)/.cache/vcpkg",target=/root/.cache/vcpkg,type=bind,consistency=cached \ --mount source="$HOME/.ccache",target=/root/.ccache,type=bind,consistency=cached \ hpccsystems/platform-build-$1:$VCPKG_REF \ - "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS} ${CMAKE_OPTIONS_EXTRA} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && \ - cmake --build /hpcc-dev/HPCC-Platform/build-$1 --target install --parallel $(nproc) && \ - /etc/init.d/hpcc-init start" + "rm -rf /hpcc-dev/HPCC-Platform/build-$1/CMakeCache.txt /hpcc-dev/HPCC-Platform/build-$1/CMakeFiles && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_ALL_OPTIONS} ${CMAKE_OPTIONS_EXTRA} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && \ + cmake --build /hpcc-dev/HPCC-Platform/build-$1 --parallel && \ + echo 'Done'" - sudo chown -R $(id -u):$(id -g) ./build-$1 +# sudo chown -R $(id -u):$(id -g) ./build-$1 # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash } -# doBuild ubuntu-20.04 -# doBuild rockylinux-8 & -# doBuild amazonlinux -# doBuild ubuntu-22.04 -# doBuild centos-8 -# doBuild centos-7 +trap 'kill $(jobs -p)' EXIT + +# ./vcpkg/bootstrap-vcpkg.sh +mkdir -p ./vcpkg-logs + +if [ "$1" != "" ]; then + doBuild $1 & +else + doBuild ubuntu-24.04 &> vcpkg-logs/ubuntu-24.04.log & + doBuild ubuntu-22.04 &> vcpkg-logs/ubuntu-22.04.log & + doBuild ubuntu-20.04 &> vcpkg-logs/ubuntu-20.04.log & + doBuild rockylinux-8 &> vcpkg-logs/rockylinux-8.log & + doBuild centos-8 &> vcpkg-logs/centos-8.log & + doBuild amazonlinux &> vcpkg-logs/amazonlinux.log & + doBuild centos-7-rh-python38 &> vcpkg-logs/centos-7-rh-python38.log & + doBuild centos-7 &> vcpkg-logs/centos-7.log & +fi wait