diff --git a/README.md b/README.md index 16ff5c6..24a40c1 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Running the testsuite for one container Use a command similar to that one: - docker run --rm -t -i -v $PWD/run-testsuite.sh:/mnt/run-testsuite.sh:z -v $HOME/Git/cgal-master:/mnt/testsuite:ro,z -v $PWD/testresults:/mnt/testresults:z cgal/testsuite-docker:debian-testing bash -x /mnt/run-testsuite.sh + docker run --rm -t -i -v $HOME/Git/cgal-master:/mnt/testsuite:ro,z -v $PWD/docker-entrypoint.sh:/mnt/testsuite/docker-entrypoint.sh:ro,z -v $PWD/run-testsuite.sh:/mnt/testsuite/run-testsuite.sh:z -v $PWD/testresults:/mnt/testresults:z cgal/testsuite-docker:debian-testing bash /mnt/testsuite/docker-entrypoint.sh If you want a limited testsuite, you can have a file `testresults/list_test_packages` containing something like: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index b7c1c9c..6219a44 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -15,4 +15,8 @@ mkdir -p /home/cgal_tester chown cgal_tester:cgal_tester /home/cgal_tester cd /home/cgal_tester -su cgal_tester -c '/bin/bash /mnt/testsuite/run-testsuite.sh' +CMD="/bin/bash -$- /mnt/testsuite/run-testsuite.sh" +if [ -n "$1" ]; then + CMD="$@" +fi +su cgal_tester -c "$CMD" diff --git a/run-testsuite.sh b/run-testsuite.sh index ef2f2a4..79a47e2 100755 --- a/run-testsuite.sh +++ b/run-testsuite.sh @@ -1,4 +1,12 @@ #!/bin/bash + +# prompt for the xtrace output +PS4='+ $(date "+%Y-%m-%d %H:%M:%S") ${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' +# use a new fd (chosen by bash) to redirect the xtrace output +exec {fd}>&2 +BASH_XTRACEFD=${fd} +unset fd + set -e if [ -z "${CGAL_TESTER}" ]; then @@ -11,8 +19,6 @@ if [ -z "${CGAL_TEST_PLATFORM}" ]; then echo "CGAL_TEST_PLATFORM not set. Using HOSTNAME: ${HOSTNAME}" fi -# HACK: We depend on this line to easily extract the platform name -# from the logs. echo "CGAL_TEST_PLATFORM=${CGAL_TEST_PLATFORM}" if [ -z "${CGAL_NUMBER_OF_JOBS}" ]; then @@ -22,42 +28,47 @@ else echo "CGAL_NUMBER_OF_JOBS is ${CGAL_NUMBER_OF_JOBS}." fi -declare -a "CGAL_CMAKE_FLAGS=${CGAL_CMAKE_FLAGS}" -echo "CGAL_CMAKE_FLAGS is ${CGAL_CMAKE_FLAGS[@]}." - # The directory where the release is stored. -export CGAL_RELEASE_DIR="/mnt/testsuite/" +export CGAL_RELEASE_DIR="/mnt/testsuite" # Directory where CGAL sources are stored. -CGAL_SRC_DIR="${CGAL_RELEASE_DIR}src/" +CGAL_SRC_DIR="${CGAL_RELEASE_DIR}/src" # Directory where CGAL tests are stored. -CGAL_TEST_DIR="${CGAL_RELEASE_DIR}test/" +CGAL_TEST_DIR="${CGAL_RELEASE_DIR}/test" # Directory where CGAL data are stored. -CGAL_DATA_DIR="${CGAL_RELEASE_DIR}data/" +CGAL_DATA_DIR="${CGAL_RELEASE_DIR}/data" # Directory where include/CGAL/version.h can be found. CGAL_VERSION_DIR="${CGAL_RELEASE_DIR}" -if ! [ -f "${CGAL_VERSION_DIR}include/CGAL/version.h" ]; then - CGAL_VERSION_DIR="${CGAL_RELEASE_DIR}Installation/" +if ! [ -f "${CGAL_VERSION_DIR}/include/CGAL/version.h" ]; then + CGAL_VERSION_DIR="${CGAL_RELEASE_DIR}/Installation" fi # Directory Testsuite (if branch build), or CGAL_RELEASE_DIR -CGAL_TESTSUITE_DIR="${CGAL_RELEASE_DIR}Testsuite/" +CGAL_TESTSUITE_DIR="${CGAL_RELEASE_DIR}/Testsuite" if ! [ -d "${CGAL_TESTSUITE_DIR}" ]; then CGAL_TESTSUITE_DIR="${CGAL_RELEASE_DIR}" fi # The directory where testresults are stored. -CGAL_TESTRESULTS="/mnt/testresults/" +CGAL_TESTRESULTS="/mnt/testresults" -CGAL_DIR="$HOME/build/src/cmake/platforms/${CGAL_TEST_PLATFORM}/" +CGAL_DIR="$HOME/build/src/cmake/platforms/${CGAL_TEST_PLATFORM}" # The directory where the release is built. CGAL_SRC_BUILD_DIR="${CGAL_DIR}" # The directory where the tests are built. -CGAL_TEST_BUILD_DIR="$HOME/build/src/cmake/platforms/${CGAL_TEST_PLATFORM}/test/" +CGAL_TEST_BUILD_DIR="$HOME/build/src/cmake/platforms/${CGAL_TEST_PLATFORM}/test" PLATFORM="$CGAL_TEST_PLATFORM" export CGAL_DIR export CGAL_TEST_PLATFORM export CGAL_DATA_DIR +# If the option xtrace (`set -x`) is not set, +# then set it, and redirect all trace output to a log file. +BASH_LOG_FILE=${CGAL_TESTRESULTS}/bash-xtrace-${CGAL_TEST_PLATFORM}.log +case "$-" in + *x*) ;; + *) echo "Redirecting xtrace output to ${BASH_LOG_FILE}"; eval "exec ${BASH_XTRACEFD}>${BASH_LOG_FILE}"; set -x ;; +esac + # Create the binary directories if [ ! -d "${CGAL_SRC_BUILD_DIR}" ]; then mkdir -p "${CGAL_SRC_BUILD_DIR}" @@ -66,26 +77,45 @@ if [ ! -d "${CGAL_TEST_BUILD_DIR}" ]; then mkdir -p "${CGAL_TEST_BUILD_DIR}" fi -# Build CGAL. The CGAL_CMAKE_FLAGS used here will affect all other -# builds using this binary directory. +CMAKE_LOG_FILE=${CGAL_TESTRESULTS}/cmake-${CGAL_TEST_PLATFORM}.log +CTEST_LOG_FILE=${CGAL_TESTRESULTS}/ctest-${CGAL_TEST_PLATFORM}.log + +rm -f "${CMAKE_LOG_FILE}" "${CTEST_LOG_FILE}" + +REDIRECT_TO() { + exec 3>&1 4>&2 + if [ -z "${SHOW_PROGRESS}" ]; then + echo "Redirecting stdout and stderr to $1" + exec > "$1" 2>&1 + else + exec > >(tee ${1}) 2>&1 + fi + _REDIRECTED=1 +} + +REVERT_REDIRECTIONS() { + [ -z "$_REDIRECTED" ] && return + exec 1>&3 3>&- 2>&4 4>&- + [ -z "${SHOW_PROGRESS}" ] && echo "Reverting redirections of stdout and stderr" + unset _REDIRECTED +} cd "${CGAL_SRC_BUILD_DIR}" + +REDIRECT_TO "${CMAKE_LOG_FILE}" + if [ -n "$DOCKERFILE_URL" ]; then - echo "Docker image built from ${DOCKERFILE_URL}" | tee "${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log" + echo "Docker image built from ${DOCKERFILE_URL}" else - echo "Docker container" > ${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log + echo "Docker container" fi -cmake ${INIT_FILE:+"-C${INIT_FILE}"} -DBUILD_TESTING=ON -DWITH_tests=ON -DCGAL_TEST_SUITE=ON $CGAL_RELEASE_DIR >${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log 2>&1 || : -rm CMakeCache.txt -CMAKE_OPTS="-DCGAL_TEST_SUITE=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DWITH_tests=ON" -if [ -z "${SHOW_PROGRESS}" ]; then - cmake ${INIT_FILE:+"-C${INIT_FILE}"} -DBUILD_TESTING=ON ${CMAKE_OPTS} $CGAL_RELEASE_DIR >${CGAL_TESTRESULTS}package_installation-${CGAL_TEST_PLATFORM}.log 2>&1 || : -else - cmake ${INIT_FILE:+"-C${INIT_FILE}"} -DBUILD_TESTING=ON ${CMAKE_OPTS} $CGAL_RELEASE_DIR 2>&1 |tee ${CGAL_TESTRESULTS}package_installation-${CGAL_TEST_PLATFORM}.log || : -fi +# Configure CGAL. +cmake ${INIT_FILE:+"-C${INIT_FILE}"} -DBUILD_TESTING=ON -DCGAL_TEST_SUITE=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DWITH_tests=ON "$CGAL_RELEASE_DIR" || _CMAKE_ERROR=$? + +REVERT_REDIRECTIONS -LIST_TEST_FILE="${CGAL_TESTRESULTS}list_test_packages" +LIST_TEST_FILE="${CGAL_TESTRESULTS}/list_test_packages" if [ -f ${LIST_TEST_FILE} ]; then LIST_TEST_PACKAGES=$(source ${LIST_TEST_FILE}) fi @@ -98,40 +128,56 @@ for pkg in $LIST_TEST_PACKAGES; do TO_TEST="${TO_TEST}|$pkg" fi done -#unsets the limit of 1024 bits for the logs through ssh -echo "SET(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1000000000)" > CTestCustom.cmake -echo "SET(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1000000000)" >> CTestCustom.cmake + +# unset the limits of 1 KiB for the logs and set them to 1 GB instead +cat < CTestCustom.cmake +set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1000000000) +set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1000000000) +EOF + +# add a configuration file for the tests (required since CMake-3.32) +cat < CTestConfiguration.ini +SourceDirectory: ${CGAL_RELEASE_DIR} +BuildDirectory: ${CGAL_SRC_BUILD_DIR} +EOF + CTEST_OPTS="-T Start -T Test --timeout 1200 ${DO_NOT_TEST:+-E execution___of__}" -if [ -z "${SHOW_PROGRESS}" ]; then - ctest ${TO_TEST:+-L ${TO_TEST} } ${CTEST_OPTS} -j${CGAL_NUMBER_OF_JOBS} ${KEEP_TESTS:+-FC .}>${CGAL_TESTRESULTS}ctest-${CGAL_TEST_PLATFORM}.log || : + +if [ -z "${_CMAKE_ERROR}" ]; then + REDIRECT_TO "${CTEST_LOG_FILE}" + ctest ${TO_TEST:+-L ${TO_TEST} } ${CTEST_OPTS} -j${CGAL_NUMBER_OF_JOBS} ${KEEP_TESTS:+-FC .} || true + REVERT_REDIRECTIONS + TAG_DIR=$(awk '/tag: /{print $4F}' "${CTEST_LOG_FILE}") else - ctest ${TO_TEST:+-L ${TO_TEST} } ${CTEST_OPTS} -j${CGAL_NUMBER_OF_JOBS} ${KEEP_TESTS:+-FC .}|tee ${CGAL_TESTRESULTS}ctest-${CGAL_TEST_PLATFORM}.log || : + TAG_DIR="error" + mkdir -p Testing/${TAG_DIR} fi -TAG_DIR=$(awk '/^Create new tag: /{print $4F}' ${CGAL_TESTRESULTS}ctest-${CGAL_TEST_PLATFORM}.log) cd Testing/${TAG_DIR} -RESULT_FILE=./"results_${CGAL_TESTER}_${PLATFORM}.txt" -rm -f "$RESULT_FILE" +RESULT_FILE=results_${CGAL_TESTER}_${PLATFORM}.txt +rm -f ./"$RESULT_FILE" touch "$RESULT_FILE" -sed -n '/The CXX compiler/s/-- The CXX compiler identification is/COMPILER_VERSION =/p' < "${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log" |sed -E "s/ = (.*)/\ = '\1\'/">> "$RESULT_FILE" -sed -n '/CGAL_VERSION /s/#define //p' < "${CGAL_VERSION_DIR}include/CGAL/version.h" >> "$RESULT_FILE" +sed -n '/The CXX compiler/s/-- The CXX compiler identification is/COMPILER_VERSION =/p' < "${CMAKE_LOG_FILE}" |sed -E "s/ = (.*)/\ = '\1\'/">> "$RESULT_FILE" +sed -n '/CGAL_VERSION /s/#define //p' < "${CGAL_VERSION_DIR}/include/CGAL/version.h" >> "$RESULT_FILE" echo "CGAL_SUMMARY_NAME ${CGAL_SUMMARY_NAME:-$CGAL_TEST_PLATFORM}" >> "$RESULT_FILE" echo "TESTER ${CGAL_TESTER}" >> "$RESULT_FILE" echo "TESTER_NAME ${CGAL_TESTER_NAME}" >> "$RESULT_FILE" echo "TESTER_ADDRESS ${CGAL_TESTER_ADDRESS}" >> "$RESULT_FILE" echo "CGAL_TEST_PLATFORM ${PLATFORM}" >> "$RESULT_FILE" -grep -e "^-- Operating system: " "${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log"|sort -u >> $RESULT_FILE -grep -e "^-- USING " "${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log"|sort -u >> $RESULT_FILE -sed -n '/^-- Third-party library /p' "${CGAL_TESTRESULTS}installation-${CGAL_TEST_PLATFORM}.log" >> $RESULT_FILE -#Use sed to get the content of DEBUG or RELEASE CXX FLAGS so that Multiconfiguration platforms do provide their CXXXFLAGS to the testsuite page (that greps USING CXXFLAGS to get info) -sed -i -E 's/(^-- USING )(DEBUG|RELEASE) (CXXFLAGS)/\1\3/' $RESULT_FILE +grep -e "^-- Operating system: " "${CMAKE_LOG_FILE}"|sort -u >> "$RESULT_FILE" +grep -e "^-- USING " "${CMAKE_LOG_FILE}"|sort -u >> "$RESULT_FILE" +sed -n '/^-- Third-party library /p' "${CMAKE_LOG_FILE}" >> "$RESULT_FILE" +# Use sed to get the content of DEBUG or RELEASE CXX FLAGS so that Multi-config platforms do provide their CXXFLAGS to the testsuite page (that greps USING CXXFLAGS to get info) +sed -i -E 's/(^-- USING )(DEBUG|RELEASE) (CXXFLAGS)/\1\3/' "$RESULT_FILE" echo "------------" >> "$RESULT_FILE" touch ../../../../../.scm-branch if [ -f /mnt/testsuite/.scm-branch ]; then cat /mnt/testsuite/.scm-branch >> ../../../../../.scm-branch fi -python3 ${CGAL_TESTSUITE_DIR}test/parse-ctest-dashboard-xml.py $CGAL_TESTER $PLATFORM +if [ -z "${_CMAKE_ERROR}" ]; then + python3 "${CGAL_TESTSUITE_DIR}/test/parse-ctest-dashboard-xml.py" "$CGAL_TESTER" "$PLATFORM" +fi for file in $(ls|grep _Tests); do mv $file "$(echo "$file" | sed 's/_Tests//g')" @@ -140,13 +186,22 @@ OUTPUT_FILE=results_${CGAL_TESTER}_${PLATFORM}.tar TEST_REPORT="TestReport_${CGAL_TESTER}_${PLATFORM}" mkdir -p Installation chmod 777 Installation -cat "${CGAL_TESTRESULTS}package_installation-${CGAL_TEST_PLATFORM}.log" >> "Installation/${TEST_REPORT}" +cat "${CMAKE_LOG_FILE}" >> "Installation/${TEST_REPORT}" #call the python script to complete the results report. -python3 ${CGAL_TESTSUITE_DIR}test/post_process_ctest_results.py Installation/${TEST_REPORT} ${TEST_REPORT} results_${CGAL_TESTER}_${PLATFORM}.txt -rm -f $OUTPUT_FILE $OUTPUT_FILE.gz +if [ -z "${_CMAKE_ERROR}" ]; then + python3 "${CGAL_TESTSUITE_DIR}/test/post_process_ctest_results.py" "Installation/${TEST_REPORT}" "${TEST_REPORT}" "$RESULT_FILE" +else + REDIRECT_TO "Installation/${TEST_REPORT}" + printf "CMake configuration failed\n\nHere is the CMake log:\n\n" + cat "${CMAKE_LOG_FILE}" + printf "\n-----------------------\nHere is the Bash log:\n\n" + cat "${BASH_LOG_FILE}" + REVERT_REDIRECTIONS +fi +rm -f ./"$OUTPUT_FILE" ./"$OUTPUT_FILE.gz" rm ../../../../../.scm-branch -tar cf $OUTPUT_FILE results_${CGAL_TESTER}_${PLATFORM}.txt */"$TEST_REPORT" +tar cf $OUTPUT_FILE "$RESULT_FILE" */"$TEST_REPORT" echo gzip -9f $OUTPUT_FILE -cp "${OUTPUT_FILE}.gz" "results_${CGAL_TESTER}_${PLATFORM}.txt" "${CGAL_TESTRESULTS}" +mv "${OUTPUT_FILE}.gz" "$RESULT_FILE" "${CGAL_TESTRESULTS}/" diff --git a/test.sh b/test.sh index 5ac8f0c..2feb5e0 100755 --- a/test.sh +++ b/test.sh @@ -40,7 +40,7 @@ function dockerbuildandtest() { echo "::group::Test image $1" docker run --rm -v $PWD/cgal:/cgal cgal/testsuite-docker:$1 bash -c 'cmake -DWITH_examples=ON -S /cgal -B /build && cmake --build /build -t terrain -v' if command -v python3 >/dev/null; then - python3 ./test_container/test_container.py --image cgal/testsuite-docker:$1 --cgal-dir $HOME/cgal + python3 ./test_container/test_container.py --image cgal/testsuite-docker:$1 --cgal-dir $PWD/cgal ${DOCKER_HOST:+--docker-url "${DOCKER_HOST}"} fi echo '::endgroup::' } diff --git a/test_container/CMakeLists.txt b/test_container/CMakeLists.txt index 5a5076b..1a1e7f6 100644 --- a/test_container/CMakeLists.txt +++ b/test_container/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.1...3.26) +cmake_minimum_required(VERSION 3.1...3.30) project(TestCGAL C CXX) # To Enable Compiler Checks # We output results at the end, because some Find scripts do not diff --git a/test_container/run-test.sh b/test_container/run-test.sh index 72f02d6..99029a9 100755 --- a/test_container/run-test.sh +++ b/test_container/run-test.sh @@ -1,5 +1,5 @@ -#!/bin/bash -l +#!/bin/sh -mkdir /cgal_test && cd /cgal_test -cp /mnt/test/CMakeLists.txt . -cmake . +set -e + +cmake -S /mnt/test -B /cgal_test diff --git a/test_container/test_container.py b/test_container/test_container.py index b7d7063..a7f02ca 100755 --- a/test_container/test_container.py +++ b/test_container/test_container.py @@ -37,6 +37,12 @@ def main(): else: chosen_name = 'CGAL-{}-test_container'.format(res.group(2)) + try: + docker_client.remove_container(container=chosen_name, force=True) + except docker.errors.NotFound: + pass + except docker.errors.APIError as e: + logging.warning("Failed to remove container %s: %s", chosen_name, e) container = docker_client.create_container( image=args.image, name=chosen_name, @@ -74,10 +80,17 @@ def main(): # our container died, time to print the log break - log = docker_client.logs(container=container) + log = docker_client.logs(container=container).decode('utf-8') for line in log.splitlines(): print(line) + exit_code = docker_client.inspect_container(container['Id'])['State']['ExitCode'] + if exit_code != 0: + logging.error('Container exited with code {}'.format(exit_code)) + exit(exit_code) + else: + logging.info('Container exited successfully') + if __name__ == "__main__": main() diff --git a/testresults/init.cmake b/testresults/init.cmake deleted file mode 100644 index 4fefad8..0000000 --- a/testresults/init.cmake +++ /dev/null @@ -1,4 +0,0 @@ -SET(CMAKE_CXX_FLAGS "-Wall -frounding-math" CACHE STRING "") -SET(CMAKE_CXX_FLAGS_DEBUG "" CACHE STRING "") -SET(CMAKE_CXX_FLAGS_RELEASE "-DCGAL_NDEBUG -O3" CACHE STRING "") -SET(CMAKE_BUILD_TYPE "Release" CACHE STRING "") diff --git a/testsuite/init.cmake b/testsuite/init.cmake deleted file mode 100644 index 4fefad8..0000000 --- a/testsuite/init.cmake +++ /dev/null @@ -1,4 +0,0 @@ -SET(CMAKE_CXX_FLAGS "-Wall -frounding-math" CACHE STRING "") -SET(CMAKE_CXX_FLAGS_DEBUG "" CACHE STRING "") -SET(CMAKE_CXX_FLAGS_RELEASE "-DCGAL_NDEBUG -O3" CACHE STRING "") -SET(CMAKE_BUILD_TYPE "Release" CACHE STRING "")