From a1b46416b2ce0c3c8465645beaeac3c9fa512f95 Mon Sep 17 00:00:00 2001 From: Edwin Lee Date: Tue, 27 Aug 2024 16:12:39 -0500 Subject: [PATCH] Remove Decent CI stuff, add develop build, cleanup alternate build --- .decent_ci-Linux.yaml | 34 ---- .decent_ci-MacOS.yaml | 3 - .decent_ci-Windows.yaml | 6 - .github/workflows/alternate_linux_build.yml | 44 +++++ .github/workflows/build_and_test_develop.yml | 104 ++++++++++++ ...ild_and_test.yml => build_and_test_pr.yml} | 4 +- .github/workflows/linux_build.yml | 84 ---------- .gitlab-ci.yml | 157 ------------------ .renamed.yaml | 12 -- CMakeLists.txt | 41 ----- cmake/ProjectMacros.cmake | 19 --- cmake/RunRegression.cmake | 17 -- scripts/dev/gha_regressions.py | 38 ++--- 13 files changed, 169 insertions(+), 394 deletions(-) delete mode 100644 .decent_ci-Linux.yaml delete mode 100644 .decent_ci-MacOS.yaml delete mode 100644 .decent_ci-Windows.yaml create mode 100644 .github/workflows/alternate_linux_build.yml create mode 100644 .github/workflows/build_and_test_develop.yml rename .github/workflows/{build_and_test.yml => build_and_test_pr.yml} (95%) delete mode 100644 .github/workflows/linux_build.yml delete mode 100644 .gitlab-ci.yml delete mode 100644 .renamed.yaml delete mode 100644 cmake/RunRegression.cmake diff --git a/.decent_ci-Linux.yaml b/.decent_ci-Linux.yaml deleted file mode 100644 index 05031b75d59..00000000000 --- a/.decent_ci-Linux.yaml +++ /dev/null @@ -1,34 +0,0 @@ -compilers: - - name: "gcc" - version: "11.4" - cmake_extra_flags: -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=3.12.2 -DPython_ROOT_DIR:PATH=~/.pyenv/versions/3.12.2/ -DBUILD_FORTRAN:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=$REGRESSION_BASELINE -DREGRESSION_SCRIPT_PATH:PATH=$REGRESSION_DIR -DREGRESSION_BASELINE_SHA:STRING=$REGRESSION_BASELINE_SHA -DCOMMIT_SHA:STRING=$COMMIT_SHA -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DBUILD_PERFORMANCE_TESTS:BOOL=ON -DVALGRIND_ANALYZE_PERFORMANCE_TESTS:BOOL=ON -DENABLE_PCH:BOOL=OFF - collect_performance_results: true - s3_upload_bucket: energyplus - - - name: "gcc" - version: "11.4" - build_type: Debug - cmake_extra_flags: -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=3.12.2 -DPython_ROOT_DIR:PATH=~/.pyenv/versions/3.12.2/ -DBUILD_FORTRAN:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=OFF -DCOMMIT_SHA:STRING=$COMMIT_SHA -DENABLE_COVERAGE:BOOL=ON -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DENABLE_PCH:BOOL=OFF - coverage_enabled: true - coverage_base_dir: src/EnergyPlus - coverage_pass_limit: 41.0 - coverage_warn_limit: 40.0 - coverage_s3_bucket: energyplus - build_tag: UnitTestsCoverage - ctest_filter: -E "integration.*" - skip_regression: true - skip_packaging: true - - - name: "gcc" - version: "11.4" - build_type: Debug - cmake_extra_flags: -DLINK_WITH_PYTHON:BOOL=ON -DPython_REQUIRED_VERSION:STRING=3.12.2 -DPython_ROOT_DIR:PATH=~/.pyenv/versions/3.12.2/ -DBUILD_FORTRAN:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=OFF -DCOMMIT_SHA:STRING=$COMMIT_SHA -DENABLE_COVERAGE:BOOL=ON -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DENABLE_PCH:BOOL=OFF - coverage_enabled: true - coverage_base_dir: src/EnergyPlus - coverage_pass_limit: 66.0 - coverage_warn_limit: 67.0 - coverage_s3_bucket: energyplus - build_tag: IntegrationCoverage - ctest_filter: -R "integration.*" - skip_regression: true - skip_packaging: true diff --git a/.decent_ci-MacOS.yaml b/.decent_ci-MacOS.yaml deleted file mode 100644 index 33ee61f4ab4..00000000000 --- a/.decent_ci-MacOS.yaml +++ /dev/null @@ -1,3 +0,0 @@ -compilers: - - name: clang - cmake_extra_flags: -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 -DBUILD_FORTRAN=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=$REGRESSION_BASELINE -DREGRESSION_SCRIPT_PATH:PATH=$REGRESSION_DIR -DREGRESSION_BASELINE_SHA:STRING=$REGRESSION_BASELINE_SHA -DCOMMIT_SHA=$COMMIT_SHA -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DLINK_WITH_PYTHON=ON -DPython_REQUIRED_VERSION:STRING=3.12.2 diff --git a/.decent_ci-Windows.yaml b/.decent_ci-Windows.yaml deleted file mode 100644 index 496d53df1f3..00000000000 --- a/.decent_ci-Windows.yaml +++ /dev/null @@ -1,6 +0,0 @@ -compilers: - - name: Visual Studio - version: 16 - architecture: Win64 - cmake_extra_flags: -DBUILD_FORTRAN:BOOL=ON -DBUILD_TESTING:BOOL=ON -DCOMMIT_SHA=%COMMIT_SHA% -DENABLE_GTEST_DEBUG_MODE:BOOL=OFF -DLINK_WITH_PYTHON=ON -DPython_EXECUTABLE:PATH=C:/Users/elee/AppData/Local/Programs/Python/Python312/python.exe - skip_regression: true diff --git a/.github/workflows/alternate_linux_build.yml b/.github/workflows/alternate_linux_build.yml new file mode 100644 index 00000000000..ee2e0dc17a9 --- /dev/null +++ b/.github/workflows/alternate_linux_build.yml @@ -0,0 +1,44 @@ +name: Alternate Linux Build + +on: + push: + branches: [ develop ] + # Sequence of patterns matched against refs/tags + tags: + - 'v*' # Push events matching v*, i.e. v1.0, v20.15.10 + pull_request: + branches: [ develop ] + +jobs: + + alternate_build_configurations: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set FC for Ubuntu 24.04 + shell: bash + run: echo "FC=gfortran-13" >> $GITHUB_ENV + + - name: Create Build Directory + run: cmake -E make_directory ./build/ + + - name: Configure CMake + shell: bash + working-directory: ./build/ + run: | + cmake -DCMAKE_BUILD_TYPE=Release \ + -DLINK_WITH_PYTHON=OFF \ + -DUSE_PSYCHROMETRICS_CACHING=OFF \ + -DUSE_GLYCOL_CACHING=OFF \ + -DOPENGL_REQUIRED=OFF \ + -DUSE_PSYCH_STATS=ON \ + -DUSE_PSYCH_ERRORS=OFF \ + -DENABLE_PCH=OFF \ + ../ + + - name: Build EnergyPlus + working-directory: ./build/ + shell: bash + run: cmake --build . --target energyplus -j 4 diff --git a/.github/workflows/build_and_test_develop.yml b/.github/workflows/build_and_test_develop.yml new file mode 100644 index 00000000000..840521bcb8f --- /dev/null +++ b/.github/workflows/build_and_test_develop.yml @@ -0,0 +1,104 @@ +name: Develop Testing + +on: + push: + branches: [ develop ] # run this on all commits to the develop branch + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FC: gfortran-13 + Python_REQUIRED_VERSION: 3.12.3 # 3.12.2 not available on Ubuntu 24 GHA + shell: bash + +jobs: + build_and_test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: macos-12 + macos_dev_target: 12.1 + arch: x86_64 + python-arch: x64 + generator: "Unix Makefiles" + nproc: 3 + - os: macos-14 + macos_dev_target: 13.0 + arch: arm64 + python-arch: arm64 + generator: "Unix Makefiles" + nproc: 3 + - os: ubuntu-24.04 + arch: x86_64 + python-arch: x64 + generator: "Unix Makefiles" + nproc: 4 + - os: windows-2019 + arch: x86_64 + python-arch: x64 + generator: "Visual Studio 16 2019" + nproc: 4 + - os: windows-2022 + arch: x86_64 + python-arch: x64 + generator: "Visual Studio 17 2022" + nproc: 4 + + steps: + + - name: Set up Python ${{ env.Python_REQUIRED_VERSION }} + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.Python_REQUIRED_VERSION }} + + - name: Install Dependencies for Mac + if: ${{ runner.os == 'macOS' }} + run: | + brew update + brew install gcc@13 + echo "FC=$(brew --prefix gcc@13)/bin/gfortran-13" >> $GITHUB_ENV + echo MACOSX_DEPLOYMENT_TARGET=${{ matrix.macos_dev_target }} >> $GITHUB_ENV + + - name: Install Dependencies for Linux + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get update + sudo apt-get install libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev + if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then + # https://github.com/actions/runner-images/issues/10025 + echo "FC=gfortran-13" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v4 + + - name: Create Build Directory + run: cmake -E make_directory ./build/ + + - name: Configure CMake + working-directory: ./build + run: > + cmake + -G "${{ matrix.generator }}" + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=${{ matrix.macos_dev_target }} + -DLINK_WITH_PYTHON:BOOL=ON + -DPython_REQUIRED_VERSION:STRING=${{ steps.setup-python.outputs.python-version }} + -DPython_ROOT_DIR:PATH=$RUNNER_TOOL_CACHE/Python/${{ steps.setup-python.outputs.python-version }}/${{ matrix.python-arch }}/ + -DBUILD_TESTING:BOOL=ON + -DBUILD_FORTRAN:BOOL=ON + -DBUILD_PACKAGE:BOOL=OFF + -DDOCUMENTATION_BUILD:STRING=DoNotBuild + -DENABLE_OPENMP:BOOL=OFF + -DUSE_OpenMP:BOOL=OFF + ../ + + - name: Build + id: build + working-directory: ./build + run: cmake --build . -j ${{ matrix.nproc }} --config Release + + - name: Test + working-directory: ./build + run: ctest -C Release -E Basement -j 3 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test_pr.yml similarity index 95% rename from .github/workflows/build_and_test.yml rename to .github/workflows/build_and_test_pr.yml index bf90b5076e8..9ee5305e6ff 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test_pr.yml @@ -1,4 +1,4 @@ -name: Build And Test +name: Pull Request Testing on: pull_request: @@ -191,5 +191,5 @@ jobs: issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: "# :warning: Regressions Detected\n - Build: ${{ matrix.os }}\n - Commit: ${{ github.action_ref }}\n - [View Results](https://github.com/NREL/EnergyPlus/actions/runs/${{ github.run_id }})\n - [Download Regressions](${{ steps.upload_regressions.outputs.artifact-url }})" + body: "### :warning: Regressions detected on ${{ matrix.os }} for commit ${{ github.sha }}\n $GITHUB_STEP_SUMMARY \n\n - [View Results](https://github.com/NREL/EnergyPlus/actions/runs/${{ github.run_id }})\n - [Download Regressions](${{ steps.upload_regressions.outputs.artifact-url }})" }) diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml deleted file mode 100644 index b5fddcc0a16..00000000000 --- a/.github/workflows/linux_build.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Linux Build - -on: - push: - branches: [ develop ] - # Sequence of patterns matched against refs/tags - tags: - - 'v*' # Push events matching v*, i.e. v1.0, v20.15.10 - pull_request: - branches: [ develop ] - -jobs: -# base_build: -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# os: [ubuntu-20.04] -# -# steps: -# - uses: actions/checkout@v4 -# -# - name: Set up Python 3.7 -# uses: actions/setup-python@v5 -# with: -# python-version: 3.7 -# -# - name: Install System Dependencies -# shell: bash -# run: | -# set -x -# echo "Using Apt to install dependencies" -# sudo apt-get update -# sudo apt-get install libxkbcommon-x11-0 xorg-dev libgl1-mesa-dev -# -# - name: Create Build Directory -# run: cmake -E make_directory ./build/ -# -# - name: Configure CMake -# shell: bash -# working-directory: ./build/ -# run: cmake -DCMAKE_BUILD_TYPE=Release -DLINK_WITH_PYTHON=ON .. -# -# - name: Build EnergyPlus -# working-directory: ./build/ -# shell: bash -# run: cmake --build . --target energyplus -j 2 - - alternate_build_configurations: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - - steps: - - uses: actions/checkout@v4 - - - name: Install System Dependencies - shell: bash - run: | - set -x - echo "Using Apt to install dependencies" - sudo apt-get update - - - name: Create Build Directory - run: cmake -E make_directory ./build/ - - - name: Configure CMake - shell: bash - working-directory: ./build/ - run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DLINK_WITH_PYTHON=OFF \ - -DUSE_PSYCHROMETRICS_CACHING=OFF \ - -DUSE_GLYCOL_CACHING=OFF \ - -DOPENGL_REQUIRED=OFF \ - -DUSE_PSYCH_STATS=ON \ - -DUSE_PSYCH_ERRORS=OFF \ - -DENABLE_PCH=OFF \ - ../ - - - name: Build EnergyPlus - working-directory: ./build/ - shell: bash - run: cmake --build . --target energyplus -j 4 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 734c4431f13..00000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,157 +0,0 @@ -stages: - - build - - test - - package - -build:mac: - stage: build - tags: - - mac - variables: - CC: /usr/bin/clang - CXX: /usr/bin/clang++ - CXXFLAGS: -stdlib=libc++ - CMAKE_OSX_ARCHITECTURES: x86_64 - MACOSX_DEPLOYMENT_TARGET: "10.7" - SDKROOT: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk - script: - - mkdir build - - cd build - - cmake -DBUILD_FORTRAN=ON -DBUILD_PACKAGE=ON -DBUILD_TESTING=ON -DENABLE_GTEST_DEBUG_MODE=OFF -DCPACK_BINARY_PACKAGEMAKER=ON -DCPACK_BINARY_STGZ=OFF -DCPACK_BINARY_TGZ=OFF .. - - cmake .. - - make -j8 - artifacts: - expire_in: 6 hrs - paths: - - build/ - -build:ubuntu: - stage: build - tags: - - clang - - ubuntu - variables: - CC: /usr/bin/clang-5.0 - CXX: /usr/bin/clang++-5.0 - script: - - mkdir build - - cd build - - cmake -DBUILD_FORTRAN=ON -DBUILD_PACKAGE=ON -DBUILD_TESTING=ON -DENABLE_GTEST_DEBUG_MODE=OFF -DCPACK_BINARY_TGZ=OFF -DCPACK_BINARY_TZ=OFF .. - - cmake .. - - make -j8 - artifacts: - expire_in: 6 hrs - paths: - - build/ - -build:windows: - stage: build - tags: - - vs2017 - - windows - before_script: - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% - script: - - mkdir build - - cd build - - '"C:\Program Files\CMake\bin\cmake.exe" -G "Visual Studio 15 2017 Win64" -DBUILD_FORTRAN=ON -DBUILD_PACKAGE=ON -DBUILD_TESTING=ON -DENABLE_GTEST_DEBUG_MODE=OFF ..' - - '"C:\Program Files\CMake\bin\cmake.exe" ..' - - '"C:\Program Files\CMake\bin\cmake.exe" --build . --config Release --target ALL_BUILD' - artifacts: - expire_in: 6 hrs - paths: - - build/ - -test:mac: - stage: test - tags: - - mac - dependencies: - - build:mac - script: - - cd build - - ctest -j8 - -test:ubuntu: - stage: test - tags: - - clang - - ubuntu - variables: - CC: /usr/bin/clang-5.0 - CXX: /usr/bin/clang++-5.0 - dependencies: - - build:ubuntu - script: - - cd build - - ctest -j8 - -test:windows: - stage: test - tags: - - vs2017 - - windows - before_script: - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% - dependencies: - - build:windows - script: - - cd build - - '"C:\Program Files\CMake\bin\ctest.exe" -C Release -j8' - -package:mac: - stage: package - tags: - - mac - dependencies: - - build:mac - variables: - CC: /usr/bin/clang - CXX: /usr/bin/clang++ - CXXFLAGS: -std=c++11 -stdlib=libc++ - CMAKE_OSX_ARCHITECTURES: x86_64 - MACOSX_DEPLOYMENT_TARGET: "10.7" - SDKROOT: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk - script: - - cd build - - make package -j8 - artifacts: - expire_in: 1 week - paths: - - build/EnergyPlus-*.dmg - -# # Ubuntu 16.04 EnergyPlus packages are currently failing -# package:ubuntu: -# stage: package -# tags: -# - clang -# - ubuntu -# dependencies: -# - build:ubuntu -# variables: -# CC: /usr/bin/clang-5.0 -# CXX: /usr/bin/clang++-5.0 -# script: -# - cd build -# - make package -j8 -# artifacts: -# expire_in: 1 week -# paths: -# - build/EnergyPlus-*.sh - -package:windows: - stage: package - tags: - - vs2017 - - windows - before_script: - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% - dependencies: - - build:windows - script: - - cd build - - '"C:\Program Files\CMake\bin\cmake.exe" --build . --config Release --target PACKAGE' - artifacts: - expire_in: 1 week - paths: - - build/EnergyPlus-*.exe diff --git a/.renamed.yaml b/.renamed.yaml deleted file mode 100644 index 011ef0a2485..00000000000 --- a/.renamed.yaml +++ /dev/null @@ -1,12 +0,0 @@ -results_repository : Myoldmopar/EnergyPlusBuildResults -results_path : _posts -results_base_url : https://myoldmopar.github.io/EnergyPlusBuildResults -regression_repository : NREL/EnergyPlusRegressionTool -regression_branch : BumpToBoto3 # this is the branch of NREL/EnergyPlusRegressionTool to use (usually main) -regression_baseline_default : develop # this is the NREL/EnergyPlus branch to use as the baseline for regressions -regression_baseline_develop : "" -regression_baseline_master : "" -notification_recipients: - - myoldmopar -aging_pull_requests_notification: true -aging_pull_requests_numdays: 28 diff --git a/CMakeLists.txt b/CMakeLists.txt index b6cded6bc15..6929707fdac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,52 +200,12 @@ else() endif() if(BUILD_TESTING) - option(ENABLE_REGRESSION_TESTING "Enable Regression Tests" OFF) option(ENABLE_REVERSE_DD_TESTING "Enable Reverse Design Day Tests" OFF) option(BUILD_WCE_TESTING "Build testing targets for Windows Calculation Engine" OFF) - if(ENABLE_REGRESSION_TESTING) - if(CMAKE_CL_64) - set(ARCH_FLAG "-64bit") - else() - set(ARCH_FLAG "") - endif() - set(REGRESSION_BASELINE_SHA - "" - CACHE STRING "SHA of baseline comparison") - set(COMMIT_SHA - "" - CACHE STRING "Commit sha of this build") - set(REGRESSION_SCRIPT_PATH - "" - CACHE PATH "Path to regression scripts") - set(REGRESSION_BASELINE_PATH - "" - CACHE PATH "Path to energyplus baseline folder to compare against") - set(DEVICE_ID - "${CMAKE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}${ARCH_FLAG}" - CACHE STRING "Identifier for this device configuration") - endif() set_property(GLOBAL PROPERTY CTEST_TARGETS_ADDED 1) # This avoids all the CTest Nightly, Continuous, etc. tests. enable_testing() endif() -if(ENABLE_REGRESSION_TESTING) - string(STRIP "${REGRESSION_SCRIPT_PATH}" REG_PATH_STRIPPED) - string(STRIP "${REGRESSION_BASELINE_PATH}" REG_BASELINE_STRIPPED) - if(REG_PATH_STRIPPED STREQUAL "" OR REG_BASELINE_STRIPPED STREQUAL "") - message(AUTHOR_WARNING "Regression testing is enabled, but no script or baseline has been provided, turning it off.") - set(DO_REGRESSION_TESTING OFF) - else() - if(BUILD_FORTRAN) - # No Problem - set(DO_REGRESSION_TESTING ON) - else() - message(AUTHOR_WARNING "For regression testing, you must enable BUILD_FORTRAN in order to call ReadVarEso, turning it off.") - set(DO_REGRESSION_TESTING OFF) - endif() - endif() -endif() - if(UNIX AND NOT APPLE) set(CMAKE_INSTALL_RPATH "$ORIGIN") endif() @@ -447,7 +407,6 @@ mark_as_advanced(RE2_BUILD_TESTING) mark_as_advanced(USEPCRE) mark_as_advanced(CCACHE_PROGRAM) mark_as_advanced(CMAKE_VERSION_BUILD) -mark_as_advanced(ENABLE_REGRESSION_TESTING) mark_as_advanced(TEST_ANNUAL_SIMULATION) mark_as_advanced(gtest_build_samples) mark_as_advanced(gtest_build_tests) diff --git a/cmake/ProjectMacros.cmake b/cmake/ProjectMacros.cmake index 6bd2aec73d7..071bc94f56c 100644 --- a/cmake/ProjectMacros.cmake +++ b/cmake/ProjectMacros.cmake @@ -112,10 +112,6 @@ function(ADD_SIMULATION_TEST) if(ADD_SIM_TEST_PERFORMANCE) # For performance testing, it's more problematic, because that'll cut on the ReadVarEso time message(WARNING "Will not be able to call ReadVarEso unless BUILD_FORTRAN=TRUE, skipping flag -r.") - elseif(DO_REGRESSION_TESTING) - # DO_REGRESSION_TESTING shouldn't really occur here since EnergyPlus/CMakeLists.txt will throw an error if BUILD_FORTRAN isn't enabled - # Not that bad, just a dev warning - message(AUTHOR_WARNING "Will not be able to call ReadVarEso unless BUILD_FORTRAN=TRUE, skipping flag -r.") endif() endif() @@ -164,21 +160,6 @@ function(ADD_SIMULATION_TEST) set_tests_properties("${TEST_CATEGORY}.${IDF_NAME}" PROPERTIES FAIL_REGULAR_EXPRESSION "ERROR;FAIL;Test Failed") endif() - if(DO_REGRESSION_TESTING AND (NOT ADD_SIM_TEST_EXPECT_FATAL)) - add_test( - NAME "regression.${IDF_NAME}" - COMMAND - ${CMAKE_COMMAND} -DBINARY_DIR=${PROJECT_BINARY_DIR} -DPYTHON_EXECUTABLE=${Python_EXECUTABLE} -DIDF_FILE=${ADD_SIM_TEST_IDF_FILE} - -DREGRESSION_SCRIPT_PATH=${REGRESSION_SCRIPT_PATH} -DREGRESSION_BASELINE_PATH=${REGRESSION_BASELINE_PATH} - -DREGRESSION_BASELINE_SHA=${REGRESSION_BASELINE_SHA} -DCOMMIT_SHA=${COMMIT_SHA} -DDEVICE_ID=${DEVICE_ID} -P - ${PROJECT_SOURCE_DIR}/cmake/RunRegression.cmake) - # Note, CMake / CTest doesn't seem to validate if this dependent name actually exists, - # but it does seem to honor the requirement - set_tests_properties("regression.${IDF_NAME}" PROPERTIES DEPENDS "${TEST_CATEGORY}.${IDF_NAME}") - set_tests_properties("regression.${IDF_NAME}" PROPERTIES PASS_REGULAR_EXPRESSION "Success") - set_tests_properties("regression.${IDF_NAME}" PROPERTIES FAIL_REGULAR_EXPRESSION "ERROR;FAIL;Test Failed") - endif() - if(ENABLE_REVERSE_DD_TESTING AND (NOT ADD_SIM_TEST_EXPECT_FATAL)) set(TEST_FILE_FOLDER "testfiles") set(ENERGYPLUS_FLAGS "-D -r") diff --git a/cmake/RunRegression.cmake b/cmake/RunRegression.cmake deleted file mode 100644 index 4e693c91cd6..00000000000 --- a/cmake/RunRegression.cmake +++ /dev/null @@ -1,17 +0,0 @@ -# These need to be defined by the caller -# IDF_FILE -# BINARY_DIR -# PYTHON_EXECUTABLE -# REGRESSION_SCRIPT_PATH -# REGRESSION_BASELINE_PATH -# REGRESSION_BASELINE_SHA -# COMMIT_SHA -# DEVICE_ID - -get_filename_component(IDF_NAME "${IDF_FILE}" NAME_WE) - -execute_process( - COMMAND - ${PYTHON_EXECUTABLE} "${REGRESSION_SCRIPT_PATH}/energyplus_regressions/diffs/ci_compare_script.py" "${IDF_NAME}" - "${REGRESSION_BASELINE_PATH}/testfiles/${IDF_NAME}" "${BINARY_DIR}/testfiles/${IDF_NAME}" ${REGRESSION_BASELINE_SHA} ${COMMIT_SHA} true - "${DEVICE_ID}") diff --git a/scripts/dev/gha_regressions.py b/scripts/dev/gha_regressions.py index 7ccff0a5123..c46b4a85afa 100644 --- a/scripts/dev/gha_regressions.py +++ b/scripts/dev/gha_regressions.py @@ -69,6 +69,8 @@ class RegressionManager: def __init__(self): + self.root_index_files_no_diff = [] + self.root_index_files_diffs = [] self.diffs_by_idf = defaultdict(list) self.diffs_by_type = defaultdict(list) self.summary_results = {} @@ -224,26 +226,26 @@ def single_test_case_html(contents: str) -> str: """ - def bundle_root_index_html(self, header_info: list[str], no_diffs: list[str], diffs: list[str]) -> str: + def bundle_root_index_html(self, header_info: list[str]) -> str: # set up header table header_content = "" for hi in header_info: header_content += f"""
  • {hi}
    • \n""" # set up diff summary listings - num_no_diff = len(no_diffs) + num_no_diff = len(self.root_index_files_no_diff) nds = 's' if num_no_diff == 0 or num_no_diff > 1 else '' no_diff_content = "" - for nd in no_diffs: + for nd in self.root_index_files_no_diff: no_diff_content += f"""
  • {nd}
    • \n""" - num_diff = len(diffs) + num_diff = len(self.root_index_files_diffs) ds = 's' if num_diff == 0 or num_diff > 1 else '' diff_content = "" - for d in diffs: + for d in self.root_index_files_diffs: diff_content += f"""{d}\n""" # set up diff type listing - diff_type_keys = self.diffs_by_type.keys() + diff_type_keys = sorted(self.diffs_by_type.keys()) num_diff_types = len(diff_type_keys) dt = 's' if num_diff_types == 0 or num_diff_types > 1 else '' diff_type_content = "" @@ -278,7 +280,9 @@ def bundle_root_index_html(self, header_info: list[str], no_diffs: list[str], di run_time_rows_text = "" sum_base_seconds = 0 sum_branch_seconds = 0 - for idf, summary in self.summary_results.items(): + sorted_idf_keys = sorted(self.summary_results.keys()) + for idf in sorted_idf_keys: + summary = self.summary_results[idf] case_1_success = summary.simulation_status_case1 == EndErrSummary.STATUS_SUCCESS case_2_success = summary.simulation_status_case2 == EndErrSummary.STATUS_SUCCESS if case_1_success: @@ -424,7 +428,7 @@ def generate_markdown_summary(self, bundle_root: Path): diff_lines += f" - {diff_type}: {len(idfs)}\n" content = f"""
    - Regression Summary - Download Build Artifact for Full Details + Regression Summary {diff_lines}
    """ @@ -432,10 +436,8 @@ def generate_markdown_summary(self, bundle_root: Path): def check_all_regressions(self, base_testfiles: Path, mod_testfiles: Path, bundle_root: Path) -> bool: any_diffs = False - root_index_files_no_diff = [] - root_index_files_diffs = [] bundle_root.mkdir(exist_ok=True) - for baseline in base_testfiles.iterdir(): + for baseline in sorted(base_testfiles.iterdir()): if not baseline.is_dir(): continue if baseline.name == 'CMakeFiles': # add more ignore dirs here @@ -445,7 +447,7 @@ def check_all_regressions(self, base_testfiles: Path, mod_testfiles: Path, bundl continue # TODO: Should we warn that it is missing? entry, diffs = self.single_file_regressions(baseline, modified) if diffs: - root_index_files_diffs.append(baseline.name) + self.root_index_files_diffs.append(baseline.name) any_diffs = True potential_diff_files = baseline.glob("*.*.*") # TODO: Could try to get this from the regression tool target_dir_for_this_file_diffs = bundle_root / baseline.name @@ -472,7 +474,7 @@ def check_all_regressions(self, base_testfiles: Path, mod_testfiles: Path, bundl so_far = ' Diffs! ' if any_diffs else 'No diffs' print(f"Diff status so far: {so_far}. This file HAZ DIFFS: {baseline.name}") else: - root_index_files_no_diff.append(baseline.name) + self.root_index_files_no_diff.append(baseline.name) so_far = ' Diffs! ' if any_diffs else 'No diffs' print(f"Diff status so far: {so_far}. This file has no diffs: {baseline.name}") meta_data = [ @@ -481,14 +483,12 @@ def check_all_regressions(self, base_testfiles: Path, mod_testfiles: Path, bundl f"Number of input files evaluated: {self.num_idf_inspected}", ] bundle_root_index_file_path = bundle_root / 'index.html' - bundle_root_index_content = self.bundle_root_index_html( - meta_data, root_index_files_no_diff, root_index_files_diffs - ) + bundle_root_index_content = self.bundle_root_index_html(meta_data) bundle_root_index_file_path.write_text(bundle_root_index_content) print() - print(f"* Files with Diffs *:\n{"\n ".join(root_index_files_diffs)}\n") - print(f"* Diffs by File *:\n{json.dumps(self.diffs_by_idf, indent=2)}\n") - print(f"* Diffs by Type *:\n{json.dumps(self.diffs_by_type, indent=2)}\n") + print(f"* Files with Diffs *:\n{"\n ".join(self.root_index_files_diffs)}\n") + print(f"* Diffs by File *:\n{json.dumps(self.diffs_by_idf, indent=2, sort_keys=True)}\n") + print(f"* Diffs by Type *:\n{json.dumps(self.diffs_by_type, indent=2, sort_keys=True)}\n") if any_diffs: self.generate_markdown_summary(bundle_root) return any_diffs