From 298fe2d2a31d5f2ce9690751021f57ef4247f982 Mon Sep 17 00:00:00 2001 From: Abdoulbari Zaher <32519851+a-zakir@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:49:35 +0200 Subject: [PATCH] test with cucumber (#910) --- .github/workflows/build_centos7.yml | 7 ++ .github/workflows/build_oracle8.yml | 7 ++ .github/workflows/build_ubuntu.yml | 7 ++ .github/workflows/build_windows.yml | 8 ++ .github/workflows/cucumber-tests/action.yml | 24 ++++++ CMakeLists.txt | 2 +- data_test/external_loop_test/lp/options.json | 13 +-- requirements-tests.txt | 1 + src/python/config.yaml.in | 2 + tests/CMakeLists.txt | 1 + tests/cpp/outer_loop/outer_loop_test.cpp | 13 ++- .../features/outer_loop_tests.feature | 12 +++ tests/end_to_end/cucumber/steps/__init__.py | 0 tests/end_to_end/cucumber/steps/steps.py | 84 +++++++++++++++++++ 14 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/cucumber-tests/action.yml create mode 100644 tests/end_to_end/cucumber/features/outer_loop_tests.feature create mode 100644 tests/end_to_end/cucumber/steps/__init__.py create mode 100644 tests/end_to_end/cucumber/steps/steps.py diff --git a/.github/workflows/build_centos7.yml b/.github/workflows/build_centos7.yml index ce2a4dc84..66851a7ea 100644 --- a/.github/workflows/build_centos7.yml +++ b/.github/workflows/build_centos7.yml @@ -176,6 +176,13 @@ jobs: run: | cmake --build _build --config Release -j$(nproc) + + - name: Run cucumber on outer_loop tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/outer_loop_tests.feature" + mpi_path: ${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin + - name: Cache vcpkg binary dir if: always() id: save-cache-vcpkg-binary diff --git a/.github/workflows/build_oracle8.yml b/.github/workflows/build_oracle8.yml index 5c92b4f28..c5b700228 100644 --- a/.github/workflows/build_oracle8.yml +++ b/.github/workflows/build_oracle8.yml @@ -141,6 +141,13 @@ jobs: run: | cmake --build _build --config Release -j$(nproc) + + - name: Run cucumber on outer_loop tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/outer_loop_tests.feature" + mpi_path: ${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin + - name: Running unit tests timeout-minutes: 120 shell: bash diff --git a/.github/workflows/build_ubuntu.yml b/.github/workflows/build_ubuntu.yml index 3b12e1fb9..5e2039d01 100644 --- a/.github/workflows/build_ubuntu.yml +++ b/.github/workflows/build_ubuntu.yml @@ -147,6 +147,13 @@ jobs: run: | cmake --build _build --config Release -j$(nproc) + - name: Run cucumber on outer_loop tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/outer_loop_tests.feature" + mpi_path: ${{ github.workspace }}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin + + - name: Test run: | export PATH=${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin:$PATH diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml index c14741ee2..2e10d0375 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/build_windows.yml @@ -124,6 +124,14 @@ jobs: run: | cmake --build _build --config Release -j4 + + - name: Run cucumber on outer_loop tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/outer_loop_tests.feature" + mpi_path: /c/Program Files/Microsoft MPI/Bin + + - name: Cache vcpkg binary dir if: always() id: save-cache-vcpkg-binary diff --git a/.github/workflows/cucumber-tests/action.yml b/.github/workflows/cucumber-tests/action.yml new file mode 100644 index 000000000..995bab7bc --- /dev/null +++ b/.github/workflows/cucumber-tests/action.yml @@ -0,0 +1,24 @@ +name: "Run cucumber tests" +description: "Run cucumber tests" +inputs: + feature: + description: 'Feature file or folder to run (default runs all features in "features" folder)' + required: false + default: 'features' + tags: + description: 'Tags to run (default skips tests marked @flaky)' + required: false + default: '~@flaky' + mpi_path: + description: "Mpi install directory" + required: true +runs: + using: "composite" + steps: + + - name: Run tests + shell: bash + run: | + export PATH="${{ inputs.mpi_path }}:$PATH" + cd tests/end_to_end + behave --tags ${{ inputs.tags }} cucumber/${{ inputs.feature }} --no-capture \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c8541acc..b1b23633f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,7 @@ option (XPRESS "Use solver FICO XPRESS" OFF) if(COIN_OR) message("Coin-OR Solvers Clp and Cbc used. Solvers version are those present in orTools release linked to project.") else () - message(FATAL_ERRROR "COIN_OR has to be true, it is required in lpnamer module of Antares Xpansion.") + message(FATAL_ERROR "COIN_OR has to be true, it is required in lpnamer module of Antares Xpansion.") endif() diff --git a/data_test/external_loop_test/lp/options.json b/data_test/external_loop_test/lp/options.json index 004b0dbcd..836cced01 100644 --- a/data_test/external_loop_test/lp/options.json +++ b/data_test/external_loop_test/lp/options.json @@ -4,21 +4,22 @@ "RELATIVE_GAP": 1e-06, "RELAXED_GAP": 1e-05, "AGGREGATION": false, - "OUTPUTROOT": "data_test/external_loop_test/lp", + "OUTPUTROOT": ".", "TRACE": true, "SLAVE_WEIGHT": "CONSTANT", - "SLAVE_WEIGHT_VALUE": 0.5, + "SLAVE_WEIGHT_VALUE": 1, "MASTER_NAME": "master", "STRUCTURE_FILE": "structure.txt", - "INPUTROOT": "data_test/external_loop_test/lp", + "INPUTROOT": ".", "CSV_NAME": "benders_output_trace", "BOUND_ALPHA": true, "SEPARATION_PARAM": 0.5, "BATCH_SIZE": 0, - "JSON_FILE": "data_test/external_loop_test/expansion/out.json", - "LAST_ITERATION_JSON_FILE": "data_test/external_loop_test/expansion/last_iteration.json", + "JSON_FILE": "../expansion/out.json", + "LAST_ITERATION_JSON_FILE": "../expansion/last_iteration.json", "MASTER_FORMULATION": "integer", - "SOLVER_NAME": "XPRESS", + "###UNTIL XPRESS LICENSE IS NOT UPDATED ###": "USE COIN", + "SOLVER_NAME": "COIN", "TIME_LIMIT": 1000000000000.0, "LOG_LEVEL": 0, "LAST_MASTER_MPS": "master_last_iteration", diff --git a/requirements-tests.txt b/requirements-tests.txt index 71db3427b..9b3c71add 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,3 +2,4 @@ pytest numpy pytest-cov +behave \ No newline at end of file diff --git a/src/python/config.yaml.in b/src/python/config.yaml.in index 3660c00a1..0d857c2ae 100644 --- a/src/python/config.yaml.in +++ b/src/python/config.yaml.in @@ -9,5 +9,7 @@ OUTER_LOOP : $ SENSITIVITY : $ ANTARES_ARCHIVE_UPDATER : $ mpiexec : @MPIEXEC_EXECUTABLE@ +#for test only +allow_run_as_root : ${ALLOW_RUN_AS_ROOT} AVAILABLE_SOLVER : @AVAILABLE_SOLVER_YML_LIST@ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 277674256..332eca6b4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,6 +11,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build_config.yaml.in ${CMAKE_CURRENT_ find_python_module(pytest) find_python_module(numpy) + # for centos docker to run MPI tests as root set(allow_run_as_root_option "") diff --git a/tests/cpp/outer_loop/outer_loop_test.cpp b/tests/cpp/outer_loop/outer_loop_test.cpp index 12af2b37b..45e19101f 100644 --- a/tests/cpp/outer_loop/outer_loop_test.cpp +++ b/tests/cpp/outer_loop/outer_loop_test.cpp @@ -28,7 +28,7 @@ int main(int argc, char** argv) { //-------------------- MasterUpdateBaseTest ------------------------- const auto STUDY_PATH = std::filesystem::path("data_test") / "external_loop_test"; -const auto LP_DIR = STUDY_PATH / "lp"; +const auto LP_DIR = std::filesystem::absolute((STUDY_PATH / "lp")); const auto OPTIONS_FILE = LP_DIR / "options.json"; const auto OUTER_OPTIONS_FILE = "adequacy_criterion.yml"; @@ -40,6 +40,11 @@ class MasterUpdateBaseTest : public ::testing::TestWithParam { Writer writer; void SetUp() override { + // Save the current working directory + original_working_dir_ = std::filesystem::current_path(); + + // Change to the desired working directory + std::filesystem::current_path(LP_DIR); math_log_driver = MathLoggerFactory::get_void_logger(); logger = build_void_logger(); writer = build_void_writer(); @@ -48,6 +53,12 @@ class MasterUpdateBaseTest : public ::testing::TestWithParam { SimulationOptions options(OPTIONS_FILE); return options.get_benders_options(); } + + void TearDown() override { + // Restore the original working directory after the test + std::filesystem::current_path(original_working_dir_); + } + std::filesystem::path original_working_dir_; }; auto solvers() { diff --git a/tests/end_to_end/cucumber/features/outer_loop_tests.feature b/tests/end_to_end/cucumber/features/outer_loop_tests.feature new file mode 100644 index 000000000..26f356c05 --- /dev/null +++ b/tests/end_to_end/cucumber/features/outer_loop_tests.feature @@ -0,0 +1,12 @@ +Feature: outer loop tests + + @fast @short @outerloop + Scenario: a system with 4 nodes, on 1 timestep, 2 scenarios + Given the study path is "data_test/external_loop_test" + When I run outer loop with 1 proc(s) + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the expected overall cost is 92.70005 + And the solution is + | variable | value | + | G_p_max_0_0 | 2.900004 | \ No newline at end of file diff --git a/tests/end_to_end/cucumber/steps/__init__.py b/tests/end_to_end/cucumber/steps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/end_to_end/cucumber/steps/steps.py b/tests/end_to_end/cucumber/steps/steps.py new file mode 100644 index 000000000..6a802caf5 --- /dev/null +++ b/tests/end_to_end/cucumber/steps/steps.py @@ -0,0 +1,84 @@ +import json +import os +import subprocess +from pathlib import Path + +import numpy as np +from behave import * + +from utils_functions import get_mpi_command, get_conf + + +@given('the study path is "{string}"') +def study_path_is(context, string): + context.study_path = os.path.join(Path() / "../../", + string.replace("/", os.sep)) + + +def build_outer_loop_command(context): + command = get_mpi_command(allow_run_as_root=context.allow_run_as_root, nproc=context.nproc) + exe_path = Path(get_conf("DEFAULT_INSTALL_DIR")) / get_conf("OUTER_LOOP") + command.append(str(exe_path)) + command.append("options.json") + return command + + + +def read_outputs(output_path): + with open(output_path, 'r') as file: + outputs = json.load(file) + + return outputs + + +@when('I run outer loop with {n} proc(s)') +def run_outer_loop(context, n): + context.nproc = int(n) + context.allow_run_as_root = get_conf("allow_run_as_root") + command = build_outer_loop_command(context) + print(f"Running command: {command}") + old_cwd = os.getcwd() + lp_path = Path(context.study_path) / "lp" + + os.chdir(lp_path) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + out, err = process.communicate() + # print(out) + # print("*****") + # print(err) + context.return_code = process.returncode + context.outputs = read_outputs(Path("..") / "expansion" / "out.json") + os.chdir(old_cwd) + + +@then("the simulation takes less than {seconds} seconds") +def check_simu_time(context, seconds): + assert context.outputs["run_duration"] <= float(seconds) + + +@then("the simulation succeeds") +def simu_success(context): + return context.return_code == 0 + + +@then("the expected overall cost is {value}") +def check_overall_cost(context, value): + np.testing.assert_allclose(float(value), + context.outputs["solution"]["overall_cost"], rtol=1e-6, atol=0) + + +def assert_dict_allclose(actual, expected, rtol=1e-06, atol=0): + for key in expected: + np.testing.assert_allclose( + actual[key], + expected[key], + rtol=rtol, + atol=atol, + err_msg=f"Mismatch found at key '{key}'" + ) + + +@then("the solution is") +def check_solution(context): + expected_solution = {row['variable']: float(row['value']) for row in context.table} + assert_dict_allclose(context.outputs["solution"]["values"], expected_solution)