test with cucumber (#910)

.github/workflows/build_centos7.yml

@@ -176,6 +176,13 @@ jobs:
         run: |
           cmake --build _build --config Release -j$(nproc)
 
+
+      - name: Run cucumber on outer_loop tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/outer_loop_tests.feature"
+          mpi_path: ${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin
+
       - name: Cache vcpkg binary dir
         if: always()
         id: save-cache-vcpkg-binary

.github/workflows/build_oracle8.yml

@@ -141,6 +141,13 @@ jobs:
         run: |
           cmake --build _build --config Release -j$(nproc)
 
+
+      - name: Run cucumber on outer_loop tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/outer_loop_tests.feature"
+          mpi_path: ${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin
+
       - name: Running unit tests
         timeout-minutes: 120
         shell: bash

.github/workflows/build_ubuntu.yml

@@ -147,6 +147,13 @@ jobs:
         run: |
           cmake --build _build --config Release -j$(nproc)
 
+      - name: Run cucumber on outer_loop tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/outer_loop_tests.feature"
+          mpi_path: ${{ github.workspace }}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin
+
+
       - name: Test
         run: |
           export PATH=${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin:$PATH

.github/workflows/build_windows.yml

@@ -124,6 +124,14 @@ jobs:
         run: |
           cmake --build _build --config Release -j4
 
+
+      - name: Run cucumber on outer_loop tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/outer_loop_tests.feature"
+          mpi_path: /c/Program Files/Microsoft MPI/Bin
+
+
       - name: Cache vcpkg binary dir
         if: always()
         id: save-cache-vcpkg-binary

.github/workflows/cucumber-tests/action.yml

@@ -0,0 +1,24 @@
+name: "Run cucumber tests"
+description: "Run cucumber tests"
+inputs:
+  feature:
+    description: 'Feature file or folder to run (default runs all features in "features" folder)'
+    required: false
+    default: 'features'
+  tags:
+    description: 'Tags to run (default skips tests marked @flaky)'
+    required: false
+    default: '~@flaky'
+  mpi_path:
+    description: "Mpi install directory"
+    required: true
+runs:
+  using: "composite"
+  steps:
+
+    - name: Run tests
+      shell: bash
+      run: |
+        export PATH="${{ inputs.mpi_path }}:$PATH"
+        cd tests/end_to_end
+        behave --tags ${{ inputs.tags }} cucumber/${{ inputs.feature }} --no-capture

CMakeLists.txt

@@ -45,7 +45,7 @@ option (XPRESS  "Use solver FICO XPRESS" OFF)
 if(COIN_OR)
 	message("Coin-OR Solvers Clp and Cbc used. Solvers version are those present in orTools release linked to project.")
 else ()
-	message(FATAL_ERRROR "COIN_OR has to be true, it is required in lpnamer module of Antares Xpansion.")	
+	message(FATAL_ERROR "COIN_OR has to be true, it is required in lpnamer module of Antares Xpansion.")
 endif()
 
 

data_test/external_loop_test/lp/options.json

@@ -4,21 +4,22 @@
     "RELATIVE_GAP": 1e-06,
     "RELAXED_GAP": 1e-05,
     "AGGREGATION": false,
-    "OUTPUTROOT": "data_test/external_loop_test/lp",
+    "OUTPUTROOT": ".",
     "TRACE": true,
     "SLAVE_WEIGHT": "CONSTANT",
-    "SLAVE_WEIGHT_VALUE": 0.5,
+    "SLAVE_WEIGHT_VALUE": 1,
     "MASTER_NAME": "master",
     "STRUCTURE_FILE": "structure.txt",
-    "INPUTROOT": "data_test/external_loop_test/lp",
+    "INPUTROOT": ".",
     "CSV_NAME": "benders_output_trace",
     "BOUND_ALPHA": true,
     "SEPARATION_PARAM": 0.5,
     "BATCH_SIZE": 0,
-    "JSON_FILE": "data_test/external_loop_test/expansion/out.json",
-    "LAST_ITERATION_JSON_FILE": "data_test/external_loop_test/expansion/last_iteration.json",
+    "JSON_FILE": "../expansion/out.json",
+    "LAST_ITERATION_JSON_FILE": "../expansion/last_iteration.json",
     "MASTER_FORMULATION": "integer",
-    "SOLVER_NAME": "XPRESS",
+    "###UNTIL XPRESS LICENSE IS NOT UPDATED ###": "USE COIN",
+    "SOLVER_NAME": "COIN",
     "TIME_LIMIT": 1000000000000.0,
     "LOG_LEVEL": 0,
     "LAST_MASTER_MPS": "master_last_iteration",

requirements-tests.txt

@@ -2,3 +2,4 @@
 pytest
 numpy
 pytest-cov
+behave

src/python/config.yaml.in

@@ -9,5 +9,7 @@ OUTER_LOOP                      : $<TARGET_FILE_NAME:outer_loop>
 SENSITIVITY                     : $<TARGET_FILE_NAME:sensitivity>
 ANTARES_ARCHIVE_UPDATER         : $<TARGET_FILE_NAME:antares_archive_updater>
 mpiexec                         : @MPIEXEC_EXECUTABLE@
+#for test only
+allow_run_as_root               : ${ALLOW_RUN_AS_ROOT}
 AVAILABLE_SOLVER :
 @AVAILABLE_SOLVER_YML_LIST@

tests/CMakeLists.txt

@@ -11,6 +11,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build_config.yaml.in ${CMAKE_CURRENT_
 find_python_module(pytest)
 find_python_module(numpy)
 
+
 # for centos docker to run MPI tests as root
 
 set(allow_run_as_root_option "")

tests/cpp/outer_loop/outer_loop_test.cpp

@@ -28,7 +28,7 @@ int main(int argc, char** argv) {
 //-------------------- MasterUpdateBaseTest -------------------------
 const auto STUDY_PATH =
     std::filesystem::path("data_test") / "external_loop_test";
-const auto LP_DIR = STUDY_PATH / "lp";
+const auto LP_DIR = std::filesystem::absolute((STUDY_PATH / "lp"));
 const auto OPTIONS_FILE = LP_DIR / "options.json";
 const auto OUTER_OPTIONS_FILE = "adequacy_criterion.yml";
 
@@ -40,6 +40,11 @@ class MasterUpdateBaseTest : public ::testing::TestWithParam<std::string> {
   Writer writer;
 
   void SetUp() override {
+    // Save the current working directory
+    original_working_dir_ = std::filesystem::current_path();
+
+    // Change to the desired working directory
+    std::filesystem::current_path(LP_DIR);
     math_log_driver = MathLoggerFactory::get_void_logger();
     logger = build_void_logger();
     writer = build_void_writer();
@@ -48,6 +53,12 @@ class MasterUpdateBaseTest : public ::testing::TestWithParam<std::string> {
     SimulationOptions options(OPTIONS_FILE);
     return options.get_benders_options();
   }
+
+  void TearDown() override {
+    // Restore the original working directory after the test
+    std::filesystem::current_path(original_working_dir_);
+  }
+  std::filesystem::path original_working_dir_;
 };
 
 auto solvers() {

tests/end_to_end/cucumber/features/outer_loop_tests.feature

@@ -0,0 +1,12 @@
+Feature: outer loop tests
+
+  @fast @short @outerloop
+  Scenario: a system with 4 nodes, on 1 timestep, 2 scenarios
+    Given the study path is "data_test/external_loop_test"
+    When I run outer loop with 1 proc(s)
+    Then the simulation takes less than 5 seconds
+    And the simulation succeeds
+    And the expected overall cost is 92.70005
+    And the solution is
+      | variable    | value    |
+      | G_p_max_0_0 | 2.900004 |

tests/end_to_end/cucumber/steps/steps.py

@@ -0,0 +1,84 @@
+import json
+import os
+import subprocess
+from pathlib import Path
+
+import numpy as np
+from behave import *
+
+from utils_functions import get_mpi_command, get_conf
+
+
+@given('the study path is "{string}"')
+def study_path_is(context, string):
+    context.study_path = os.path.join(Path() / "../../",
+                                      string.replace("/", os.sep))
+
+
+def build_outer_loop_command(context):
+    command = get_mpi_command(allow_run_as_root=context.allow_run_as_root, nproc=context.nproc)
+    exe_path = Path(get_conf("DEFAULT_INSTALL_DIR")) / get_conf("OUTER_LOOP")
+    command.append(str(exe_path))
+    command.append("options.json")
+    return command
+
+
+
+def read_outputs(output_path):
+    with open(output_path, 'r') as file:
+        outputs = json.load(file)
+
+    return outputs
+
+
+@when('I run outer loop with {n} proc(s)')
+def run_outer_loop(context, n):
+    context.nproc = int(n)
+    context.allow_run_as_root = get_conf("allow_run_as_root")
+    command = build_outer_loop_command(context)
+    print(f"Running command: {command}")
+    old_cwd = os.getcwd()
+    lp_path = Path(context.study_path) / "lp"
+
+    os.chdir(lp_path)
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    out, err = process.communicate()
+    # print(out)
+    # print("*****")
+    # print(err)
+    context.return_code = process.returncode
+    context.outputs = read_outputs(Path("..") / "expansion" / "out.json")
+    os.chdir(old_cwd)
+
+
+@then("the simulation takes less than {seconds} seconds")
+def check_simu_time(context, seconds):
+    assert context.outputs["run_duration"] <= float(seconds)
+
+
+@then("the simulation succeeds")
+def simu_success(context):
+    return context.return_code == 0
+
+
+@then("the expected overall cost is {value}")
+def check_overall_cost(context, value):
+    np.testing.assert_allclose(float(value),
+                               context.outputs["solution"]["overall_cost"], rtol=1e-6, atol=0)
+
+
+def assert_dict_allclose(actual, expected, rtol=1e-06, atol=0):
+    for key in expected:
+        np.testing.assert_allclose(
+            actual[key],
+            expected[key],
+            rtol=rtol,
+            atol=atol,
+            err_msg=f"Mismatch found at key '{key}'"
+        )
+
+
+@then("the solution is")
+def check_solution(context):
+    expected_solution = {row['variable']: float(row['value']) for row in context.table}
+    assert_dict_allclose(context.outputs["solution"]["values"], expected_solution)