Feature/ant 2561 api e2e solver consistency (#981)

CMakeLists.txt

@@ -8,8 +8,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
 # Project configuration
 # ===========================================================================
 
-PROJECT(antaresXpansion VERSION 1.3.2)
-set(ANTARES_XPANSION_RC 0)
+PROJECT(antaresXpansion VERSION 1.4.0)
+set(ANTARES_XPANSION_RC 4)
 
 # ===========================================================================
 # Default parameters

antares-version.json

@@ -1,7 +1,7 @@
 {
-  "antares_version": "9.2.0-RC5",
+  "antares_version": "9.2.0-rc9",
   "antares_version_executable": "9.2",
-  "antares_xpansion_version": "1.3.2",
+  "antares_xpansion_version": "1.4.0",
   "minizip_ng_version": "3.0.6",
   "or-tools-rte": "9.11-rte1.1"
 }

data_test/examples/dummy-3areas-3candidates-3links/settings/generaldata.ini

@@ -90,3 +90,5 @@ seed-thermal-costs = 8005489
 seed-hydro-costs = 9005489
 seed-initial-reservoir-levels = 10005489
 
+[compatibility]
+hydro-pmax = hourly

docs/changelog/CHANGELOG.md

@@ -1,6 +1,30 @@
 Antares-Xpansion changelog
 =================
 
+v1.4.0 (12/2024)
+--------------------------------------------------------
+### Features
+* Update Antares Simulator to  9.2-RC7
+* Output file **criterions.txt** is renamed **LOLD.txt** as it gathers the loss of load for each area at each iteration of the algorithm
+* End to end coherence of solvers : if Xpress is selected, it is also used for Antares step (does not work yet with memory mode), if coin is selected, sirius is used in Antares
+* Support for in memory problems between Antares and problem generation
+* Remove local UI support [#972](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/972)
+* Properly handle Antares Simulator errors. Especially error -9 occurring when running out of memory where Xpansion tried to continue in an improper state.
+
+### For developers
+* Various improvement on dependency management through vcpkg.
+* Update and improve documentation
+* Use devtoolset 11 for CentOS and OL8
+* Add some tests using cucumber
+* Improve maintainability [#965](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/965)
+* Fix performance regression with missing TBB dependency [#970](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/970)
+* Add script to convert markdown docs for support in xwiki [#973](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/973)
+
+### New Contributors
+* @pet-mit made their first contribution in https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/944
+
+**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v1.4.0-rc3
+
 v1.3.2 (07/2024)
 --------------------------------------------------------
 ### Bug Fixes

sonar-project.properties

@@ -1,7 +1,7 @@
 sonar.projectName=antares-xpansion
 sonar.projectKey=AntaresSimulatorTeam_antares-xpansion
 sonar.organization=antaressimulatorteam
-sonar.projectVersion=1.3.2
+sonar.projectVersion=1.4.0
 
 # =====================================================
 #   Properties that will be shared amongst all modules

src/cpp/lpnamer/main/ProblemGeneration.cpp

@@ -1,15 +1,13 @@
 
-#include <execution>
-#include <tbb/tbb.h>
-
 #include "antares-xpansion/lpnamer/main/ProblemGeneration.h"
 
 #include <antares/api/solver.h>
+#include <tbb/tbb.h>
 
+#include <execution>
 #include <iostream>
 #include <utility>
 
-#include "antares-xpansion/lpnamer/input_reader/SettingsReader.h"
 #include "Version.h"
 #include "antares-xpansion/helpers/Timer.h"
 #include "antares-xpansion/lpnamer/helper/ProblemGenerationLogger.h"
@@ -63,8 +61,29 @@ ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options)
   }
 }
 
-std::filesystem::path ProblemGeneration::performAntaresSimulation() {
-  auto results = Antares::API::PerformSimulation(options_.StudyPath());
+static std::string lowerCase(std::string data) {
+  std::transform(data.begin(), data.end(), data.begin(),
+                 [](unsigned char c) { return std::tolower(c); });
+  return data;
+}
+
+static std::string solverXpansionToSimulator(const std::string& in) {
+  // in could be Cbc or CBC depending on whether it is defined or not in the
+  // settings file 
+  // Use lowerCase in any case to be robust to these subtleties
+  std::string lower_case_in = lowerCase(in);
+  if (lower_case_in == "xpress") return "xpress";
+  if (lower_case_in == "cbc" || lower_case_in == "coin") return "coin";
+  throw std::invalid_argument("Invalid solver");
+}
+
+void ProblemGeneration::performAntaresSimulation(
+    const std::filesystem::path& output) {
+  Antares::Solver::Optimization::OptimizationOptions optOptions;
+
+  optOptions.ortoolsSolver = solverXpansionToSimulator(solver_name_);
+  auto results =
+      Antares::API::PerformSimulation(options_.StudyPath(), output, optOptions);
   // Add parallel
 
   // Handle errors
@@ -74,7 +93,32 @@ std::filesystem::path ProblemGeneration::performAntaresSimulation() {
   }
 
   lps_ = std::move(results.antares_problems);
-  return {results.simulationPath};
+}
+
+static std::string getCurrentTimestamp() {
+  // Get the current time point
+  auto now = std::chrono::system_clock::now();
+
+  // Convert to time_t for formatting
+  std::time_t now_time_t = std::chrono::system_clock::to_time_t(now);
+
+  // Convert to tm structure
+  std::tm now_tm;
+#ifdef _WIN32
+  localtime_s(&now_tm, &now_time_t);  // Windows-specific
+#else
+  localtime_r(&now_time_t, &now_tm);  // POSIX-specific
+#endif
+
+  // Format the timestamp
+  std::ostringstream oss;
+  oss << std::put_time(&now_tm, "%Y%m%d-%H%Meco");
+  return oss.str();
+}
+
+// Useful only for "API" mode
+std::filesystem::path generateOutputName(const std::filesystem::path& study) {
+  return study / "output" / getCurrentTimestamp();
 }
 
 std::filesystem::path ProblemGeneration::updateProblems() {
@@ -86,20 +130,23 @@ std::filesystem::path ProblemGeneration::updateProblems() {
   if (mode_ == SimulationInputMode::ARCHIVE) {
     xpansion_output_dir =
         options_.deduceXpansionDirIfEmpty(xpansion_output_dir, archive_path);
-    study_dir = std::filesystem::absolute(archive_path).parent_path().parent_path();
-    //Assume study/output/archive.zip
+    study_dir =
+        std::filesystem::absolute(archive_path).parent_path().parent_path();
+    // Assume study/output/archive.zip
   }
 
   if (mode_ == SimulationInputMode::ANTARES_API) {
-    simulation_dir_ = performAntaresSimulation();
     study_dir = options_.StudyPath();
+    simulation_dir_ = generateOutputName(study_dir);
   }
 
   if (mode_ == SimulationInputMode::FILE) {
     simulation_dir_ = options_.XpansionOutputDir();  // Legacy naming.
     // options_.XpansionOutputDir() point in fact to a simulation output from
     // antares
-    study_dir = std::filesystem::absolute(simulation_dir_).parent_path().parent_path(); //Assume study/output/simulation
+    study_dir = std::filesystem::absolute(simulation_dir_)
+                    .parent_path()
+                    .parent_path();  // Assume study/output/simulation
   }
 
   if (mode_ == SimulationInputMode::ANTARES_API ||
@@ -114,14 +161,18 @@ std::filesystem::path ProblemGeneration::updateProblems() {
   auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout,
                                                   "Problem Generation"s);
 
+  set_solver(study_dir, logger.get());
+
+  if (mode_ == SimulationInputMode::ANTARES_API) {
+    performAntaresSimulation(simulation_dir_);
+  }
+
   auto master_formulation = options_.MasterFormulation();
   auto additionalConstraintFilename_l =
       options_.AdditionalConstraintsFilename();
   auto weights_file = options_.WeightsFile();
   auto unnamed_problems = options_.UnnamedProblems();
 
-  set_solver(study_dir, logger.get());
-
   RunProblemGeneration(xpansion_output_dir, master_formulation,
                        additionalConstraintFilename_l, archive_path, logger,
                        log_file_path, weights_file, unnamed_problems);
@@ -338,9 +389,12 @@ void ProblemGeneration::RunProblemGeneration(
       << "Problem Generation ran in: "
       << format_time_str(problem_generation_timer.elapsed()) << std::endl;
 }
-void ProblemGeneration::set_solver(std::filesystem::path study_dir, ProblemGenerationLog::ProblemGenerationLogger* logger) {
-    SettingsReader settingsReader(study_dir / "user" / "expansion" / "settings.ini", logger);
-    solver_name_ = settingsReader.Solver();
+void ProblemGeneration::set_solver(
+    std::filesystem::path study_dir,
+    ProblemGenerationLog::ProblemGenerationLogger* logger) {
+  SettingsReader settingsReader(
+      study_dir / "user" / "expansion" / "settings.ini", logger);
+  solver_name_ = settingsReader.Solver();
 }
 
 std::shared_ptr<ArchiveReader> InstantiateZipReader(

src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h

@@ -7,9 +7,10 @@
 #include <antares/solver/lps/LpsFromAntares.h>
 
 #include <filesystem>
-#include <string>
 #include <optional>
+#include <string>
 
+// clang-format off
 #include "antares-xpansion/lpnamer/input_reader/MpsTxtWriter.h"
 #include "antares-xpansion/lpnamer/model/Problem.h"
 #include "antares-xpansion/lpnamer/model/SimulationInputMode.h"
@@ -18,13 +19,14 @@
 #include "antares-xpansion/lpnamer/helper/ProblemGenerationLogger.h"
 #include "ProblemGenerationOptions.h"
 #include "antares-xpansion/multisolver_interface/SolverAbstract.h"
+// clang-format on
 
 class ProblemGeneration {
  public:
   explicit ProblemGeneration(ProblemGenerationOptions& options);
   virtual ~ProblemGeneration() = default;
   std::filesystem::path updateProblems();
-  ProblemGenerationOptions& options_;
+  const ProblemGenerationOptions& options_;
 
  private:
   virtual void RunProblemGeneration(
@@ -49,11 +51,13 @@ class ProblemGeneration {
       const std::vector<ProblemData>& mpsList, std::filesystem::path& lpDir_,
       std::shared_ptr<ArchiveReader> reader,
       const Antares::Solver::LpsFromAntares& lps);
-  virtual void set_solver(std::filesystem::path study_dir, ProblemGenerationLog::ProblemGenerationLogger* logger);
+  virtual void set_solver(
+      std::filesystem::path study_dir,
+      ProblemGenerationLog::ProblemGenerationLogger* logger);
 
   Antares::Solver::LpsFromAntares lps_;
   std::optional<SimulationInputMode> mode_;
-  virtual std::filesystem::path performAntaresSimulation();
+  virtual void performAntaresSimulation(const std::filesystem::path& output);
   std::filesystem::path simulation_dir_;
   std::string solver_name_;
 };

src/python/antares_xpansion/antares_driver.py

@@ -107,9 +107,9 @@ def _set_simulation_name(self):
     def _get_antares_cmd(self):
         cmd = [str(self.antares_exe_path), self.data_dir, self.ANTARES_N_CPU_OPTION, str(self.antares_n_cpu), self.zip_option]
         if self.use_xpress:
-            cmd.extend(["--use-ortools", "--ortools-solver=xpress"])
+            cmd.extend(["--solver=xpress"])
         else:
-            cmd.extend(["--use-ortools", "--ortools-solver=sirius"])
+            cmd.extend(["--solver=sirius"])
         simulator_version = version.parse(__antares_simulator_version__)
         simulator_version_with_named_mps = version.parse(self.FIRST_VERSION_WITH_NAMED_PROBLEMS)
         if (simulator_version.major > simulator_version_with_named_mps.major) or (simulator_version.major >= simulator_version_with_named_mps.major and simulator_version.minor >= simulator_version_with_named_mps.minor):

tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp

@@ -50,14 +50,10 @@ class ProblemGenerationSpyAndMock : public ProblemGeneration {
   }
 
  private:
-  std::filesystem::path performAntaresSimulation() override {
-    return options_.StudyPath() / "simulation";
-  }
+  void performAntaresSimulation(const std::filesystem::path&) override {}
   void set_solver(
       std::filesystem::path study_dir,
-      ProblemGenerationLog::ProblemGenerationLogger* logger) override {
-
-  }
+      ProblemGenerationLog::ProblemGenerationLogger* logger) override {}
 
  public:
   std::filesystem::path xpansion_output_dir_;

tests/end_to_end/cucumber/features/steps/steps.py

@@ -239,7 +239,7 @@ def get_results_file_path_from_logs(logs: bytes) -> str:
 
 @then('Simulator has been launched with solver "{string}"')
 def check_simulator_solver(context, string):
-    string_to_find = f"ortools solver {string} used for problem resolution"
+    string_to_find = f"solver {string} is used for problem resolution"
     assert(find_in_simulator_log(context.tmp_study / "output", string_to_find))
 
 @then('Benders has been launched with solver "{string}"')