Feature/low level logs

cmake/dependencies/CMakeLists.txt

@@ -9,7 +9,7 @@ if (NOT antares-solver_FOUND)
 
     set(REPOSITORY "https://github.com/AntaresSimulatorTeam/Antares_Simulator.git")
     set(TAG "v${ANTARES_VERSION_TAG}")
-    set(CMAKE_ARGS "-DBUILD_UI=OFF -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DDEPS_INSTALL_DIR=${DEPS_INSTALL_DIR} -DBUILD_not_system=OFF -DBUILD_ortools=ON")
+    set(CMAKE_ARGS "-DBUILD_UI=OFF -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DDEPS_INSTALL_DIR=${DEPS_INSTALL_DIR} -DBUILD_not_system=OFF -DBUILD_ortools=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_DIR}/../antares-xpansion/vcpkg_installed")
 
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
       set(ANTARES_BUILD_TYPE "debug")

docs/user-guide/get-started/settings-definition.md

@@ -21,6 +21,7 @@ The following section lists the configurable parameters. If the user does not sp
 |[`separation_parameter`](#separation_parameter) | `0.5` | Step size for the in-out separation |
 |[`relaxed_optimality_gap`](#relaxed_optimality_gap) | `1e-5` | Threshold to switch from relaxed to integer master |
 |[`batch_size`](#batch_size) | `0` | Number of subproblems per batch |
+|[`expert_logs`](#expert_logs) | `false` | display pure optimization data (/!\ the corresponding option(`EXPERT_LOGS`) for benders is `true` by default ) |
 
 The format is a standard `.ini` and should follow this template:
 ```ini

src/cpp/benders/benders_by_batch/BendersByBatch.cpp

@@ -8,10 +8,11 @@
 #include "BatchCollection.h"
 #include "RandomBatchShuffler.h"
 #include "glog/logging.h"
-BendersByBatch::BendersByBatch(BendersBaseOptions const &options, Logger logger,
-                               Writer writer, mpi::environment &env,
-                               mpi::communicator &world)
-    : BendersMpi(options, logger, writer, env, world) {}
+BendersByBatch::BendersByBatch(
+    BendersBaseOptions const &options, Logger logger, Writer writer,
+    mpi::environment &env, mpi::communicator &world,
+    std::shared_ptr<MathLoggerDriver> mathLoggerDriver)
+    : BendersMpi(options, logger, writer, env, world, mathLoggerDriver) {}
 
 void BendersByBatch::InitializeProblems() {
   MatchProblemToId();
@@ -75,8 +76,8 @@ void BendersByBatch::MasterLoop() {
   random_batch_permutation_.resize(number_of_batch_);
   batch_counter_ = 0;
   current_batch_id_ = 0;
-
-  _data.number_of_subproblem_resolved = 0;
+  _data.number_of_subproblem_solved = 0;
+  _data.cumulative_number_of_subproblem_solved = 0;
   cumulative_subproblems_timer_per_iter_ = 0;
   first_unsolved_batch_ = 0;
   while (!_data.stop) {
@@ -112,21 +113,22 @@ void BendersByBatch::MasterLoop() {
               random_batch_permutation_.size(), rank_0);
     SeparationLoop();
     if (Rank() == rank_0) {
-      _data.elapsed_time = GetBendersTime();
       _data.stop = ShouldBendersStop();
     }
     BroadCast(_data.stop, rank_0);
     BroadCast(batch_counter_, rank_0);
     SetSubproblemsCumulativeCpuTime(cumulative_subproblems_timer_per_iter_);
-    _logger->cumulative_number_of_sub_problem_resolved(
-        _data.number_of_subproblem_resolved +
-        GetNumOfSubProblemsResolvedBeforeResume());
+    _logger->cumulative_number_of_sub_problem_solved(
+        _data.cumulative_number_of_subproblem_solved +
+        GetNumOfSubProblemsSolvedBeforeResume());
     _logger->LogSubproblemsSolvingCumulativeCpuTime(
         GetSubproblemsCumulativeCpuTime());
     _logger->LogSubproblemsSolvingWalltime(GetSubproblemsWalltime());
     _logger->display_message(
         "\\________________________________________________________________"
         "________");
+    _data.elapsed_time = GetBendersTime();
+    mathLoggerDriver_->Print(_data);
   }
 }
 void BendersByBatch::SeparationLoop() {
@@ -135,13 +137,16 @@ void BendersByBatch::SeparationLoop() {
   batch_counter_ = 0;
   while (misprice_ && batch_counter_ < number_of_batch_) {
     _data.it++;
+    ResetSimplexIterationsBounds();
 
     _logger->log_at_initialization(_data.it + GetNumIterationsBeforeRestart());
     ComputeXCut();
     _logger->log_iteration_candidates(bendersDataToLogData(_data));
     BroadcastXCut();
     UpdateRemainingEpsilon();
+    _data.number_of_subproblem_solved = 0;
     SolveBatches();
+
     if (Rank() == rank_0) {
       UpdateTrace();
       SaveCurrentBendersData();
@@ -195,7 +200,8 @@ void BendersByBatch::SolveBatches() {
     Reduce(GetSubproblemsCpuTime(), cumulative_subproblems_timer_per_iter_,
            std::plus<double>(), rank_0);
     if (Rank() == rank_0) {
-      _data.number_of_subproblem_resolved += batch_sub_problems.size();
+      _data.number_of_subproblem_solved += batch_sub_problems.size();
+      _data.cumulative_number_of_subproblem_solved += batch_sub_problems.size();
       remaining_epsilon_ -= batch_subproblems_costs_contribution_in_gap;
     }
 
@@ -229,6 +235,7 @@ void BendersByBatch::BuildCut(
   for (const auto &subproblem_map : gathered_subproblem_map) {
     for (auto &&[_, subproblem_data] : subproblem_map) {
       SetSubproblemCost(GetSubproblemCost() + subproblem_data.subproblem_cost);
+      BoundSimplexIterations(subproblem_data.simplex_iter);
     }
   }
   for (const auto &subproblem_map : gathered_subproblem_map) {

src/cpp/benders/benders_by_batch/include/BendersByBatch.h

@@ -9,8 +9,8 @@ class BendersByBatch : public BendersMpi {
 
  public:
   BendersByBatch(BendersBaseOptions const &options, Logger logger,
-                 Writer writer, mpi::environment &env,
-                 mpi::communicator &world);
+                 Writer writer, mpi::environment &env, mpi::communicator &world,
+                 std::shared_ptr<MathLoggerDriver> mathLoggerDriver);
   ~BendersByBatch() override = default;
   void Run() override;
   void BuildCut(const std::vector<std::string> &batch_sub_problems,

src/cpp/benders/benders_core/BendersBase.cpp

@@ -13,12 +13,14 @@
 #include "solver_utils.h"
 
 BendersBase::BendersBase(BendersBaseOptions options, Logger logger,
-                         Writer writer)
+                         Writer writer,
+                         std::shared_ptr<MathLoggerDriver> mathLoggerDriver)
     : _options(std::move(options)),
       _csv_file_path(std::filesystem::path(_options.OUTPUTROOT) /
                      (_options.CSV_NAME + ".csv")),
       _logger(std::move(logger)),
-      _writer(std::move(writer)) {}
+      _writer(std::move(writer)),
+      mathLoggerDriver_(mathLoggerDriver) {}
 
 /*!
  *  \brief Initialize set of data used in the loop
@@ -34,7 +36,7 @@ void BendersBase::init_data() {
   _data.best_it = 0;
   _data.stopping_criterion = StoppingCriterion::empty;
   _data.is_in_initial_relaxation = false;
-  _data.number_of_subproblem_resolved = 0;
+  _data.cumulative_number_of_subproblem_solved = 0;
 }
 
 void BendersBase::OpenCsvFile() {
@@ -467,7 +469,7 @@ LogData BendersBase::FinalLogData() const {
   result.subproblem_cost = best_iteration_data.subproblem_cost;
   result.invest_cost = best_iteration_data.invest_cost;
   result.cumulative_number_of_subproblem_resolved =
-      _data.number_of_subproblem_resolved +
+      _data.cumulative_number_of_subproblem_solved +
       cumulative_number_of_subproblem_resolved_before_resume;
 
   return result;
@@ -525,7 +527,7 @@ Output::Iteration BendersBase::iteration(
       masterDataPtr_l->_invest_cost + masterDataPtr_l->_operational_cost;
   iteration.candidates = candidates_data(masterDataPtr_l);
   iteration.cumulative_number_of_subproblem_resolved =
-      _data.number_of_subproblem_resolved +
+      _data.cumulative_number_of_subproblem_solved +
       cumulative_number_of_subproblem_resolved_before_resume;
   return iteration;
 }
@@ -651,7 +653,7 @@ LogData BendersBase::bendersDataToLogData(
           data.elapsed_time,
           data.timer_master,
           data.subproblems_walltime,
-          data.number_of_subproblem_resolved +
+          data.cumulative_number_of_subproblem_solved +
               cumulative_number_of_subproblem_resolved_before_resume};
 }
 void BendersBase::set_log_file(const std::filesystem::path &log_file) {
@@ -773,6 +775,24 @@ void BendersBase::SetSubproblemCost(const double &subproblem_cost) {
   _data.subproblem_cost = subproblem_cost;
 }
 
+/*!
+*	\brief Update maximum and minimum of simplex iterations
+*
+*	\param subproblem_iterations : number of iterations done with the subproblem
+*
+*/
+void BendersBase::BoundSimplexIterations(int subproblem_iterations){
+
+  _data.max_simplexiter = (_data.max_simplexiter < subproblem_iterations) ? subproblem_iterations : _data.max_simplexiter; 
+  _data.min_simplexiter = (_data.min_simplexiter > subproblem_iterations) ? subproblem_iterations : _data.min_simplexiter; 
+
+}
+
+void BendersBase::ResetSimplexIterationsBounds()
+{
+	_data.max_simplexiter = 0;
+	_data.min_simplexiter = std::numeric_limits<int>::max();
+}
 bool BendersBase::IsResumeMode() const { return _options.RESUME; }
 
 void BendersBase::UpdateMaxNumberIterationResumeMode(
@@ -793,7 +813,7 @@ LogData BendersBase::GetBestIterationData() const {
 }
 
 void BendersBase::ChecksResumeMode() {
-  benders_timer = Timer();
+  ITE_TIMEr = Timer();
   if (IsResumeMode()) {
     auto reader = LastIterationReader(LastIterationFile());
     LogData last_iter;
@@ -809,7 +829,7 @@ void BendersBase::ChecksResumeMode() {
         LastIterationPrinter(_logger, best_iteration_data, last_iter);
     restart_data_printer.Print();
     UpdateMaxNumberIterationResumeMode(last_iter.it);
-    benders_timer = Timer(last_iter.benders_elapsed_time);
+    ITE_TIMEr = Timer(last_iter.benders_elapsed_time);
     _data.stop = ShouldBendersStop();
     iterations_before_resume = last_iter.it;
     cumulative_number_of_subproblem_resolved_before_resume =
@@ -836,7 +856,7 @@ void BendersBase::EndWritingInOutputFile() const {
   _writer->write_duration(_data.elapsed_time);
   SaveSolutionInOutputFile();
 }
-double BendersBase::GetBendersTime() const { return benders_timer.elapsed(); }
+double BendersBase::GetBendersTime() const { return ITE_TIMEr.elapsed(); }
 void BendersBase::write_basis() const {
   const auto filename(std::filesystem::path(_options.OUTPUTROOT) /
                       (_options.LAST_MASTER_BASIS));

src/cpp/benders/benders_core/BendersMathLogger.cpp

@@ -0,0 +1,82 @@
+#include "BendersMathLogger.h"
+
+// #ifdef _WIN32
+// #include <Windows.h>
+// #endif
+
+HeadersManager::HeadersManager(HEADERSTYPE type, const BENDERSMETHOD& method) {
+  if (type == HEADERSTYPE::SHORT) {
+    headers_list.push_back("ITE");
+    headers_list.push_back("LB");
+    if (method == BENDERSMETHOD::BENDERS) {
+      headers_list.push_back("UB");
+      headers_list.push_back("BESTUB");
+      headers_list.push_back("AGAP");
+      headers_list.push_back("RGAP");
+    }
+    headers_list.push_back("MinSpx");
+    headers_list.push_back("MaxSpx");
+    if (method == BENDERSMETHOD::BENDERSBYBATCH) {
+      headers_list.push_back("NbSubPbSolv");
+    }
+    headers_list.push_back("IteTime");
+    headers_list.push_back("MasterTime");
+    // headers_list.push_back("SubPbCpuTime");
+    headers_list.push_back("SPWallTime");
+  } else {
+    headers_list.push_back("ITERATION");
+    headers_list.push_back("LB");
+    if (method == BENDERSMETHOD::BENDERS) {
+      headers_list.push_back("UB");
+      headers_list.push_back("BESTUB");
+      headers_list.push_back("ABSOLUTE GAP");
+      headers_list.push_back("RELATIVE GAP");
+    }
+    headers_list.push_back("MINSIMPLEX");
+    headers_list.push_back("MAXSIMPLEX");
+    headers_list.push_back("NUMBER OF SUBPROBLEMS SOLVED");
+    headers_list.push_back("CUMULATIVE NUMBER OF SUBPROBLEMS SOLVED ");
+    headers_list.push_back("ITERATION TIME");
+    headers_list.push_back("MASTER TIME");
+    headers_list.push_back("SUB-PROBLEMS TIME (CPU)");
+    headers_list.push_back("SUB-PROBLEMS TIME (WALL)");
+    headers_list.push_back("TIME NOT DOING MASTER OR SUB-PROBLEMS (WALL)");
+  }
+}
+
+LogDestination::LogDestination(std::ostream* stream, std::streamsize width)
+    : stream_(stream), width_(width) {
+  // _COORD coordinates;
+  // coordinates.X = 1000;
+  // coordinates.Y = 1000;
+
+  // if (0 == SetConsoleScreenBufferSize(GetStdHandle(STD_OUTPUT_HANDLE),
+  //                                     coordinates)) {
+  //   std::cout << "could not resize the console screen\n";
+  //   // return -1;
+  // }
+}
+void MathLoggerDriver::add_logger(
+    std::shared_ptr<MathLoggerImplementation> logger) {
+  if (logger) {
+    math_loggers_.push_back(logger);
+  }
+}
+
+void MathLoggerDriver::Print(const CurrentIterationData& data) {
+  for (auto logger : math_loggers_) {
+    logger->Print(data);
+  }
+}
+
+void MathLoggerDriver::write_header() {
+  for (auto logger : math_loggers_) {
+    logger->write_header();
+  }
+}
+
+void MathLoggerDriver::display_message(const std::string& str) {
+  for (auto logger : math_loggers_) {
+    logger->display_message(str);
+  }
+}

src/cpp/benders/benders_core/BendersStructsDatas.cpp

@@ -8,3 +8,4 @@ Point WorkerMasterData::get_x_cut() const { return *_x_cut; }
 Point WorkerMasterData::get_min_invest() const { return *_min_invest; }
 
 Point WorkerMasterData::get_max_invest() const { return *_max_invest; }
+

src/cpp/benders/benders_core/CMakeLists.txt

@@ -26,7 +26,9 @@ add_library (benders_core STATIC
 	${CMAKE_CURRENT_SOURCE_DIR}/LastIterationWriter.cpp 
 	${CMAKE_CURRENT_SOURCE_DIR}/LastIterationReader.cpp 
 	${CMAKE_CURRENT_SOURCE_DIR}/LastIterationPrinter.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/StartUp.cpp)
+	${CMAKE_CURRENT_SOURCE_DIR}/StartUp.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/BendersMathLogger.cpp
+	)
 
 get_target_property(JSON_INC_PATH jsoncpp_lib INTERFACE_INCLUDE_DIRECTORIES)
 

src/cpp/benders/benders_core/SimulationOptions.cpp

@@ -112,7 +112,7 @@ void SimulationOptions::set_weights() {
 void SimulationOptions::print(std::ostream &stream) const {
 #define BENDERS_OPTIONS_MACRO(name__, type__, default__, \
                               deserialization_method__)  \
-  stream << std::setw(30) << #name__ << std::setw(50) << name__ << std::endl;
+  stream << std::setw(30) << #name__ << std::setw(50)<<std::boolalpha << name__ << std::endl;
 #include "SimulationOptions.hxx"
 #undef BENDERS_OPTIONS_MACRO
   stream << std::endl;
@@ -167,6 +167,7 @@ BendersBaseOptions SimulationOptions::get_benders_options() const {
   result.LAST_MASTER_MPS = LAST_MASTER_MPS;
   result.LAST_MASTER_BASIS = LAST_MASTER_BASIS;
   result.BATCH_SIZE = BATCH_SIZE;
+  result.EXPERT_LOGS = EXPERT_LOGS;
 
   return result;
 }

src/cpp/benders/benders_core/include/BendersBase.h

@@ -3,6 +3,7 @@
 #include <execution>
 #include <filesystem>
 
+#include "BendersMathLogger.h"
 #include "BendersStructsDatas.h"
 #include "ILogger.h"
 #include "OutputWriter.h"
@@ -29,7 +30,8 @@ auto selectPolicy(lambda f, bool shouldParallelize) {
 class BendersBase {
  public:
   virtual ~BendersBase() = default;
-  BendersBase(BendersBaseOptions options, Logger logger, Writer writer);
+  BendersBase(BendersBaseOptions options, Logger logger, Writer writer,
+              std::shared_ptr<MathLoggerDriver> mathLoggerDriver);
   virtual void launch() = 0;
   void set_log_file(const std::filesystem::path &log_name);
   [[nodiscard]] std::filesystem::path log_name() const { return _log_name; }
@@ -40,6 +42,7 @@ class BendersBase {
   CurrentIterationData _data;
   VariableMap master_variable_map;
   CouplingMap coupling_map;
+  std::shared_ptr<MathLoggerDriver> mathLoggerDriver_;
 
  protected:
   virtual void free() = 0;
@@ -132,10 +135,13 @@ class BendersBase {
   BendersBaseOptions Options() const { return _options; }
   virtual void UpdateStoppingCriterion();
   virtual bool ShouldRelaxationStop() const;
-  int GetNumOfSubProblemsResolvedBeforeResume() {
+  int GetNumOfSubProblemsSolvedBeforeResume() {
     return cumulative_number_of_subproblem_resolved_before_resume;
   }
 
+  void BoundSimplexIterations(int subproblem_iteration);
+  void ResetSimplexIterationsBounds();
+
   SolverLogManager solver_log_manager_;
 
  private:
@@ -171,7 +177,7 @@ class BendersBase {
   LogData best_iteration_data;
   int iterations_before_resume = 0;
   int cumulative_number_of_subproblem_resolved_before_resume = 0;
-  Timer benders_timer;
+  Timer ITE_TIMEr;
 
  public:
   Logger _logger;