[ANT-1912] Make Outer loop lib independant from Benders (#878)

data_test/external_loop_test/lp/master_last_basis.bss

@@ -1,5 +1,5 @@
 NAME                                                                          
  LL G_p_max_0_0                            0
- XU alpha             R1                          19             0
- XL alpha_0           R2                          19           -19
+ XU alpha             R1                         114             0
+ XL alpha_0           R2                         114          -114
 ENDATA 

docs/user-guide/get-started/launching-optimization.md

@@ -57,24 +57,23 @@ If the value is `last`, the most recent run will be used. This option only has a
 among `{problem_generation, benders, study_update, sensitivity}`.
 In a step by step workflow keep both _.zip_ file and _-Xpansion_ corresponding folder.
 
-#### `-m, --method {benders, benders_by_batch, mergeMPS, adequacy_criterion}`
+#### `-m, --method {benders, mergeMPS, adequacy_criterion}`
 
 Default value: `benders`. 
 
 Sets the optimization method used by Antares-Xpansion.
 
 | Option | Description                           |
 | ---------- | ----------------------------------------------------------------------- |
-| `benders` | Launch the classical Benders decomposition.
-| `benders_by_batch` | Launch the Benders by batch algorithm. |
+| `benders` | Launch the classical Benders decomposition or the Benders by batch algorithm depending on `batch_size`.
 | `mergeMPS`   | Launch a frontal resolution of the investment problem (i.e. without decomposition). This is much more time-consuming than using Benders decomposition.|
 | `adequacy_criterion`   | Launch Antares-Xpansion with reliability constraints, see [Adequacy criterion](adequacy-criterion.md). |
 
 #### `-n, --np`
 
 Default value: 2. 
 
-Sets the number of MPI processes to use for the Benders decomposition. This option only has an effect when `-m` is set to `benders` or `benders_by_batch`.
+Sets the number of MPI processes to use for the Benders decomposition. This option only has an effect when `-m` is set to `benders`.
 
 #### `--antares-n-cpu`
 

docs/user-guide/get-started/settings-definition.md

@@ -294,4 +294,4 @@ In the first iterations, the algorithm computes upper and lower bounds on the op
 
 Positive integer. Default value: `0`.
 
-This parameter is used only when `--method` is `benders_by_batch`, otherwise it has no effect. The `batch_size` specifies the number of subproblems per batch. If set to `0`, then all subproblems are in the same batch. In this case, the Benders by batch algorithm is launched with a single batch, which is mathematically equivalent to the classical Benders algorithm.
+This parameter specifies the number of subproblems per batch. If set to `0`, then all subproblems are in the same batch. In this case, the classical Benders is launched.

src/cpp/benders/benders_by_batch/BendersByBatch.cpp

@@ -49,7 +49,7 @@ void BendersByBatch::InitializeProblems() {
   }
 
   // if (Rank() == rank_0) {
-  SetSubproblemsVariablesIndex();
+  //  SetSubproblemsVariablesIndex();
   // }
   init_problems_ = false;
 }

src/cpp/benders/benders_core/BendersBase.cpp

@@ -9,25 +9,17 @@
 #include "LastIterationReader.h"
 #include "LastIterationWriter.h"
 #include "LogUtils.h"
-#include "VariablesGroup.h"
-
 #include "solver_utils.h"
 
 BendersBase::BendersBase(const BendersBaseOptions &options, Logger logger,
                          Writer writer,
                          std::shared_ptr<MathLoggerDriver> mathLoggerDriver)
-    : _options(options),
+    : _options(std::move(options)),
       _csv_file_path(std::filesystem::path(_options.OUTPUTROOT) /
                      (_options.CSV_NAME + ".csv")),
       _logger(std::move(logger)),
       _writer(std::move(writer)),
       mathLoggerDriver_(std::move(mathLoggerDriver)) {
-  if (options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
-    //TODO maybe the input format will change?
-    outer_loop_input_data_ =
-        Outerloop::OuterLoopInputFromYaml().Read(OuterloopOptionsFile());
-    outer_loop_biLevel_ = OuterLoopBiLevel(outer_loop_input_data_);
-  }
 }
 
 std::filesystem::path BendersBase::OuterloopOptionsFile() const {
@@ -390,27 +382,31 @@ void BendersBase::GetSubproblemCut(SubProblemDataMap &subproblem_data_map) {
             policy, nameAndWorkers.begin(), nameAndWorkers.end(),
             [this, &m, &subproblem_data_map](
                 const std::pair<std::string, SubproblemWorkerPtr> &kvp) {
-              const auto &[name, worker] = kvp;
-              Timer subproblem_timer;
               PlainData::SubProblemData subproblem_data;
-              worker->fix_to(_data.x_cut);
-              worker->solve(subproblem_data.lpstatus, _options.OUTPUTROOT,
-                            _options.LAST_MASTER_MPS + MPS_SUFFIX, _writer);
-              worker->get_value(subproblem_data.subproblem_cost);
-              if (_options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
-                std::vector<double> solution;
-                worker->get_solution(solution);
-                ComputeOuterLoopCriterion(name, solution, subproblem_data);
-              }
-              worker->get_subgradient(subproblem_data.var_name_and_subgradient);
-              worker->get_splex_num_of_ite_last(subproblem_data.simplex_iter);
-              subproblem_data.subproblem_timer = subproblem_timer.elapsed();
-              std::lock_guard guard(m);
+              const auto &[name, worker] = kvp;
+              SolveSubproblem(subproblem_data_map, subproblem_data, name,
+                              worker);
+
               subproblem_data_map[name] = subproblem_data;
+              std::lock_guard guard(m);
             });
       },
       shouldParallelize());
 }
+void BendersBase::SolveSubproblem(
+    SubProblemDataMap &subproblem_data_map,
+    PlainData::SubProblemData &subproblem_data, const std::string &name,
+    const std::shared_ptr<SubproblemWorker> &worker) {
+  Timer subproblem_timer;
+  worker->fix_to(_data.x_cut);
+  worker->solve(subproblem_data.lpstatus, _options.OUTPUTROOT,
+                _options.LAST_MASTER_MPS + MPS_SUFFIX, _writer);
+  worker->get_value(subproblem_data.subproblem_cost);
+
+  worker->get_subgradient(subproblem_data.var_name_and_subgradient);
+  worker->get_splex_num_of_ite_last(subproblem_data.simplex_iter);
+  subproblem_data.subproblem_timer = subproblem_timer.elapsed();
+}
 
 /*!
  *  \brief Add cut to Master Problem and store the cut in a set
@@ -779,19 +775,7 @@ void BendersBase::MatchProblemToId() {
   }
 }
 
-// Search for variables in sub problems that satify patterns
-// var_indices is a vector(for each patterns p) of vector (var indices related
-// to p)
-void BendersBase::SetSubproblemsVariablesIndex() {
-  if (!subproblem_map.empty() && _options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
-    auto subproblem = subproblem_map.begin();
-    subproblems_vars_names_.clear();
-    subproblems_vars_names_ = subproblem->second->_solver->get_col_names();
-    Outerloop::VariablesGroup variablesGroup(subproblems_vars_names_,
-                                             outer_loop_input_data_.OuterLoopData());
-    var_indices_ = variablesGroup.Indices();
-  }
-}
+
 
 void BendersBase::AddSubproblemName(const std::string &name) {
   subproblems.push_back(name);
@@ -1000,16 +984,6 @@ WorkerMasterData BendersBase::BestIterationWorkerMaster() const {
   return relevantIterationData_.best;
 }
 
-void BendersBase::InitExternalValues(bool is_bilevel_check_all, double lambda) {
-  // _data.outer_loop_current_iteration_data.outer_loop_criterion = 0;
-  // _data.outer_loop_current_iteration_data.benders_num_run = 1;
-  is_bilevel_check_all_ = is_bilevel_check_all;
-  outer_loop_biLevel_.Init(MasterObjectiveFunctionCoeffs(),
-                           BestIterationWorkerMaster().get_max_invest(),
-                           MasterVariables());
-  outer_loop_biLevel_.SetLambda(lambda);
-}
-
 CurrentIterationData BendersBase::GetCurrentIterationData() const {
   return _data;
 }
@@ -1022,39 +996,6 @@ std::vector<double> BendersBase::GetOuterLoopCriterionAtBestBenders() const {
               : outer_loop_criterion_[_data.best_it - 1]);
 }
 
-void BendersBase::ComputeOuterLoopCriterion(
-    const std::string &subproblem_name,
-    const std::vector<double> &sub_problem_solution,
-    PlainData::SubProblemData &subproblem_data) {
-  auto outer_loop_input_size = var_indices_.size(); // num of patterns
-  subproblem_data.outer_loop_criterions.resize(outer_loop_input_size, 0.);
-  subproblem_data.outer_loop_patterns_values.resize(outer_loop_input_size, 0.);
-
-  auto subproblem_weight = SubproblemWeight(_data.nsubproblem, subproblem_name);
-  double criterion_count_threshold =
-      outer_loop_input_data_.CriterionCountThreshold();
-
-  for (int pattern_index(0); pattern_index < outer_loop_input_size;
-       ++pattern_index) {
-    auto pattern_variables_indices = var_indices_[pattern_index];
-    for (auto variables_index : pattern_variables_indices) {
-      const auto &solution = sub_problem_solution[variables_index];
-      subproblem_data.outer_loop_patterns_values[pattern_index] += solution;
-      if (solution > criterion_count_threshold)
-        // 1h of no supplied energy
-        subproblem_data.outer_loop_criterions[pattern_index] +=
-            subproblem_weight;
-    }
-  }
-}
-
-double BendersBase::ExternalLoopLambdaMax() const {
-  return outer_loop_biLevel_.LambdaMax();
-}
-double BendersBase::ExternalLoopLambdaMin() const {
-  return outer_loop_biLevel_.LambdaMin();
-}
-
 void BendersBase::init_data(double external_loop_lambda,
                             double external_loop_lambda_min,
                             double external_loop_lambda_max) {
@@ -1076,19 +1017,23 @@ void BendersBase::init_data(double external_loop_lambda,
       external_loop_lambda_max;
 }
 
-bool BendersBase::ExternalLoopFoundFeasible() const {
-  return outer_loop_biLevel_.FoundFeasible();
-}
-double BendersBase::OuterLoopStoppingThreshold() const { return outer_loop_input_data_.StoppingThreshold(); }
 
-void BendersBase::UpdateOuterLoopMaxCriterionArea()  {
-  auto criterions_begin =
-      _data.outer_loop_current_iteration_data.outer_loop_criterion.cbegin();
-  auto criterions_end =
-      _data.outer_loop_current_iteration_data.outer_loop_criterion.cend();
-  auto max_criterion_it = std::max_element(criterions_begin, criterions_end);
-  _data.outer_loop_current_iteration_data.max_criterion = *max_criterion_it;
-  auto  max_criterion_index = std::distance(criterions_begin, max_criterion_it);
- _data.outer_loop_current_iteration_data.max_criterion_area = outer_loop_input_data_.OuterLoopData()[max_criterion_index].Pattern().GetBody();
-}
+
 bool BendersBase::isExceptionRaised() const { return exception_raised_; }
+/*
+ * after the 1st loop of the outer loop, we must  re-build the objective
+ * function and costs
+ */
+void BendersBase::UpdateOverallCosts() {
+  auto obj = MasterObjectiveFunctionCoeffs();
+  _data.invest_cost = 0;
+  for (const auto &[var_name, var_id] : MasterVariables()) {
+    _data.invest_cost += obj[var_id] * _data.x_cut.at(var_name);
+  }
+
+  relevantIterationData_.best._invest_cost = _data.invest_cost;
+}
+void BendersBase::SetBilevelBestub(double bilevel_best_ub) {
+  _data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub =
+      bilevel_best_ub;
+}

src/cpp/benders/benders_core/BendersMathLogger.cpp

@@ -83,6 +83,14 @@ LogDestination::LogDestination(const std::filesystem::path& file_path,
   }
 }
 
+void MathLoggerBehaviour::write_header() {
+  setHeadersList();
+  for (const auto& header : Headers()) {
+    LogsDestination() << header;
+  }
+  LogsDestination() << std::endl;
+}
+
 void MathLoggerBehaviour::PrintIterationSeparatorBegin() {
   std::string sep_msg("/*\\");
   sep_msg += std::string(74, '-');
@@ -95,6 +103,23 @@ void MathLoggerBehaviour::PrintIterationSeparatorEnd() {
   LogsDestination() << sep_msg << std::endl;
 }
 
+MathLogger::MathLogger(const std::filesystem::path& file_path,
+                       std::streamsize width, HEADERSTYPE type)
+    : log_destination_(file_path, width), type_(type) {}
+
+MathLogger::MathLogger(std::streamsize width, HEADERSTYPE type)
+    : log_destination_(width), type_(type) {}
+
+void MathLogger::display_message(const std::string& str) {
+  LogsDestination() << str << std::endl;
+}
+
+std::vector<std::string> MathLogger::Headers() const { return headers_; }
+
+LogDestination& MathLogger::LogsDestination() { return log_destination_; }
+
+HEADERSTYPE MathLogger::HeadersType() const { return type_; }
+
 void MathLoggerBase::Print(const CurrentIterationData& data) {
   PrintBendersData(LogsDestination(), data, HeadersType(),
                    BENDERSMETHOD::BENDERS);
@@ -164,9 +189,6 @@ void PrintExternalLoopData(LogDestination& log_destination,
                            const HEADERSTYPE& type,
                            const BENDERSMETHOD& method) {
   log_destination << data.outer_loop_current_iteration_data.benders_num_run;
-  // TODO
-  // log_destination << std::scientific << std::setprecision(10)
-  //                 << data.outer_loop_criterion;
   log_destination << std::scientific << std::setprecision(10)
                   << data.outer_loop_current_iteration_data.max_criterion;
   log_destination << data.outer_loop_current_iteration_data.max_criterion_area;
@@ -303,3 +325,31 @@ MathLoggerImplementation::MathLoggerImplementation(const BENDERSMETHOD& method,
 MathLoggerImplementation::MathLoggerImplementation(
     std::shared_ptr<MathLogger> implementation)
     : implementation_(std::move(implementation)) {}
+
+void MathLoggerImplementation::display_message(const std::string& str) {
+  implementation_->display_message(str);
+}
+
+void MathLoggerImplementation::Print(const CurrentIterationData& data) {
+  implementation_->Print(data);
+}
+
+void MathLoggerImplementation::PrintIterationSeparatorBegin() {
+  implementation_->PrintIterationSeparatorBegin();
+}
+
+void MathLoggerImplementation::PrintIterationSeparatorEnd() {
+  implementation_->PrintIterationSeparatorEnd();
+}
+
+void MathLoggerImplementation::setHeadersList() {
+  implementation_->setHeadersList();
+}
+
+std::vector<std::string> MathLoggerImplementation::Headers() const {
+  return implementation_->Headers();
+}
+
+LogDestination& MathLoggerImplementation::LogsDestination() {
+  return implementation_->LogsDestination();
+}

src/cpp/benders/benders_core/CMakeLists.txt

@@ -6,7 +6,6 @@
 # Targets
 # ===========================================================================
 
-find_package(yaml-cpp CONFIG REQUIRED)
 add_library (benders_core STATIC
 	${CMAKE_CURRENT_SOURCE_DIR}/SubproblemWorker.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/SimulationOptions.cpp
@@ -20,9 +19,8 @@ add_library (benders_core STATIC
 	${CMAKE_CURRENT_SOURCE_DIR}/LastIterationPrinter.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/StartUp.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/BendersMathLogger.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/VariablesGroup.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/OuterLoopBiLevel.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/OuterLoopInputDataReader.cpp
+		${CMAKE_CURRENT_SOURCE_DIR}/MasterUpdateBase.cpp
+		${CMAKE_CURRENT_SOURCE_DIR}/CutsManagement.cpp
 
 	)
 
@@ -38,7 +36,5 @@ target_link_libraries (benders_core
 		output_core
 		solvers
 		${JSONCPP_LIB}
-		yaml-cpp::yaml-cpp
+		outer_loop_lib
 )
-
-add_library (${PROJECT_NAME}::benders_core ALIAS benders_core)