Merge branch 'develop' into release/1.3.1

data_test/external_loop_test/lp/options.json

@@ -24,5 +24,5 @@
     "LAST_MASTER_MPS": "master_last_iteration",
     "LAST_MASTER_BASIS": "master_last_basis.bss",
     "DO_OUTER_LOOP": true,
-    "OUTER_LOOP_OPTION_FILE": "data_test/external_loop_test/lp/outer_loop_options.yml"
+    "OUTER_LOOP_OPTION_FILE": "data_test/external_loop_test/lp/adequacy_criterion.yml"
 }

docs/index.md

@@ -44,7 +44,7 @@ Antares-Xpansion is currently under development. Feel free to submit any issue.
 
 
 [ubuntu_system_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ubuntu-release.yml/badge.svg?query=branch%3Adevelop
-[ubuntu_release]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ubuntu-release.yml?query=branch%3Adevelop
+[ubuntu_system_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ubuntu-release.yml?query=branch%3Adevelop
 [windows_vcpkg_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/windows-vcpkg.yml/badge.svg?query=branch%3Adevelop
 [windows_vcpkg_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/windows-vcpkg.yml?query=branch%3Adevelop
 [centos_system_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/centos-release.yml/badge.svg?query=branch%3Adevelop

docs/user-guide/get-started/adequacy-criterion.md

@@ -0,0 +1,49 @@
+# Adequacy criterion
+
+By default, Antares-Xpansion tries to solve an investment problem by minimising the sum of expected operational costs and investment cost. Nothing guarantees that the expected number of hours of loss of load is under a certain reliability threshold. 
+
+However, Antares-Xpansion is able to enforce solutions to satisfy a maximum number of expected hours of loss of load per area. To use this feature, the user must:
+
+1. Define the reliability constraints in an input file `user/expansion/adequacy_criterion/adequacy_criterion.yml`.
+2. Launch the optimization with the `-m adequacy_criterion` flag.
+
+The resolution of the reliability-constrained investment problem is based on a heuristic that is described in [Reliability-constrained investment problem](../optimization-principles/problem-formalization.md#reliability-constrained-investment-problem).
+
+## Input data for adequacy criterion in `adequacy_criterion.yml`
+
+The input file for the adequacy criterion should be located in `antares-study-folder/user/expansion/adequacy_criterion/adequacy_criterion.yml`. It is in YAML format and follows the following template:
+
+```yaml
+stopping_threshold: 1e4
+criterion_count_threshold: 1
+patterns:
+  - area: "fr"
+    criterion: 3
+  - area: "de"
+    criterion: 5
+```
+
+- `stopping_threshold` in euros (positive float): Stopping criterion of the algorithm, that is the difference between `lambda_min` and `lambda_max`, see [Reliability-constrained investment problem](../optimization-principles/problem-formalization.md#reliability-constrained-investment-problem).
+    - Default value: `1e-4`
+- `criterion_count_threshold` in MWh (positive float): Number of MWh of unsupplied energy in a given hour to consider that this is an hour with loss of load (ex: An hour is considered to be with loss of load if there is more than 1 MWh of unsupplied energy).
+    - Default value: `1`
+- `patterns`: A list giving the area names and the associated criterion.
+    - required at least a pair of (criterion; area)
+    - `area` (string): Name of the area
+    - `criterion` in hours (positive float): Maximum expected number of hours with loss of load over all scenarios for the corresponding area
+
+## Launching the optimization with the adequacy criterion
+
+To launch the optimization with the adequacy criterion, simply use the option `-m adequacy_criterion` in the command line:
+```
+antares-xpansion-launcher.exe -i examples\SmallTestFiveCandidates -m adequacy_criterion
+```
+
+## Outputs
+
+The final solution can be read in the file `output/<simulation-name>/expansion/out.json`, in the field `solution`.
+
+Several log files are written:
+
+- `reportbenders.txt` gives information on the progress of the algorithm with an operational perspective,
+- `benders_solver.log` contains more detailed information on all data of interest to follow the progress of the algorithm (`lambda_min`, `lambda_max`, detailed solving times, ...).

docs/user-guide/get-started/get-started.rst

@@ -9,4 +9,5 @@ Getting started
     Define the candidates <candidate-definition>
     Settings of the algorithm <settings-definition>
     Launch the optimization <launching-optimization>
+    Adequacy criterion <adequacy-criterion>
     Sensitivity analysis <sensitivity-analysis>

docs/user-guide/get-started/installation.md

@@ -8,6 +8,7 @@ This software suite has been tested under:
 *   Ubuntu 20.04 [![Status][ubuntu_system_svg]][ubuntu_system_link]
 *   Microsoft Windows with Visual Studio 2019 (64-bit) [![Status][windows_vcpkg_svg]][windows_vcpkg_link]
 *   Centos 7 [![Status][centos_system_svg]][centos_system_link]
+*   Oracle Linux 8 [![Status][oracle_svg]][oracle_link]
 
 Antares-Xpansion is built using CMake.
 
@@ -27,10 +28,12 @@ antares-xpansion-launcher.exe -i examples\SmallTestFiveCandidates
 
 See [Launch the optimization](launching-optimization.md) for more details on how to launch the software.
 
-[ubuntu_system_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/linux-system.yml/badge.svg
-[ubuntu_system_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/linux-system.yml
-[windows_vcpkg_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/workflows/Windows%20CI%20(VCPKG)/badge.svg
-[windows_vcpkg_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions?query=workflow%3A"Windows%20CI%20(VCPKG)"
-[centos_system_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/workflows/Centos7%20CI%20(system%20libs)/badge.svg
-[centos_system_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions?query=workflow%3A"Centos7%20CI%20(system%20libs)"
+[ubuntu_system_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ubuntu-release.yml/badge.svg?query=branch%3Adevelop
+[ubuntu_system_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ubuntu-release.yml?query=branch%3Adevelop
+[windows_vcpkg_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/windows-vcpkg.yml/badge.svg?query=branch%3Adevelop
+[windows_vcpkg_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/windows-vcpkg.yml?query=branch%3Adevelop
+[centos_system_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/centos-release.yml/badge.svg?query=branch%3Adevelop
+[centos_system_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/centos-release.yml?query=branch%3Adevelop
+[oracle_svg]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ol8-release.yml/badge.svg?query=branch%3Adevelop
+[oracle_link]: https://github.com/AntaresSimulatorTeam/antares-xpansion/actions/workflows/ol8-release.yml?query=branch%3Adevelop
 [antares_xpansion_release_url]: https://github.com/AntaresSimulatorTeam/antares-xpansion/releases

docs/user-guide/get-started/launching-optimization.md

@@ -57,7 +57,7 @@ If the value is `last`, the most recent run will be used. This option only has a
 among `{problem_generation, benders, study_update, sensitivity}`.
 In a step by step workflow keep both _.zip_ file and _-Xpansion_ corresponding folder.
 
-#### `-m, --method {benders, benders_by_batch, mergeMPS}`
+#### `-m, --method {benders, benders_by_batch, mergeMPS, adequacy_criterion}`
 
 Default value: `benders`. 
 
@@ -68,6 +68,7 @@ Sets the optimization method used by Antares-Xpansion.
 | `benders` | Launch the classical Benders decomposition.
 | `benders_by_batch` | Launch the Benders by batch algorithm. |
 | `mergeMPS`   | Launch a frontal resolution of the investment problem (i.e. without decomposition). This is much more time-consuming than using Benders decomposition.|
+| `adequacy_criterion`   | Launch Antares-Xpansion with reliability constraints, see [Adequacy criterion](adequacy-criterion.md). |
 
 #### `-n, --np`
 
@@ -108,6 +109,7 @@ investment combination has been found, the
 package produces an archive `simulation-name.zip` located in the `output` folder of the Antares study.
 
 Once you unzip the archive, the files `lp/reportbenders.txt` and the `expansion/out.json` log information for each iteration:
+
 1. The investment combination that has been evaluated,
 2. The operational, investment and overall costs, 
 3. The current lower and upper bounds (no upper bound for the Benders by batch),

docs/user-guide/optimization-principles/investment-problem.md

@@ -113,31 +113,3 @@ decomposition - is an iterative method, which for each iteration:
   These derivatives are also called Bender cuts.
 
 A mathematical formulation of the optimization problem solved by Antares-Xpansion as well as details on the Benders decomposition method ar given in [Mathematical formulation of the investment problem](../optimization-principles/problem-formalization.md).
-
-## Notes on the computation time
-
-The Benders decomposition method has been shown to converge towards the optimal solution of the investment problem described above and is also commonly used to solve large
-stochastic problems. The number of iterations required to reach the
-optimum strongly depends on the structure of the problem and on the
-variants/algorithmic parameters used. In general, it strongly increases with the number of defined investment variables, see **Table 1**.
-
-| **Number of investment candidates** | **Order of magnitude of the number of iterations** |
-| ----------------------------------- | -------------------------------------------------- |
-| 5                                   | 10                                                 |
-| 10                                  | 40                                                 |
-| 25                                  | 100                                                |
-| 50                                  | 300                                                |
-| 100                                 | 800                                                |
-
-**Table 1** - Order of magnitude of the number of iterations
-required to reach the optimum with Antares-Xpansion v0.12.
-
-Each iteration of the Antares-Xpansion algorithm includes an Antares
-simulation. However, the simulation of "operational" studies of several
-tens of nodes and with several hundred Monte-Carlo scenarios (TYNDP,
-Generation adequacy report on the electricity supply-demand, etc.)
-requires a significant amount of computing time, sometimes several
-hours. The search for the optimal solution to the problem solved by
-Antares-Xpansion can therefore be relatively long, and in some cases
-requires a simplification of the problem being solved.
-

docs/user-guide/optimization-principles/problem-formalization.md

@@ -183,4 +183,18 @@ The optimality gap is the difference (either absolute or relative) between the l
 
 ### The Benders by batch algorithm
 
-In the classical Benders algorithm, all subproblems are solved at each iteration, resulting in \\(52 N\\) resolutions and as many cuts are added in the master problem. This can be quite time-consuming. The idea of the Benders by batch algorithm is to solve subproblems by batch and stop solving them at a given iteration once we know that the master solution is not optimal. Each iteration consists in fewer subproblems resolution (and fewer cuts added to the master) but we need more iterations to converge. Overall the tradeoff makes the Benders by batch algorithm usually faster than the classical Benders method. A comprehensive description of the Benders by batch algorithm can be found in the thesis of Xavier Blanchot[@blanchot_solving_2022].
+In the classical Benders algorithm, all subproblems are solved at each iteration, resulting in \\(52 N\\) resolutions and as many cuts are added in the master problem. This can be quite time-consuming. The idea of the Benders by batch algorithm is to solve subproblems by batch and stop solving them at a given iteration once we know that the master solution is not optimal. Each iteration consists in fewer subproblems resolution (and fewer cuts added to the master) but we need more iterations to converge. Overall the tradeoff makes the Benders by batch algorithm usually faster than the classical Benders method. A comprehensive description of the Benders by batch algorithm can be found in the thesis of Xavier Blanchot[@blanchot_solving_2022].
+
+## Reliability-constrained investment problem
+
+Starting from version 1.3.0, Antares-Xpansion can take into account a reliability constraint on the maximum expected number of hours of loss of load. This means that the user is able to specify, for each area, an expected number of hours of loss a load that should not be exceeded, see [Adequacy criterion](../get-started/adequacy-criterion.md). Antares-Xpansion will output a solution that satisfy this reliability criterion using a Benders-based matheuristic designed in Chapter 5 of the thesis of Xavier Blanchot[@blanchot_solving_2022].
+
+The heuristic is based on the insight that increasing the investment cost is strongly correlated to a decrease in loss of load. It works by iteratively solving the classical investment problem (without the reliability constraint) to which we add a minimum investment cost constraint \\(c^{\top}x \geq \Lambda\\). 
+
+After each resolution, the expected loss of load is checked and the minimum investment cost \\(\Lambda\\) is adjusted using a dichotomy of an interval \\([\underline{\Lambda}, \overline{\Lambda}]\\). If we denote by \\(\Lambda^{(k)}, \underline{\Lambda}^{(k)}, \overline{\Lambda}^{(k)}\\) the values from the \\(k\\)-th resolution of the investment problem:
+
+- If there is too much loss of load, we increase \\(\underline{\Lambda}\\) : \\(\underline{\Lambda}^{(k+1)} = \Lambda^{(k)}\\)`, 
+- Otherwise, we decrease \\(\overline{\Lambda}\\): \\(\overline{\Lambda}^{(k+1)} = \Lambda^{(k)}\\)`.
+- Then, we take \\(\Lambda^{(k+1)} = \frac{\overline{\Lambda}^{(k+1)}-\underline{\Lambda}^{(k+1)}}{2}\\)
+
+The algorithm stops when \\(\overline{\Lambda}^{(k+1)}-\underline{\Lambda}^{(k+1)} < \varepsilon\\) where \\(\varepsilon\\) is user-defined. Antares-Xpansion outputs a feasible solution for the reliability-constrained that should be of good quality thanks to the initial insight linking investment cost and loss of load. This procedure is a heuristic so there is no guarantee to get the theoretical optimal solution.

mkdocs.yml

@@ -44,6 +44,7 @@ nav:
         - 'Define the candidates': 'user-guide/get-started/candidate-definition.md'
         - 'Settings of the algorithm': 'user-guide/get-started/settings-definition.md'
       - 'Launch the optimization': 'user-guide/get-started/launching-optimization.md'
+      - 'Adequacy criterion': 'user-guide/get-started/adequacy-criterion.md'
       - 'Sensitivity analysis': 'user-guide/get-started/sensitivity-analysis.md'
   - 'Developer guide': 
     - 'Install from sources':
@@ -85,8 +86,8 @@ markdown_extensions:
       permalink: true
       toc_depth: 3
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - admonition
   - pymdownx.details
   - pymdownx.superfences

src/cpp/benders/benders_core/BendersBase.cpp

@@ -400,8 +400,7 @@ void BendersBase::GetSubproblemCut(SubProblemDataMap &subproblem_data_map) {
               if (_options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
                 std::vector<double> solution;
                 worker->get_solution(solution);
-                subproblem_data.outer_loop_criterions =
-                    ComputeOuterLoopCriterion(name, solution);
+                ComputeOuterLoopCriterion(name, solution, subproblem_data);
               }
               worker->get_subgradient(subproblem_data.var_name_and_subgradient);
               worker->get_splex_num_of_ite_last(subproblem_data.simplex_iter);
@@ -1023,12 +1022,14 @@ std::vector<double> BendersBase::GetOuterLoopCriterionAtBestBenders() const {
               : outer_loop_criterion_[_data.best_it - 1]);
 }
 
-std::vector<double> BendersBase::ComputeOuterLoopCriterion(
+void BendersBase::ComputeOuterLoopCriterion(
     const std::string &subproblem_name,
-    const std::vector<double> &sub_problem_solution) {
+    const std::vector<double> &sub_problem_solution,
+    PlainData::SubProblemData &subproblem_data) {
   auto outer_loop_input_size = var_indices_.size(); // num of patterns
-  std::vector<double> outer_loop_criterion_per_sub_problem(outer_loop_input_size,
-                                                           {});
+  subproblem_data.outer_loop_criterions.resize(outer_loop_input_size, 0.);
+  subproblem_data.outer_loop_patterns_values.resize(outer_loop_input_size, 0.);
+
   auto subproblem_weight = SubproblemWeight(_data.nsubproblem, subproblem_name);
   double criterion_count_threshold =
       outer_loop_input_data_.CriterionCountThreshold();
@@ -1037,14 +1038,14 @@ std::vector<double> BendersBase::ComputeOuterLoopCriterion(
        ++pattern_index) {
     auto pattern_variables_indices = var_indices_[pattern_index];
     for (auto variables_index : pattern_variables_indices) {
-      if (auto solution = sub_problem_solution[variables_index];
-          solution > criterion_count_threshold)
+      const auto &solution = sub_problem_solution[variables_index];
+      subproblem_data.outer_loop_patterns_values[pattern_index] += solution;
+      if (solution > criterion_count_threshold)
         // 1h of no supplied energy
-        outer_loop_criterion_per_sub_problem[pattern_index] +=
+        subproblem_data.outer_loop_criterions[pattern_index] +=
             subproblem_weight;
     }
   }
-  return outer_loop_criterion_per_sub_problem;
 }
 
 double BendersBase::ExternalLoopLambdaMax() const {

src/cpp/benders/benders_core/BendersMathLogger.cpp

@@ -99,6 +99,7 @@ void MathLoggerBase::Print(const CurrentIterationData& data) {
   PrintBendersData(LogsDestination(), data, HeadersType(),
                    BENDERSMETHOD::BENDERS);
 }
+
 void MathLoggerBase::setHeadersList() {
   auto type = HeadersType();
   HeadersManager headers_manager(type, BENDERSMETHOD::BENDERS);
@@ -298,3 +299,7 @@ MathLoggerImplementation::MathLoggerImplementation(const BENDERSMETHOD& method,
       break;
   }
 }
+
+MathLoggerImplementation::MathLoggerImplementation(
+    std::shared_ptr<MathLogger> implementation)
+    : implementation_(std::move(implementation)) {}

src/cpp/benders/benders_core/OuterLoopInputDataReader.cpp

@@ -2,6 +2,8 @@
 
 #include <utility>
 
+#include "StringManip.h"
+
 using namespace Outerloop;
 
 /**
@@ -56,6 +58,23 @@ std::vector<OuterLoopSingleInputData> OuterLoopInputData::OuterLoopData()
   return outer_loop_data_;
 }
 
+std::vector<std::string> OuterLoopInputData::PatternBodies() const {
+  std::vector<std::string> ret;
+  for (const auto &data : outer_loop_data_) {
+    ret.push_back(data.Pattern().GetBody());
+  }
+  return ret;
+}
+
+std::string OuterLoopInputData::PatternsPrefix() const {
+  std::string ret("");
+  if (!outer_loop_data_.empty()) {
+    ret =
+        StringManip::split(outer_loop_data_[0].Pattern().GetPrefix(), "::")[0];
+  }
+  return ret;
+}
+
 void OuterLoopInputData::SetStoppingThreshold(
     double outer_loop_stopping_threshold) {
   outer_loop_stopping_threshold_ = outer_loop_stopping_threshold;
@@ -94,22 +113,6 @@ OuterLoopInputData OuterLoopInputFromYaml::Read(
   return yaml_content.as<OuterLoopInputData>();
 }
 
-/*
-# critère d'arrêt de l'algo
-stopping_threshold: 1e-4
-# seuil
-criterion_count_threshold: 1e-1
-patterns:
-  - area: "N0"
-    criterion: 1
-  - area: "N1"
-    criterion: 1
-  - area: "N2"
-    criterion: 1
-  - area: "N3"
-    criterion: 1
-*/
-
 namespace YAML {
 
 template <>

src/cpp/benders/benders_core/include/BendersBase.h

@@ -235,9 +235,10 @@ class BendersBase {
   SolverLogManager solver_log_manager_;
 
   // outer loop criterion per pattern
-  std::vector<double> ComputeOuterLoopCriterion(
+  void ComputeOuterLoopCriterion(
       const std::string &subproblem_name,
-      const std::vector<double> &sub_problem_solution);
+      const std::vector<double> &sub_problem_solution,
+      PlainData::SubProblemData &subproblem_data);
 
   void UpdateOuterLoopMaxCriterionArea();
   void UpdateOuterLoopSolution();