From 90fd6146b11219add7be28b0feccd740417159f8 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 30 Oct 2024 16:25:02 +0100 Subject: [PATCH 01/19] Changelog --- docs/changelog/CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index e6c85e116..2f4e55354 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -1,6 +1,23 @@ Antares-Xpansion changelog ================= +v1.4.0 (09/2024) +-------------------------------------------------------- +### Features +* Update Antares Simulator to 9.2-RC4 +* Properly handle Antares Simulator errors. Especially error -9 occurring when running out of memory where Xpansion tried to continue in an improper state. + +### For developers +* Various improvement on dependency management through vcpkg. +* Update and improve documentation +* Use devtoolset 11 for CentOS and OL8 +* Add some tests using cucumber + +### New Contributors +* @pet-mit made their first contribution in https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/944 + +**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v2.0-RC1 + v1.3.2 (07/2024) -------------------------------------------------------- ### Bug Fixes From af4fb6d3e07f8d02c6194df743237430c893a301 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 30 Oct 2024 16:27:14 +0100 Subject: [PATCH 02/19] Version number --- CMakeLists.txt | 4 ++-- antares-version.json | 4 ++-- sonar-project.properties | 2 +- vcpkg.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6faf48e68..5711bd8e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) # Project configuration # =========================================================================== -PROJECT(antaresXpansion VERSION 1.3.2) -set(ANTARES_XPANSION_RC 0) +PROJECT(antaresXpansion VERSION 1.4.0) +set(ANTARES_XPANSION_RC 1) # =========================================================================== # Default parameters diff --git a/antares-version.json b/antares-version.json index 543a1eb99..78c22d625 100644 --- a/antares-version.json +++ b/antares-version.json @@ -1,7 +1,7 @@ { - "antares_version": "9.2.0-rc1", + "antares_version": "9.2.0-rc4", "antares_version_executable": "9.2", - "antares_xpansion_version": "1.3.2", + "antares_xpansion_version": "1.4.0", "minizip_ng_version": "3.0.6", "or-tools-rte": "9.11-rte1.1" } diff --git a/sonar-project.properties b/sonar-project.properties index c49ba0a4c..7e17684d9 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ sonar.projectName=antares-xpansion sonar.projectKey=AntaresSimulatorTeam_antares-xpansion sonar.organization=antaressimulatorteam -sonar.projectVersion=1.3.2 +sonar.projectVersion=1.4.0 # ===================================================== # Properties that will be shared amongst all modules diff --git a/vcpkg.json b/vcpkg.json index c006b78fc..615446ed0 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,6 @@ { "name": "antares-xpansion", - "version-string": "1.3.2", + "version-string": "1.4.0", "$comment 1": "libuuid by transitivity for Simulator-> yuni", "$comment 2": "sirius-solver by transitivity for Simulator", "dependencies": [ From 8f784eaecf1e25c4e7c6f787e9ff1bd5cc8f007a Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Wed, 30 Oct 2024 17:04:20 +0100 Subject: [PATCH 03/19] Update changelog --- docs/changelog/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 2f4e55354..74df8113c 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -5,6 +5,8 @@ v1.4.0 (09/2024) -------------------------------------------------------- ### Features * Update Antares Simulator to 9.2-RC4 +* Output file **criterions.txt** is renamed **LOLD.txt** as it gathers the loss of load for each area at each iteration of the algorithm +* Support for in memory problems between Antares and problem generation * Properly handle Antares Simulator errors. Especially error -9 occurring when running out of memory where Xpansion tried to continue in an improper state. ### For developers @@ -16,7 +18,7 @@ v1.4.0 (09/2024) ### New Contributors * @pet-mit made their first contribution in https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/944 -**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v2.0-RC1 +**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v1.4.0-RC1 v1.3.2 (07/2024) -------------------------------------------------------- From 2d3f7c09e708c5d24479a0b60cb4d05659d7c26b Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Wed, 30 Oct 2024 17:06:14 +0100 Subject: [PATCH 04/19] Correct link --- docs/changelog/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 74df8113c..c859f0f62 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -18,7 +18,7 @@ v1.4.0 (09/2024) ### New Contributors * @pet-mit made their first contribution in https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/944 -**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v1.4.0-RC1 +**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v1.4.0-rc1 v1.3.2 (07/2024) -------------------------------------------------------- From 7872222d804f87ed3e6fb6915b062dc36de7327d Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 15 Nov 2024 15:46:59 +0100 Subject: [PATCH 05/19] Update rc2 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 455d7cd59..e80233c4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) # =========================================================================== PROJECT(antaresXpansion VERSION 1.4.0) -set(ANTARES_XPANSION_RC 1) +set(ANTARES_XPANSION_RC 2) # =========================================================================== # Default parameters From 4e7f0d163ddf382f0561427e2d2046448b89e13e Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Wed, 11 Dec 2024 18:27:56 +0100 Subject: [PATCH 06/19] Update changelog and rc number --- CMakeLists.txt | 2 +- docs/changelog/CHANGELOG.md | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0589c0281..7aa392731 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) # =========================================================================== PROJECT(antaresXpansion VERSION 1.4.0) -set(ANTARES_XPANSION_RC 2) +set(ANTARES_XPANSION_RC 3) # =========================================================================== # Default parameters diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index c859f0f62..819dd1c7d 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -1,12 +1,14 @@ Antares-Xpansion changelog ================= -v1.4.0 (09/2024) +v1.4.0 (12/2024) -------------------------------------------------------- ### Features -* Update Antares Simulator to 9.2-RC4 +* Update Antares Simulator to 9.2-RC5 * Output file **criterions.txt** is renamed **LOLD.txt** as it gathers the loss of load for each area at each iteration of the algorithm +* End to end coherence of solvers : if Xpress is selected, it is also used for Antares step (does not work yet with memory mode), if coin is selected, sirius is used in Antares * Support for in memory problems between Antares and problem generation +* Remove local UI support [#972](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/972) * Properly handle Antares Simulator errors. Especially error -9 occurring when running out of memory where Xpansion tried to continue in an improper state. ### For developers @@ -14,11 +16,14 @@ v1.4.0 (09/2024) * Update and improve documentation * Use devtoolset 11 for CentOS and OL8 * Add some tests using cucumber +* Improve maintainability [#965](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/965) +* Fix performance regression with missing TBB dependency [#970](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/970) +* Add script to convert markdown docs for support in xwiki [#973](https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/973) ### New Contributors * @pet-mit made their first contribution in https://github.com/AntaresSimulatorTeam/antares-xpansion/pull/944 -**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v1.4.0-rc1 +**Full Changelog**: https://github.com/AntaresSimulatorTeam/antares-xpansion/compare/v1.3.2...v1.4.0-rc3 v1.3.2 (07/2024) -------------------------------------------------------- From cd784dee96fa07b742793775f637fd24dcdac214 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Fri, 13 Dec 2024 13:57:47 +0100 Subject: [PATCH 07/19] Update to antares v9.2.0-rc7 --- CMakeLists.txt | 2 +- antares-version.json | 2 +- docs/changelog/CHANGELOG.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aa392731..0bf47bfd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.5) # =========================================================================== PROJECT(antaresXpansion VERSION 1.4.0) -set(ANTARES_XPANSION_RC 3) +set(ANTARES_XPANSION_RC 4) # =========================================================================== # Default parameters diff --git a/antares-version.json b/antares-version.json index cbaa4f6ab..017a6eabf 100644 --- a/antares-version.json +++ b/antares-version.json @@ -1,5 +1,5 @@ { - "antares_version": "9.2.0-RC5", + "antares_version": "9.2.0-RC7", "antares_version_executable": "9.2", "antares_xpansion_version": "1.4.0", "minizip_ng_version": "3.0.6", diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 819dd1c7d..1d170917a 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -4,7 +4,7 @@ Antares-Xpansion changelog v1.4.0 (12/2024) -------------------------------------------------------- ### Features -* Update Antares Simulator to 9.2-RC5 +* Update Antares Simulator to 9.2-RC7 * Output file **criterions.txt** is renamed **LOLD.txt** as it gathers the loss of load for each area at each iteration of the algorithm * End to end coherence of solvers : if Xpress is selected, it is also used for Antares step (does not work yet with memory mode), if coin is selected, sirius is used in Antares * Support for in memory problems between Antares and problem generation From 316d6e7f4398c46140a08c4323d5cca5bf574ab9 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Fri, 13 Dec 2024 14:48:44 +0100 Subject: [PATCH 08/19] Typo correction --- antares-version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/antares-version.json b/antares-version.json index 017a6eabf..616c1bc7b 100644 --- a/antares-version.json +++ b/antares-version.json @@ -1,5 +1,5 @@ { - "antares_version": "9.2.0-RC7", + "antares_version": "9.2.0-rc7", "antares_version_executable": "9.2", "antares_xpansion_version": "1.4.0", "minizip_ng_version": "3.0.6", From a12396a9ae18ab952531381f0168f0557da5e393 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Fri, 13 Dec 2024 15:06:45 +0100 Subject: [PATCH 09/19] Fix compilation with new Antares API --- src/cpp/lpnamer/main/ProblemGeneration.cpp | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/cpp/lpnamer/main/ProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp index b2a480a5a..2c006e3a4 100644 --- a/src/cpp/lpnamer/main/ProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/ProblemGeneration.cpp @@ -1,15 +1,13 @@ -#include -#include - #include "antares-xpansion/lpnamer/main/ProblemGeneration.h" #include +#include +#include #include #include -#include "antares-xpansion/lpnamer/input_reader/SettingsReader.h" #include "Version.h" #include "antares-xpansion/helpers/Timer.h" #include "antares-xpansion/lpnamer/helper/ProblemGenerationLogger.h" @@ -64,7 +62,9 @@ ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options) } std::filesystem::path ProblemGeneration::performAntaresSimulation() { - auto results = Antares::API::PerformSimulation(options_.StudyPath()); + Antares::Solver::Optimization::OptimizationOptions optOptions; + auto results = + Antares::API::PerformSimulation(options_.StudyPath(), optOptions); // Add parallel // Handle errors @@ -86,8 +86,9 @@ std::filesystem::path ProblemGeneration::updateProblems() { if (mode_ == SimulationInputMode::ARCHIVE) { xpansion_output_dir = options_.deduceXpansionDirIfEmpty(xpansion_output_dir, archive_path); - study_dir = std::filesystem::absolute(archive_path).parent_path().parent_path(); - //Assume study/output/archive.zip + study_dir = + std::filesystem::absolute(archive_path).parent_path().parent_path(); + // Assume study/output/archive.zip } if (mode_ == SimulationInputMode::ANTARES_API) { @@ -99,7 +100,9 @@ std::filesystem::path ProblemGeneration::updateProblems() { simulation_dir_ = options_.XpansionOutputDir(); // Legacy naming. // options_.XpansionOutputDir() point in fact to a simulation output from // antares - study_dir = std::filesystem::absolute(simulation_dir_).parent_path().parent_path(); //Assume study/output/simulation + study_dir = std::filesystem::absolute(simulation_dir_) + .parent_path() + .parent_path(); // Assume study/output/simulation } if (mode_ == SimulationInputMode::ANTARES_API || @@ -338,9 +341,12 @@ void ProblemGeneration::RunProblemGeneration( << "Problem Generation ran in: " << format_time_str(problem_generation_timer.elapsed()) << std::endl; } -void ProblemGeneration::set_solver(std::filesystem::path study_dir, ProblemGenerationLog::ProblemGenerationLogger* logger) { - SettingsReader settingsReader(study_dir / "user" / "expansion" / "settings.ini", logger); - solver_name_ = settingsReader.Solver(); +void ProblemGeneration::set_solver( + std::filesystem::path study_dir, + ProblemGenerationLog::ProblemGenerationLogger* logger) { + SettingsReader settingsReader( + study_dir / "user" / "expansion" / "settings.ini", logger); + solver_name_ = settingsReader.Solver(); } std::shared_ptr InstantiateZipReader( From baa047930dd3dfba3bab6cfe58ff9759cfa3a389 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Fri, 13 Dec 2024 16:09:44 +0100 Subject: [PATCH 10/19] Fix Antares command line --- src/python/antares_xpansion/antares_driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/antares_xpansion/antares_driver.py b/src/python/antares_xpansion/antares_driver.py index d98953170..65b7634c6 100644 --- a/src/python/antares_xpansion/antares_driver.py +++ b/src/python/antares_xpansion/antares_driver.py @@ -107,9 +107,9 @@ def _set_simulation_name(self): def _get_antares_cmd(self): cmd = [str(self.antares_exe_path), self.data_dir, self.ANTARES_N_CPU_OPTION, str(self.antares_n_cpu), self.zip_option] if self.use_xpress: - cmd.extend(["--use-ortools", "--ortools-solver=xpress"]) + cmd.extend(["--solver=xpress"]) else: - cmd.extend(["--use-ortools", "--ortools-solver=sirius"]) + cmd.extend(["--solver=sirius"]) simulator_version = version.parse(__antares_simulator_version__) simulator_version_with_named_mps = version.parse(self.FIRST_VERSION_WITH_NAMED_PROBLEMS) if (simulator_version.major > simulator_version_with_named_mps.major) or (simulator_version.major >= simulator_version_with_named_mps.major and simulator_version.minor >= simulator_version_with_named_mps.minor): From b4f25c2c95cf71fd24d9127abe94f1b52453da7d Mon Sep 17 00:00:00 2001 From: Florian OMNES Date: Mon, 16 Dec 2024 10:31:02 +0100 Subject: [PATCH 11/19] Try to fix dummy-3areas-3candidates-3links example --- .../dummy-3areas-3candidates-3links/settings/generaldata.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_test/examples/dummy-3areas-3candidates-3links/settings/generaldata.ini b/data_test/examples/dummy-3areas-3candidates-3links/settings/generaldata.ini index 03ea230ed..a82676dbf 100644 --- a/data_test/examples/dummy-3areas-3candidates-3links/settings/generaldata.ini +++ b/data_test/examples/dummy-3areas-3candidates-3links/settings/generaldata.ini @@ -90,3 +90,5 @@ seed-thermal-costs = 8005489 seed-hydro-costs = 9005489 seed-initial-reservoir-levels = 10005489 +[compatibility] +hydro-pmax = hourly \ No newline at end of file From cc5f8e0bb5603cfee160795b512528ddd7dbde96 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Mon, 16 Dec 2024 11:09:00 +0100 Subject: [PATCH 12/19] Fix test --- tests/end_to_end/cucumber/features/steps/steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end_to_end/cucumber/features/steps/steps.py b/tests/end_to_end/cucumber/features/steps/steps.py index c54f9d671..13180d379 100644 --- a/tests/end_to_end/cucumber/features/steps/steps.py +++ b/tests/end_to_end/cucumber/features/steps/steps.py @@ -239,7 +239,7 @@ def get_results_file_path_from_logs(logs: bytes) -> str: @then('Simulator has been launched with solver "{string}"') def check_simulator_solver(context, string): - string_to_find = f"ortools solver {string} used for problem resolution" + string_to_find = f"solver {string} used for problem resolution" assert(find_in_simulator_log(context.tmp_study / "output", string_to_find)) @then('Benders has been launched with solver "{string}"') From ccac65d06a3a59f548d03553d8157530da18b7c3 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Mon, 16 Dec 2024 13:05:16 +0100 Subject: [PATCH 13/19] Fix test --- tests/end_to_end/cucumber/features/steps/steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end_to_end/cucumber/features/steps/steps.py b/tests/end_to_end/cucumber/features/steps/steps.py index 13180d379..96b701a9d 100644 --- a/tests/end_to_end/cucumber/features/steps/steps.py +++ b/tests/end_to_end/cucumber/features/steps/steps.py @@ -239,7 +239,7 @@ def get_results_file_path_from_logs(logs: bytes) -> str: @then('Simulator has been launched with solver "{string}"') def check_simulator_solver(context, string): - string_to_find = f"solver {string} used for problem resolution" + string_to_find = f"solver {string} is used for problem resolution" assert(find_in_simulator_log(context.tmp_study / "output", string_to_find)) @then('Benders has been launched with solver "{string}"') From b54256bb90e97805057926378402d43439fde1c8 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Mon, 16 Dec 2024 13:23:51 +0100 Subject: [PATCH 14/19] Fix antares driver tests --- tests/python/test_antares_driver.py | 84 +++++++++++++++++++---------- 1 file changed, 57 insertions(+), 27 deletions(-) diff --git a/tests/python/test_antares_driver.py b/tests/python/test_antares_driver.py index 8f3c8faef..d3ebe4aac 100644 --- a/tests/python/test_antares_driver.py +++ b/tests/python/test_antares_driver.py @@ -2,18 +2,18 @@ import os from pathlib import Path from unittest.mock import patch -import pytest -from packaging import version +import pytest +from antares_xpansion.__version__ import __antares_simulator_version__ from antares_xpansion.antares_driver import AntaresDriver from antares_xpansion.general_data_processor import ( GeneralDataFileExceptions, GeneralDataProcessor, ) -from antares_xpansion.general_data_reader import IniReader, GeneralDataIniReader +from antares_xpansion.general_data_reader import GeneralDataIniReader, IniReader +from packaging import version from tests.build_config_reader import get_antares_solver_path -from antares_xpansion.__version__ import __antares_simulator_version__ SUBPROCESS_RUN = "antares_xpansion.antares_driver.subprocess.run" @@ -111,7 +111,6 @@ def test_values_change_in_general_file_accurate_mode(self, tmp_path): "key2 = value2\n" "[input]\n" "import = blabla\n" - ) gen_data_path.write_text(default_val) @@ -209,15 +208,16 @@ def test_values_change_in_general_file_fast_mode(self, tmp_path): ) def verify_that_general_data_contains_expected_vals( - self, general_data_ini_file, expected_val + self, general_data_ini_file, expected_val ): actual_config = configparser.ConfigParser() actual_config.read(general_data_ini_file) - for (section, key) in expected_val: + for section, key in expected_val: value = actual_config.get(section, key, fallback=None) assert value is not None print( - f"Section {section}, key {key}, value {value}, expected {expected_val[(section, key)]}") + f"Section {section}, key {key}, value {value}, expected {expected_val[(section, key)]}" + ) assert value == expected_val[(section, key)] with open(general_data_ini_file, "r") as reader: @@ -250,9 +250,15 @@ def test_antares_cmd(self, tmp_path): # mock subprocess.run with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, 1) - expected_cmd = [exe_path, study_dir, "--force-parallel", "1", "-z", "--use-ortools", - "--ortools-solver=sirius"] - if (self.nammed_problems): + expected_cmd = [ + exe_path, + study_dir, + "--force-parallel", + "1", + "-z", + "--solver=sirius", + ] + if self.nammed_problems: expected_cmd.append("--named-mps-problems") run_function.assert_called_once_with( @@ -267,9 +273,15 @@ def test_antares_cmd_force_parallel_option(self, tmp_path): with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [exe_path, study_dir, - "--force-parallel", str(n_cpu), "-z", "--use-ortools", "--ortools-solver=sirius"] - if (self.nammed_problems): + expected_cmd = [ + exe_path, + study_dir, + "--force-parallel", + str(n_cpu), + "-z", + "--solver=sirius", + ] + if self.nammed_problems: expected_cmd.append("--named-mps-problems") run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 @@ -288,9 +300,10 @@ def test_invalid_n_cpu(self, tmp_path): study_dir, "--force-parallel", str(expected_n_cpu), - "-z", "--use-ortools", "--ortools-solver=sirius" + "-z", + "--solver=sirius", ] - if (self.nammed_problems): + if self.nammed_problems: expected_cmd.append("--named-mps-problems") run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 @@ -305,9 +318,15 @@ def test_remove_log_file(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [str(exe_path), study_dir, - "--force-parallel", str(n_cpu), "-z", "--use-ortools", "--ortools-solver=sirius"] - if (self.nammed_problems): + expected_cmd = [ + str(exe_path), + study_dir, + "--force-parallel", + str(n_cpu), + "-z", + "--solver=sirius", + ] + if self.nammed_problems: expected_cmd.append("--named-mps-problems") run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 @@ -368,9 +387,15 @@ def test_preserve_adequacy_option_after_run(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [str(exe_path), study_dir, - "--force-parallel", str(n_cpu), "-z", "--use-ortools", "--ortools-solver=sirius"] - if (self.nammed_problems): + expected_cmd = [ + str(exe_path), + study_dir, + "--force-parallel", + str(n_cpu), + "-z", + "--solver=sirius", + ] + if self.nammed_problems: expected_cmd.append("--named-mps-problems") run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 @@ -380,8 +405,7 @@ def test_preserve_adequacy_option_after_run(self, tmp_path): config_reader.read(gen_data_path) assert config_reader.getboolean("adequacy patch", "dummy") is False assert config_reader.get("adequacy patch", "foo") == "bar" - assert config_reader.getboolean( - "adequacy patch", "include-adq-patch") is True + assert config_reader.getboolean("adequacy patch", "include-adq-patch") is True def test_preserve_general_file_section_missing(self, tmp_path): settings_dir = TestGeneralDataProcessor.get_settings_dir(tmp_path) @@ -394,9 +418,15 @@ def test_preserve_general_file_section_missing(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [str(exe_path), study_dir, - "--force-parallel", str(n_cpu), "-z", "--use-ortools", "--ortools-solver=sirius"] - if (self.nammed_problems): + expected_cmd = [ + str(exe_path), + study_dir, + "--force-parallel", + str(n_cpu), + "-z", + "--solver=sirius", + ] + if self.nammed_problems: expected_cmd.append("--named-mps-problems") run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 From f9c72b06d8567d3bf267574fd7ae950b845468b0 Mon Sep 17 00:00:00 2001 From: Florian OMNES Date: Fri, 20 Dec 2024 14:01:47 +0100 Subject: [PATCH 15/19] Add const qualifier, call set_solver earlier --- src/cpp/lpnamer/main/ProblemGeneration.cpp | 4 ++-- .../include/antares-xpansion/lpnamer/main/ProblemGeneration.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cpp/lpnamer/main/ProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp index 2c006e3a4..8586537ac 100644 --- a/src/cpp/lpnamer/main/ProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/ProblemGeneration.cpp @@ -83,6 +83,8 @@ std::filesystem::path ProblemGeneration::updateProblems() { const auto archive_path = options_.ArchivePath(); std::filesystem::path study_dir; + set_solver(study_dir, logger.get()); + if (mode_ == SimulationInputMode::ARCHIVE) { xpansion_output_dir = options_.deduceXpansionDirIfEmpty(xpansion_output_dir, archive_path); @@ -123,8 +125,6 @@ std::filesystem::path ProblemGeneration::updateProblems() { auto weights_file = options_.WeightsFile(); auto unnamed_problems = options_.UnnamedProblems(); - set_solver(study_dir, logger.get()); - RunProblemGeneration(xpansion_output_dir, master_formulation, additionalConstraintFilename_l, archive_path, logger, log_file_path, weights_file, unnamed_problems); diff --git a/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h b/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h index e74caa310..076ba23f7 100644 --- a/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h +++ b/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h @@ -24,7 +24,7 @@ class ProblemGeneration { explicit ProblemGeneration(ProblemGenerationOptions& options); virtual ~ProblemGeneration() = default; std::filesystem::path updateProblems(); - ProblemGenerationOptions& options_; + const ProblemGenerationOptions& options_; private: virtual void RunProblemGeneration( From 2817f49a378fd0a3cde7de43005db541c5c00aab Mon Sep 17 00:00:00 2001 From: Florian OMNES Date: Fri, 20 Dec 2024 16:13:41 +0100 Subject: [PATCH 16/19] Use updated antares-solver API (output path changes) [ANT-2561] --- src/cpp/lpnamer/main/ProblemGeneration.cpp | 50 ++++++++++++++++--- .../lpnamer/main/ProblemGeneration.h | 10 ++-- .../ProblemGenerationExeOptionsTest.cpp | 8 +-- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/cpp/lpnamer/main/ProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp index 8586537ac..d19494d01 100644 --- a/src/cpp/lpnamer/main/ProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/ProblemGeneration.cpp @@ -61,10 +61,19 @@ ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options) } } -std::filesystem::path ProblemGeneration::performAntaresSimulation() { +static std::string lowerCase(std::string data) { + std::transform(data.begin(), data.end(), data.begin(), + [](unsigned char c) { return std::tolower(c); }); + return data; +} + +void ProblemGeneration::performAntaresSimulation( + const std::filesystem::path& output) { Antares::Solver::Optimization::OptimizationOptions optOptions; + + optOptions.ortoolsSolver = lowerCase(solver_name_); auto results = - Antares::API::PerformSimulation(options_.StudyPath(), optOptions); + Antares::API::PerformSimulation(options_.StudyPath(), output, optOptions); // Add parallel // Handle errors @@ -74,7 +83,32 @@ std::filesystem::path ProblemGeneration::performAntaresSimulation() { } lps_ = std::move(results.antares_problems); - return {results.simulationPath}; +} + +static std::string getCurrentTimestamp() { + // Get the current time point + auto now = std::chrono::system_clock::now(); + + // Convert to time_t for formatting + std::time_t now_time_t = std::chrono::system_clock::to_time_t(now); + + // Convert to tm structure + std::tm now_tm; +#ifdef _WIN32 + localtime_s(&now_tm, &now_time_t); // Windows-specific +#else + localtime_r(&now_time_t, &now_tm); // POSIX-specific +#endif + + // Format the timestamp + std::ostringstream oss; + oss << std::put_time(&now_tm, "%Y%m%d-%H%Meco"); + return oss.str(); +} + +// Useful only for "API" mode +std::filesystem::path generateOutputName(const std::filesystem::path& study) { + return study / "output" / getCurrentTimestamp(); } std::filesystem::path ProblemGeneration::updateProblems() { @@ -83,8 +117,6 @@ std::filesystem::path ProblemGeneration::updateProblems() { const auto archive_path = options_.ArchivePath(); std::filesystem::path study_dir; - set_solver(study_dir, logger.get()); - if (mode_ == SimulationInputMode::ARCHIVE) { xpansion_output_dir = options_.deduceXpansionDirIfEmpty(xpansion_output_dir, archive_path); @@ -94,8 +126,8 @@ std::filesystem::path ProblemGeneration::updateProblems() { } if (mode_ == SimulationInputMode::ANTARES_API) { - simulation_dir_ = performAntaresSimulation(); study_dir = options_.StudyPath(); + simulation_dir_ = generateOutputName(study_dir); } if (mode_ == SimulationInputMode::FILE) { @@ -119,6 +151,12 @@ std::filesystem::path ProblemGeneration::updateProblems() { auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout, "Problem Generation"s); + set_solver(study_dir, logger.get()); + + if (mode_ == SimulationInputMode::ANTARES_API) { + performAntaresSimulation(simulation_dir_); + } + auto master_formulation = options_.MasterFormulation(); auto additionalConstraintFilename_l = options_.AdditionalConstraintsFilename(); diff --git a/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h b/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h index 076ba23f7..6a7275edc 100644 --- a/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h +++ b/src/cpp/lpnamer/main/include/antares-xpansion/lpnamer/main/ProblemGeneration.h @@ -7,9 +7,10 @@ #include #include -#include #include +#include +// clang-format off #include "antares-xpansion/lpnamer/input_reader/MpsTxtWriter.h" #include "antares-xpansion/lpnamer/model/Problem.h" #include "antares-xpansion/lpnamer/model/SimulationInputMode.h" @@ -18,6 +19,7 @@ #include "antares-xpansion/lpnamer/helper/ProblemGenerationLogger.h" #include "ProblemGenerationOptions.h" #include "antares-xpansion/multisolver_interface/SolverAbstract.h" +// clang-format on class ProblemGeneration { public: @@ -49,11 +51,13 @@ class ProblemGeneration { const std::vector& mpsList, std::filesystem::path& lpDir_, std::shared_ptr reader, const Antares::Solver::LpsFromAntares& lps); - virtual void set_solver(std::filesystem::path study_dir, ProblemGenerationLog::ProblemGenerationLogger* logger); + virtual void set_solver( + std::filesystem::path study_dir, + ProblemGenerationLog::ProblemGenerationLogger* logger); Antares::Solver::LpsFromAntares lps_; std::optional mode_; - virtual std::filesystem::path performAntaresSimulation(); + virtual void performAntaresSimulation(const std::filesystem::path& output); std::filesystem::path simulation_dir_; std::string solver_name_; }; diff --git a/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp b/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp index 61d5534d8..713c08a43 100644 --- a/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp @@ -50,14 +50,10 @@ class ProblemGenerationSpyAndMock : public ProblemGeneration { } private: - std::filesystem::path performAntaresSimulation() override { - return options_.StudyPath() / "simulation"; - } + void performAntaresSimulation(const std::filesystem::path&) override {} void set_solver( std::filesystem::path study_dir, - ProblemGenerationLog::ProblemGenerationLogger* logger) override { - - } + ProblemGenerationLog::ProblemGenerationLogger* logger) override {} public: std::filesystem::path xpansion_output_dir_; From efa9e573184c4b00cbfe979392a3d5123cbffdf1 Mon Sep 17 00:00:00 2001 From: Florian OMNES Date: Thu, 2 Jan 2025 10:29:42 +0100 Subject: [PATCH 17/19] Use conversion function instead of lowerCase --- src/cpp/lpnamer/main/ProblemGeneration.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cpp/lpnamer/main/ProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp index d19494d01..6f1d75623 100644 --- a/src/cpp/lpnamer/main/ProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/ProblemGeneration.cpp @@ -61,17 +61,17 @@ ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options) } } -static std::string lowerCase(std::string data) { - std::transform(data.begin(), data.end(), data.begin(), - [](unsigned char c) { return std::tolower(c); }); - return data; +static std::string solverXpansionToSimulator(const std::string& in) { + if (in == "Xpress") return "xpress"; + if (in == "Cbc") return "coin"; + throw std::invalid_argument("Invalid solver"); } void ProblemGeneration::performAntaresSimulation( const std::filesystem::path& output) { Antares::Solver::Optimization::OptimizationOptions optOptions; - optOptions.ortoolsSolver = lowerCase(solver_name_); + optOptions.ortoolsSolver = solverXpansionToSimulator(solver_name_); auto results = Antares::API::PerformSimulation(options_.StudyPath(), output, optOptions); // Add parallel From 26f3828718b277f3257662019133bcc599e6f67c Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 3 Jan 2025 10:49:22 +0100 Subject: [PATCH 18/19] update rc-9 --- antares-version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/antares-version.json b/antares-version.json index 616c1bc7b..5a57da495 100644 --- a/antares-version.json +++ b/antares-version.json @@ -1,5 +1,5 @@ { - "antares_version": "9.2.0-rc7", + "antares_version": "9.2.0-rc9", "antares_version_executable": "9.2", "antares_xpansion_version": "1.4.0", "minizip_ng_version": "3.0.6", From a9535bb80cd7ea23c226ee9155cab1ee8bcb9631 Mon Sep 17 00:00:00 2001 From: Thomas Bittar Date: Fri, 3 Jan 2025 18:09:57 +0100 Subject: [PATCH 19/19] Implement case-insensitive handling for Xpansion solver names to be passed to Simulator --- src/cpp/lpnamer/main/ProblemGeneration.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/cpp/lpnamer/main/ProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp index 6f1d75623..ee7024def 100644 --- a/src/cpp/lpnamer/main/ProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/ProblemGeneration.cpp @@ -61,9 +61,19 @@ ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options) } } +static std::string lowerCase(std::string data) { + std::transform(data.begin(), data.end(), data.begin(), + [](unsigned char c) { return std::tolower(c); }); + return data; +} + static std::string solverXpansionToSimulator(const std::string& in) { - if (in == "Xpress") return "xpress"; - if (in == "Cbc") return "coin"; + // in could be Cbc or CBC depending on whether it is defined or not in the + // settings file + // Use lowerCase in any case to be robust to these subtleties + std::string lower_case_in = lowerCase(in); + if (lower_case_in == "xpress") return "xpress"; + if (lower_case_in == "cbc" || lower_case_in == "coin") return "coin"; throw std::invalid_argument("Invalid solver"); }