From c158febb569d1d1e23762b141731b52fe03003c8 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Mon, 9 Oct 2023 09:23:37 +0200 Subject: [PATCH 01/13] Create lp directories in c++ --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 28 +++++++++++++++++++ .../problem_generator_driver.py | 18 ++++-------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index c6cbf682f..006be4e4b 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -59,6 +59,8 @@ struct Version { int major; int minor; }; +static const std::string LP_DIRNAME = "lp"; + std::shared_ptr InstantiateZipReader( const std::filesystem::path& antares_archive_path); void ProcessWeights( @@ -128,6 +130,27 @@ std::vector> getXpansionProblems( problem_names); return adapter->provideProblems(solver_name, solver_log_manager); } +bool CreateDirectories( + const std::filesystem::path& xpansion_output_path, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) { + if (!std::filesystem::exists(xpansion_output_path)) { + (*logger)(LogUtils::LOGLEVEL::ERR) + << "Output directory " << xpansion_output_path << " does not exist" + << std::endl; + return false; + } + auto xpansion_output_dir = + xpansion_output_path.parent_path() / + (xpansion_output_path.stem().string() + "-Xpansion"); + if (std::filesystem::exists(xpansion_output_dir)) { + std::filesystem::remove_all(xpansion_output_dir); + } + std::filesystem::create_directories(xpansion_output_dir); + std::filesystem::create_directories(xpansion_output_dir / LP_DIRNAME); + + return true; +} + void RunProblemGeneration( const std::filesystem::path& xpansion_output_dir, const std::string& master_formulation, @@ -138,6 +161,11 @@ void RunProblemGeneration( const std::filesystem::path& weights_file, bool unnamed_problems) { (*logger)(LogUtils::LOGLEVEL::INFO) << "Launching Problem Generation" << std::endl; + if (!CreateDirectories( + xpansion_output_dir, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer())) { + return; + } validateMasterFormulation(master_formulation, logger); std::string solver_name = "CBC"; // TODO Use solver selected by user diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index 776f8628d..65b633209 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -76,19 +76,11 @@ def launch(self, output_path: Path, is_relaxed: bool): self._lp_step() def set_output_path(self, output_path): - - if output_path.exists(): - self._output_path = output_path - self.xpansion_output_dir = output_path.parent / \ - (output_path.stem+"-Xpansion") - if self.xpansion_output_dir.exists(): - shutil.rmtree(self.xpansion_output_dir) - os.makedirs(self.xpansion_output_dir) - self._lp_path = os.path.normpath( - os.path.join(self.xpansion_output_dir, 'lp')) - else: - raise ProblemGeneratorDriver.OutputPathError( - f"{output_path} not found") + self._output_path = output_path + self.xpansion_output_dir = output_path.parent / \ + (output_path.stem + "-Xpansion") + self._lp_path = os.path.normpath( + os.path.join(self.xpansion_output_dir, 'lp')) def get_output_path(self): return self._output_path From 6cd005e1a201b7f3aa57451c52b77d3c2ede1842 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 11 Oct 2023 09:42:17 +0200 Subject: [PATCH 02/13] Add test in CPP fo LP dir --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 109 +++++------------- tests/cpp/lp_namer/CMakeLists.txt | 3 +- tests/cpp/lp_namer/ProblemGenerationTest.cpp | 27 +++++ 3 files changed, 56 insertions(+), 83 deletions(-) create mode 100644 tests/cpp/lp_namer/ProblemGenerationTest.cpp diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 006be4e4b..8a880860d 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -1,13 +1,11 @@ #include "RunProblemGeneration.h" -#include #include #include #include "ActiveLinks.h" #include "AdditionalConstraints.h" -#include "Clock.h" #include "GeneralDataReader.h" #include "LauncherHelpers.h" #include "LinkProblemsGenerator.h" @@ -17,14 +15,12 @@ #include "MasterGeneration.h" #include "MasterProblemBuilder.h" #include "MpsTxtWriter.h" -#include "ProblemGenerationExeOptions.h" #include "ProblemVariablesFromProblemAdapter.h" #include "ProblemVariablesZipAdapter.h" #include "StringManip.h" #include "Timer.h" #include "WeightsFileReader.h" #include "WeightsFileWriter.h" -#include "XpansionProblemsFromAntaresProvider.h" #include "ZipProblemsProviderAdapter.h" #include "config.h" @@ -132,7 +128,7 @@ std::vector> getXpansionProblems( } bool CreateDirectories( const std::filesystem::path& xpansion_output_path, - ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) { + ProblemGenerationLog::ProblemGenerationLogger* logger) { if (!std::filesystem::exists(xpansion_output_path)) { (*logger)(LogUtils::LOGLEVEL::ERR) << "Output directory " << xpansion_output_path << " does not exist" @@ -162,8 +158,7 @@ void RunProblemGeneration( (*logger)(LogUtils::LOGLEVEL::INFO) << "Launching Problem Generation" << std::endl; if (!CreateDirectories( - xpansion_output_dir, - ProblemGenerationLog::ProblemGenerationLoggerSharedPointer())) { + xpansion_output_dir, logger.get())) { return; } validateMasterFormulation(master_formulation, logger); @@ -197,14 +192,10 @@ void RunProblemGeneration( auto files_mapper = FilesMapper(antares_archive_path); auto mpsList = files_mapper.MpsAndVariablesFilesVect(); - bool use_zip_implementation = true; - bool use_file_implementation = false; - auto solver_log_manager = SolverLogManager(log_file_path); Couplings couplings; LinkProblemsGenerator linkProblemsGenerator( lpDir_, links, solver_name, logger, solver_log_manager, rename_problems); - if (use_zip_implementation) { std::shared_ptr reader = InstantiateZipReader(antares_archive_path); @@ -213,79 +204,33 @@ void RunProblemGeneration( getXpansionProblems(solver_log_manager, solver_name, mpsList, lpDir_, reader); - std::vector, ProblemData>> - problems_and_data; - for (int i = 0; i < xpansion_problems.size(); ++i) { - xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps; - problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i)); - } - auto mps_file_writer = std::make_shared(lpDir_); - std::for_each( - std::execution::par, problems_and_data.begin(), problems_and_data.end(), - [&](const auto& problem_and_data) { - const auto& [problem, data] = problem_and_data; - std::shared_ptr variables_provider; - if (rename_problems) { - variables_provider = std::make_shared( - reader, data, links, logger); - } else { - variables_provider = - std::make_shared( - problem, links, logger); - } - linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(), - variables_provider.get(), mps_file_writer.get()); - }); - - reader->Close(); - reader->Delete(); - } else if (use_file_implementation) { - /* Main stuff */ - auto mps_file_writer = std::make_shared(lpDir_); - linkProblemsGenerator.treatloop(xpansion_output_dir, couplings, mpsList, + std::vector, ProblemData>> + problems_and_data; + for (int i = 0; i < xpansion_problems.size(); ++i) { + xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps; + problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i)); + } + auto mps_file_writer = std::make_shared(lpDir_); + std::for_each( + std::execution::par, problems_and_data.begin(), problems_and_data.end(), + [&](const auto& problem_and_data) { + const auto& [problem, data] = problem_and_data; + std::shared_ptr variables_provider; + if (rename_problems) { + variables_provider = std::make_shared( + reader, data, links, logger); + } else { + variables_provider = + std::make_shared( + problem, links, logger); + } + linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(), + variables_provider.get(), mps_file_writer.get()); - } else { - std::filesystem::path path = - xpansion_output_dir.parent_path().parent_path() / - "fichierDeSerialisation"; - std::ifstream ifs(xpansion_output_dir.parent_path().parent_path() / - "fichierDeSerialisation"); - boost::archive::text_iarchive ia(ifs); + }); - LpsFromAntares lps; - ia >> lps; - lps._constant->Mdeb.push_back(lps._constant->NombreDeCoefficients); - - XpansionProblemsFromAntaresProvider adapter(lps); - auto xpansion_problems = - adapter.provideProblems(solver_name, solver_log_manager); - std::vector, ProblemData>> - problems_and_data; - for (int i = 0; i < xpansion_problems.size(); ++i) { - xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps; - problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i)); - } - - auto reader = InstantiateZipReader(antares_archive_path); - auto mps_file_writer = std::make_shared(lpDir_); - - std::for_each( - std::execution::par, problems_and_data.begin(), problems_and_data.end(), - [&](const auto& problem_and_data) { - const auto& [problem, data] = problem_and_data; - std::shared_ptr variables_provider; - if (rename_problems) { - variables_provider = std::make_shared( - reader, data, links, logger); - } else { - variables_provider = - std::make_shared( - problem, links, logger); - } - linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(), - variables_provider.get(), mps_file_writer.get()); - }); - } + reader->Close(); + reader->Delete(); MasterGeneration master_generation( xpansion_output_dir, links, additionalConstraints, couplings, diff --git a/tests/cpp/lp_namer/CMakeLists.txt b/tests/cpp/lp_namer/CMakeLists.txt index 845ad7b50..619f4f46b 100644 --- a/tests/cpp/lp_namer/CMakeLists.txt +++ b/tests/cpp/lp_namer/CMakeLists.txt @@ -36,7 +36,8 @@ add_executable (lp_namer_tests WeightsFileReaderTest.cpp LpFilesExtractorTest.cpp MpsTxtWriterTest - ) + ProblemGenerationTest.cpp +) target_link_libraries (lp_namer_tests PRIVATE GTest::Main diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp new file mode 100644 index 000000000..049a5f967 --- /dev/null +++ b/tests/cpp/lp_namer/ProblemGenerationTest.cpp @@ -0,0 +1,27 @@ +// +// Created by marechaljas on 10/10/23. +// + +#include + +#include "ArchiveIO.h" +#include "LoggerBuilder.h" +#include "RunProblemGeneration.h" +#include "gtest/gtest.h" + +TEST(InitializationTest, FoldersAreCreated) { + auto workingDirectory = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + auto simulationDirectory = workingDirectory / "output" / "simulation"; + auto logger = emptyLogger(); + std::filesystem::create_directories(simulationDirectory); + EXPECT_THROW( + RunProblemGeneration(simulationDirectory, "integer", "", "", logger, + simulationDirectory / "logs.txt", "", false), + ArchiveIOGeneralException); + auto xpansionDirectory = + (simulationDirectory.parent_path() / + (simulationDirectory.filename().string() + "-Xpansion")) / + "lp"; + EXPECT_TRUE(std::filesystem::exists(xpansionDirectory)); +} \ No newline at end of file From 8952166def5ec42ddfbc056efa4bd542830c19fd Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 11 Oct 2023 10:24:07 +0200 Subject: [PATCH 03/13] Don't create lp dir in driver --- src/python/antares_xpansion/problem_generator_driver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index 65b633209..273f5717b 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -70,8 +70,6 @@ def launch(self, output_path: Path, is_relaxed: bool): self.logger.info("Problem Generation") self.output_path = output_path - self.create_lp_dir() - self.is_relaxed = is_relaxed self._lp_step() From 1b235cfb1acd740ae575c52bb1e3252c8627eecc Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 11 Oct 2023 10:56:47 +0200 Subject: [PATCH 04/13] Clean LP directory at the beginning of problem generation --- .../problem_generator_driver.py | 1 - tests/cpp/lp_namer/ProblemGenerationTest.cpp | 27 ++++++- tests/python/test_problem_generator_driver.py | 74 +------------------ 3 files changed, 27 insertions(+), 75 deletions(-) diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index 273f5717b..805425c6b 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -67,7 +67,6 @@ def launch(self, output_path: Path, is_relaxed: bool): problem generation step : getnames + lp_namer """ self.clear_old_log() - self.logger.info("Problem Generation") self.output_path = output_path self.is_relaxed = is_relaxed diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp index 049a5f967..d5558b40a 100644 --- a/tests/cpp/lp_namer/ProblemGenerationTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationTest.cpp @@ -19,9 +19,30 @@ TEST(InitializationTest, FoldersAreCreated) { RunProblemGeneration(simulationDirectory, "integer", "", "", logger, simulationDirectory / "logs.txt", "", false), ArchiveIOGeneralException); - auto xpansionDirectory = - (simulationDirectory.parent_path() / + auto lp = (simulationDirectory.parent_path() / (simulationDirectory.filename().string() + "-Xpansion")) / "lp"; - EXPECT_TRUE(std::filesystem::exists(xpansionDirectory)); + EXPECT_TRUE(std::filesystem::exists(lp)); +} + +TEST(InitializationTest, FoldersAreEmpty) { + auto workingDirectory = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + auto simulationDirectory = workingDirectory / "output" / "simulation"; + auto logger = emptyLogger(); + std::filesystem::create_directories(simulationDirectory); + + auto xpansionDirectory = + (simulationDirectory.parent_path() / + (simulationDirectory.filename().string() + "-Xpansion")); + std::filesystem::create_directories(xpansionDirectory); + std::ofstream emptyfile(xpansionDirectory / "garbage.txt"); + + EXPECT_TRUE(std::filesystem::exists(xpansionDirectory / "garbage.txt")); + EXPECT_THROW( + RunProblemGeneration(simulationDirectory, "integer", "", "", logger, + simulationDirectory / "logs.txt", "", false), + ArchiveIOGeneralException); + + EXPECT_FALSE(std::filesystem::exists(simulationDirectory / "garbage.txt")); } \ No newline at end of file diff --git a/tests/python/test_problem_generator_driver.py b/tests/python/test_problem_generator_driver.py index fcfb58294..40d0b2479 100644 --- a/tests/python/test_problem_generator_driver.py +++ b/tests/python/test_problem_generator_driver.py @@ -1,16 +1,9 @@ -from typing import Text -import pytest -import os -from pathlib import Path -from datetime import date, datetime -from unittest.mock import ANY, mock_open, patch import shutil +from datetime import date, datetime +from pathlib import Path -from file_creation import _create_weight_file +import pytest from antares_xpansion.problem_generator_driver import ProblemGeneratorData, ProblemGeneratorDriver -from antares_xpansion.antares_driver import AntaresDriver -from antares_xpansion.xpansion_study_reader import XpansionStudyReader -from antares_xpansion.general_data_reader import GeneralDataIniReader SUBPROCESS_RUN = "antares_xpansion.problem_generator_driver.subprocess.run" zipfile_ZipFile = "antares_xpansion.problem_generator_driver.zipfile.ZipFile" @@ -65,67 +58,6 @@ def test_lp_namer_exe_does_not_exit(self, tmp_path): def xpansion_output(self, tmp_path): return tmp_path.parent / (tmp_path.name + '-Xpansion') - def test_clear_old_log(self, tmp_path): - - lp_namer_file = tmp_path / self.lp_exe - lp_namer_file.write_text("") - pblm_gen_data = ProblemGeneratorData(keep_mps=False, - additional_constraints="", - user_weights_file_path=Path(""), - weight_file_name_for_lp="", - lp_namer_exe_path=lp_namer_file, - active_years=[]) - self._create_empty_area_file(tmp_path) - self._create_empty_interco_file(tmp_path) - output_zipped = get_zipped_output(tmp_path) - log_file_name = self.lp_exe + ".log" - xpansion_dir = self.xpansion_output(tmp_path) - log_file = xpansion_dir / log_file_name - xpansion_dir.mkdir() - log_file.write_text("bla bla") - assert log_file.exists() - - pblm_gen = ProblemGeneratorDriver(pblm_gen_data) - with patch(SUBPROCESS_RUN, autospec=True): - with pytest.raises(ProblemGeneratorDriver.LPNamerExecutionError): - pblm_gen.launch(output_zipped, False) - - assert not log_file.exists() - - def test_clean_lp_dir_before_run(self, tmp_path): - - lp_namer_file = tmp_path / self.lp_exe - lp_namer_file.write_text("") - pblm_gen_data = ProblemGeneratorData(keep_mps=False, - additional_constraints="", - user_weights_file_path=Path(""), - weight_file_name_for_lp="", - lp_namer_exe_path=lp_namer_file, - active_years=[]) - self._create_empty_area_file(tmp_path) - self._create_empty_interco_file(tmp_path) - output_zipped = get_zipped_output(tmp_path) - log_file_name = self.lp_exe + ".log" - xpansion_dir = self.xpansion_output(tmp_path) - xpansion_dir.mkdir() - log_file = xpansion_dir / log_file_name - log_file.write_text("bla bla") - - lp_dir = xpansion_dir / "lp" - lp_dir.mkdir() - lp_dir_sub_file_1 = lp_dir / "file1" - lp_dir_sub_file_1.write_text("") - - assert lp_dir.exists() - assert lp_dir_sub_file_1.exists() - problem_generator_driver = ProblemGeneratorDriver(pblm_gen_data) - with patch(SUBPROCESS_RUN, autospec=True) as run_function: - run_function.return_value.returncode = 0 - problem_generator_driver.launch(output_zipped, False) - - assert lp_dir.exists() - assert not lp_dir_sub_file_1.exists() - def _get_expected_mps_txt(self, tmp_path): weeks = [1, 2, 3] From b006c924fcff56eff8f9032df74985630db5aa60 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 11 Oct 2023 11:26:06 +0200 Subject: [PATCH 05/13] Expect error if output path does not exist --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 3 ++- tests/cpp/lp_namer/ProblemGenerationTest.cpp | 16 ++++++++++++++++ tests/python/test_problem_generator_driver.py | 6 ------ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 8a880860d..c54e451c8 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -159,7 +159,8 @@ void RunProblemGeneration( << "Launching Problem Generation" << std::endl; if (!CreateDirectories( xpansion_output_dir, logger.get())) { - return; + throw LogUtils::XpansionError( + "Error creating LP directory", LOGLOCATION); } validateMasterFormulation(master_formulation, logger); std::string solver_name = "CBC"; // TODO Use solver selected by user diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp index d5558b40a..18f16faf4 100644 --- a/tests/cpp/lp_namer/ProblemGenerationTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationTest.cpp @@ -45,4 +45,20 @@ TEST(InitializationTest, FoldersAreEmpty) { ArchiveIOGeneralException); EXPECT_FALSE(std::filesystem::exists(simulationDirectory / "garbage.txt")); +} + +TEST(InitializationTest, ErrorIfSimulationPathDoesNotExists) { + auto workingDirectory = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + auto simulationDirectory = workingDirectory / "output" / "simulation"; + auto logger = emptyLogger(); + + EXPECT_THROW( + RunProblemGeneration(simulationDirectory, "integer", "", "", logger, + simulationDirectory / "logs.txt", "", false), + LogUtils::XpansionError); + auto lp = (simulationDirectory.parent_path() / + (simulationDirectory.filename().string() + "-Xpansion")) / + "lp"; + EXPECT_FALSE(std::filesystem::exists(lp)); } \ No newline at end of file diff --git a/tests/python/test_problem_generator_driver.py b/tests/python/test_problem_generator_driver.py index 40d0b2479..21ae640e4 100644 --- a/tests/python/test_problem_generator_driver.py +++ b/tests/python/test_problem_generator_driver.py @@ -39,12 +39,6 @@ def test_problem_generator_data(self): assert problem_generator_driver.user_weights_file_path == self.empty_pblm_gen_data.user_weights_file_path assert problem_generator_driver.lp_namer_exe_path == self.empty_pblm_gen_data.lp_namer_exe_path - def test_output_path(self, tmp_path): - problem_generator_driver = ProblemGeneratorDriver( - self.empty_pblm_gen_data) - with pytest.raises(ProblemGeneratorDriver.OutputPathError): - problem_generator_driver.launch(tmp_path / "i_don_t_exist", False) - def test_lp_namer_exe_does_not_exit(self, tmp_path): self._create_empty_area_file(tmp_path) From ce5f5be1a6719f60fbf093a22d3b964b06e6688e Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 12 Oct 2023 10:23:47 +0200 Subject: [PATCH 06/13] Don't delete previous directory (useless) --- .../antares_xpansion/full_run_driver.py | 13 ++++----- .../problem_generator_driver.py | 27 ++++++++++--------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/python/antares_xpansion/full_run_driver.py b/src/python/antares_xpansion/full_run_driver.py index 27af0ff84..d10ef46bb 100644 --- a/src/python/antares_xpansion/full_run_driver.py +++ b/src/python/antares_xpansion/full_run_driver.py @@ -1,13 +1,13 @@ - -from typing import List +import os import subprocess import sys +from pathlib import Path +from typing import List + from antares_xpansion.benders_driver import BendersDriver -from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver from antares_xpansion.logger import step_logger +from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver from antares_xpansion.study_output_cleaner import StudyOutputCleaner -import os -from pathlib import Path class FullRunDriver: @@ -29,9 +29,6 @@ def prepare_drivers(self, output_path: Path, """ problem generation step : getnames + lp_namer """ - # Pb Gen pre-step - self.problem_generation_driver.clear_old_log() - self.problem_generation_driver.is_relaxed = problem_generation_is_relaxed self.keep_mps = benders_keep_mps diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index 805425c6b..09af5e0fd 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -3,7 +3,6 @@ """ import os -import shutil import subprocess import sys from dataclasses import dataclass @@ -66,26 +65,30 @@ def launch(self, output_path: Path, is_relaxed: bool): """ problem generation step : getnames + lp_namer """ - self.clear_old_log() + self.logger.info("Problem Generation") self.output_path = output_path + self.create_lp_dir() + self.is_relaxed = is_relaxed self._lp_step() def set_output_path(self, output_path): - self._output_path = output_path - self.xpansion_output_dir = output_path.parent / \ - (output_path.stem + "-Xpansion") - self._lp_path = os.path.normpath( - os.path.join(self.xpansion_output_dir, 'lp')) + + if output_path.exists(): + self._output_path = output_path + self.xpansion_output_dir = output_path.parent / \ + (output_path.stem + "-Xpansion") + os.makedirs(self.xpansion_output_dir) + self._lp_path = os.path.normpath( + os.path.join(self.xpansion_output_dir, 'lp')) + else: + raise ProblemGeneratorDriver.OutputPathError( + f"{output_path} not found") def get_output_path(self): return self._output_path - def clear_old_log(self): - if (os.path.isfile(str(self.lp_namer_exe_path) + '.log')): - os.remove(str(self.lp_namer_exe_path) + '.log') - def _lp_step(self): """ copies area and interco files and launches the lp_namer @@ -101,8 +104,6 @@ def _lp_step(self): "ERROR: exited lpnamer with status %d" % returned_l.returncode) def create_lp_dir(self): - if os.path.isdir(self._lp_path): - shutil.rmtree(self._lp_path) os.makedirs(self._lp_path) def lp_namer_options(self): From 92919dbf155b335627a3d8c36f4057bbc8b4c69e Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 12 Oct 2023 10:52:39 +0200 Subject: [PATCH 07/13] Don't delete previous directory (useless) --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index c54e451c8..00ebc509a 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -138,9 +138,6 @@ bool CreateDirectories( auto xpansion_output_dir = xpansion_output_path.parent_path() / (xpansion_output_path.stem().string() + "-Xpansion"); - if (std::filesystem::exists(xpansion_output_dir)) { - std::filesystem::remove_all(xpansion_output_dir); - } std::filesystem::create_directories(xpansion_output_dir); std::filesystem::create_directories(xpansion_output_dir / LP_DIRNAME); From 3dd85fb6451f22b3c9b7dda75ee059065fc95266 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 12 Oct 2023 14:50:43 +0200 Subject: [PATCH 08/13] Create LP dir if not existing --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 25 ++++++------------- tests/cpp/lp_namer/ProblemGenerationTest.cpp | 16 ------------ 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 00ebc509a..cc57681e8 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -126,22 +126,15 @@ std::vector> getXpansionProblems( problem_names); return adapter->provideProblems(solver_name, solver_log_manager); } -bool CreateDirectories( - const std::filesystem::path& xpansion_output_path, +void CreateDirectories(const std::filesystem::path& xpansion_output_path, ProblemGenerationLog::ProblemGenerationLogger* logger) { if (!std::filesystem::exists(xpansion_output_path)) { - (*logger)(LogUtils::LOGLEVEL::ERR) - << "Output directory " << xpansion_output_path << " does not exist" - << std::endl; - return false; + std::filesystem::create_directories(xpansion_output_path); + } + auto lp_path = xpansion_output_path / LP_DIRNAME; + if (!std::filesystem::exists(lp_path)) { + std::filesystem::create_directories(lp_path); } - auto xpansion_output_dir = - xpansion_output_path.parent_path() / - (xpansion_output_path.stem().string() + "-Xpansion"); - std::filesystem::create_directories(xpansion_output_dir); - std::filesystem::create_directories(xpansion_output_dir / LP_DIRNAME); - - return true; } void RunProblemGeneration( @@ -154,11 +147,7 @@ void RunProblemGeneration( const std::filesystem::path& weights_file, bool unnamed_problems) { (*logger)(LogUtils::LOGLEVEL::INFO) << "Launching Problem Generation" << std::endl; - if (!CreateDirectories( - xpansion_output_dir, logger.get())) { - throw LogUtils::XpansionError( - "Error creating LP directory", LOGLOCATION); - } + CreateDirectories(xpansion_output_dir, logger.get()); validateMasterFormulation(master_formulation, logger); std::string solver_name = "CBC"; // TODO Use solver selected by user diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp index 18f16faf4..d5558b40a 100644 --- a/tests/cpp/lp_namer/ProblemGenerationTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationTest.cpp @@ -45,20 +45,4 @@ TEST(InitializationTest, FoldersAreEmpty) { ArchiveIOGeneralException); EXPECT_FALSE(std::filesystem::exists(simulationDirectory / "garbage.txt")); -} - -TEST(InitializationTest, ErrorIfSimulationPathDoesNotExists) { - auto workingDirectory = - std::filesystem::temp_directory_path() / std::tmpnam(nullptr); - auto simulationDirectory = workingDirectory / "output" / "simulation"; - auto logger = emptyLogger(); - - EXPECT_THROW( - RunProblemGeneration(simulationDirectory, "integer", "", "", logger, - simulationDirectory / "logs.txt", "", false), - LogUtils::XpansionError); - auto lp = (simulationDirectory.parent_path() / - (simulationDirectory.filename().string() + "-Xpansion")) / - "lp"; - EXPECT_FALSE(std::filesystem::exists(lp)); } \ No newline at end of file From da61efb2ea970a8b77a14384adb6474b4ffe5506 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 19 Oct 2023 15:46:04 +0200 Subject: [PATCH 09/13] Fix unit tests --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 8 ++++---- tests/cpp/lp_namer/ProblemGenerationTest.cpp | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index cc57681e8..8ddf6a4d5 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -126,12 +126,12 @@ std::vector> getXpansionProblems( problem_names); return adapter->provideProblems(solver_name, solver_log_manager); } -void CreateDirectories(const std::filesystem::path& xpansion_output_path, +void CreateDirectories(const std::filesystem::path& output_path, ProblemGenerationLog::ProblemGenerationLogger* logger) { - if (!std::filesystem::exists(xpansion_output_path)) { - std::filesystem::create_directories(xpansion_output_path); + if (!std::filesystem::exists(output_path)) { + std::filesystem::create_directories(output_path); } - auto lp_path = xpansion_output_path / LP_DIRNAME; + auto lp_path = output_path / LP_DIRNAME; if (!std::filesystem::exists(lp_path)) { std::filesystem::create_directories(lp_path); } diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp index d5558b40a..1fa3a3f36 100644 --- a/tests/cpp/lp_namer/ProblemGenerationTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationTest.cpp @@ -12,16 +12,15 @@ TEST(InitializationTest, FoldersAreCreated) { auto workingDirectory = std::filesystem::temp_directory_path() / std::tmpnam(nullptr); - auto simulationDirectory = workingDirectory / "output" / "simulation"; + auto simulationDirectory = + workingDirectory / "output" / "simulation-Xpansion"; auto logger = emptyLogger(); std::filesystem::create_directories(simulationDirectory); EXPECT_THROW( RunProblemGeneration(simulationDirectory, "integer", "", "", logger, simulationDirectory / "logs.txt", "", false), ArchiveIOGeneralException); - auto lp = (simulationDirectory.parent_path() / - (simulationDirectory.filename().string() + "-Xpansion")) / - "lp"; + auto lp = (simulationDirectory) / "lp"; EXPECT_TRUE(std::filesystem::exists(lp)); } From 519acdc76e06950601e07a68ecc9cd44358aad45 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 25 Oct 2023 16:31:23 +0200 Subject: [PATCH 10/13] Create lp dir before trying to create logfile --- src/cpp/exe/lpnamer/main.cpp | 14 +++++++++++++- src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp | 5 +++-- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 12 ------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/cpp/exe/lpnamer/main.cpp b/src/cpp/exe/lpnamer/main.cpp index 1f7fffd59..dc940d881 100644 --- a/src/cpp/exe/lpnamer/main.cpp +++ b/src/cpp/exe/lpnamer/main.cpp @@ -5,6 +5,18 @@ #include "ProblemGenerationLogger.h" #include "RunProblemGeneration.h" +static const std::string LP_DIRNAME = "lp"; + +void CreateDirectories(const std::filesystem::path& output_path) { + if (!std::filesystem::exists(output_path)) { + std::filesystem::create_directories(output_path); + } + auto lp_path = output_path / LP_DIRNAME; + if (!std::filesystem::exists(lp_path)) { + std::filesystem::create_directories(lp_path); + } +} + int main(int argc, char** argv) { try { auto options_parser = ProblemGenerationExeOptions(); @@ -16,9 +28,9 @@ int main(int argc, char** argv) { const auto log_file_path = xpansion_output_dir / "lp" / "ProblemGenerationLog.txt"; + CreateDirectories(xpansion_output_dir); auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout, "Problem Generation"); - auto& loggerRef = (*logger); auto master_formulation = options_parser.MasterFormulation(); auto additionalConstraintFilename_l = diff --git a/src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp b/src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp index 03c53f279..37ce42b37 100644 --- a/src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp +++ b/src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp @@ -7,10 +7,11 @@ ProblemGenerationFileLogger::ProblemGenerationFileLogger( const std::filesystem::path& logFilePath) : logFilePath_(logFilePath) { SetType(LogUtils::LOGGERTYPE::FILE); - logFile_.open(logFilePath_, std::ofstream::out | std::ofstream::app); + auto logPath = std::filesystem::absolute(logFilePath_); + logFile_.open(logPath, std::ofstream::out | std::ofstream::app); if (logFile_.fail()) { std::cerr << "ProblemGenerationFileLogger: Invalid file (" - << logFilePath_.string() << ") passed as parameter" << std::endl; + << logPath.string() << ") passed as parameter" << std::endl; } } void ProblemGenerationFileLogger::DisplayMessage(const std::string& message) { diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 8ddf6a4d5..527075c13 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -55,7 +55,6 @@ struct Version { int major; int minor; }; -static const std::string LP_DIRNAME = "lp"; std::shared_ptr InstantiateZipReader( const std::filesystem::path& antares_archive_path); @@ -126,16 +125,6 @@ std::vector> getXpansionProblems( problem_names); return adapter->provideProblems(solver_name, solver_log_manager); } -void CreateDirectories(const std::filesystem::path& output_path, - ProblemGenerationLog::ProblemGenerationLogger* logger) { - if (!std::filesystem::exists(output_path)) { - std::filesystem::create_directories(output_path); - } - auto lp_path = output_path / LP_DIRNAME; - if (!std::filesystem::exists(lp_path)) { - std::filesystem::create_directories(lp_path); - } -} void RunProblemGeneration( const std::filesystem::path& xpansion_output_dir, @@ -147,7 +136,6 @@ void RunProblemGeneration( const std::filesystem::path& weights_file, bool unnamed_problems) { (*logger)(LogUtils::LOGLEVEL::INFO) << "Launching Problem Generation" << std::endl; - CreateDirectories(xpansion_output_dir, logger.get()); validateMasterFormulation(master_formulation, logger); std::string solver_name = "CBC"; // TODO Use solver selected by user From 24390c8de5de4149244371041710f1b21f6d62e5 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 26 Oct 2023 11:40:13 +0200 Subject: [PATCH 11/13] Update sequence diagram file manipulation --- .../FileManipulation/diagram.svg | 332 ++++++++++-------- .../FileManipulation/diagram.txt | 2 + 2 files changed, 178 insertions(+), 156 deletions(-) diff --git a/conception/SequenceDiagrams/FileManipulation/diagram.svg b/conception/SequenceDiagrams/FileManipulation/diagram.svg index 13c3c73c9..da1440a50 100644 --- a/conception/SequenceDiagrams/FileManipulation/diagram.svg +++ b/conception/SequenceDiagrams/FileManipulation/diagram.svg @@ -1,7 +1,7 @@ - + - title%20Xpansion%20folder%20and%20file%20management%20(full%20run)%0A%0Aparticipantgroup%20%23lightblue%20**Python**%0Aparticipant%20Launch.py%0Aparticipant%20XpansionDriver%0Aparticipant%20AntaresDriver%0Aparticipant%20StudyOutputCleaner%0Aparticipant%20config_loader%0Aparticipant%20problem_generator_driver%0Aparticipant%20full_run_driver%0Aend%0A%0ALaunch.py-%3EXpansionDriver%3Alaunch()%0AXpansionDriver-%3EXpansionDriver%3Alaunch_antares_step()%0Aactivate%20XpansionDriver%0AXpansionDriver-%3EXpansionDriver%3A_backup_general_data_ini%0AXpansionDriver-%3EXpansionDriver%3A_update_general_data_ini%0AXpansionDriver-%3EXpansionDriver%3Aend%20launch_antares_step%0Adeactivate%20XpansionDriver%0AXpansionDriver-%3EAntaresDriver%3ALaunch()%0Aactivate%20AntaresDriver%0AAntaresDriver-%3EAntaresDriver%3A_clear_old_log()%0Anote%20over%20AntaresDriver%3A%20remove%20antares-X.Y-solver.log%0AAntaresDriver-%3EAntaresDriver%3A_set_simulation_name%0Anote%20over%20AntaresDriver%3A%20Get%20last%20modified%20directory%5CnWARNING%20Wrong%20whith%20zipped%20results%0Acreate%20StudyOutputCleaner%0AAntaresDriver-%3EStudyOutputCleaner%3Aclean_antares_step%0AStudyOutputCleaner-%3EStudyOutputCleaner%3ARemove%20Criterion%20files%0Adeactivate%20AntaresDriver%0AAntaresDriver-%3EXpansionDriver%3A%20return%20status%0Aalt%20Antares%20ok%20(return%20false)%0AXpansionDriver-%3EXpansionDriver%3A%20_revert_general_data_ini%0Aelse%20case%20Antares%20NOK%20(return%20true)%0AXpansionDriver-%3EXpansionDriver%3A_backup_general_data_ini_on_error%0AXpansionDriver-%3EXpansionDriver%3A_revert_general_data_ini%0Aelse%20case%20Antares%20Axception%3A%0AXpansionDriver-%3EXpansionDriver%3A_backup_general_data_ini_on_error%0AXpansionDriver-%3EXpansionDriver%3A_revert_general_data_ini%0Aend%0AXpansionDriver-%3Eproblem_generator_driver%3Aset_output_path%0AXpansionDriver-%3Eproblem_generator_driver%3Acreate_lp_dir%0Anote%20over%20problem_generator_driver%3A%20output%2Fsimulation-Xpansion%2Flp%0Aproblem_generator_driver-%3Eproblem_generator_driver%3Aos.makedirs(self._lp_path)%0AXpansionDriver-%3Econfig_loader%3Abenders_pre_actions%0Aconfig_loader-%3Econfig_loader%3Asave_launcher_options%0Anote%20over%20config_loader%3ASave%20command%20line%20options%20in%20output%2Fsimulation-Xpansion%2Flp%2Flauncher_options.json%0Aalt%20Not%20Resume%0Aconfig_loader-%3Econfig_loader%3Acreate_expansion_dir%0Anote%20over%20config_loader%3Arm%2Bcreate%20output%2Fsimulation-Xpansion%2Fexpansion%0Aend%0Aconfig_loader-%3Econfig_loader%3A_set_options_for_benders_solver%0Anote%20over%20config_loader%3A%20write%20options%20in%20output%2Fsimulation-Xpansion%2Flp%2Foptions.json%0AXpansionDriver-%3Efull_run_driver%3Alaunch%0Acreate%20full_run%0Afull_run_driver-%3Efull_run%3Arun%0Aactivate%20full_run%0Anote%20over%20full_run%3A%20log_file_path%20%3D%20output%2Fsimulation-Xpansion%2Flp%2FProblemGenerationLog.txt%0Aparticipantgroup%20%23lightgreen%20**ProblemGeneration**%0Aparticipant%20ProblemGeneration%0Aparticipant%20LpFilesExtractor%0Aparticipant%20ZipProblemProviderAdapter%0Aparticipant%20FilesMapper%0Aend%0A%0Acreate%20ProblemGeneration%0Acreate%20LpFilesExtractor%0Acreate%20ZipProblemProviderAdapter%0Acreate%20FilesMapper%0Afull_run-%3EProblemGeneration%3ARunProblemGeneration%0Aactivate%20ProblemGeneration%0AProblemGeneration-%3EProblemGeneration%3AExtractUtilsFiles%0A%0AProblemGeneration-%3ELpFilesExtractor%3AExtractFiles%0AProblemGeneration-%3EFilesMapper%3AFillMap%0Anote%20over%20FilesMapper%3A%20read%20mps%20file%20list%20from%20zip%20%5Cn%20Fill%20Map%3CPair%3CYear%2C%20Week%3E%2C%20Tuple%3CMPSPAth%2C%20VariablePAth%2C%20ConstraintPath%3E%3E%20%0Adestroyafter%20FilesMapper%0A%0Aalt%20use%20zip%20implementation%20%3D%20true%20%7Bconst%7D%0Aloop%20mps%20files%0AProblemGeneration-%3EZipProblemProviderAdapter%3AProvideProblems%0Anote%20over%20ZipProblemProviderAdapter%3AExtract%20problem%20file%20from%20zip%5Cn%20create%20%22problem%22%2Fsolver%0Aend%0Aloop%20Problems%0AProblemGeneration-%3EProblemGeneration%3Atreat%0Anote%20over%20ProblemGeneration%3A%20write%20mps%0Aend%0Aend%0AProblemGeneration-%3EProblemGeneration%3AMasterGeneration()%0Adeactivateafter%20ProblemGeneration%0Adestroy%20LpFilesExtractor%0Adestroy%20ZipProblemProviderAdapter%0Adestroyafter%20ProblemGeneration%0A%0A%0Aparticipantgroup%20%23lightpink%20**Benders**%0Aparticipant%20BendersFactory%0Aparticipant%20Benders%0Aend%20%0Afull_run-%3EBendersFactory%3ARunBenders%0ABendersFactory-%3EBendersFactory%3ARead%20options%20file%0ABendersFactory-%3EBendersFactory%3ASet%20log%20destination%0Anote%20over%20BendersFactory%3A%20bendersLog-rank%3CN%3E.txt%0ABendersFactory-%3EBendersFactory%3AFileAndStdoutLoggerFactory%0Anote%20over%20BendersFactory%3Alog%20filename%3A%20reportbenders.txt%5Cndestination%3A%20%22.%22%20with%20previous%20chdir%20to%20lp%20dir%0ABendersFactory-%3EBendersFactory%3Aset%20writer%0Anote%20over%20BendersFactory%3A%20write%20in%20simulation-Xpansion%2Fexpansion%2Fout.json%0ABendersFactory-%3EBenders%3Alaunch%0A%0A%0Afull_run-%3Efull_run%3AupdateStudy%0A%0A%0Adeactivateafter%20full_run%0Adestroyafter%20full_run%0Aalt%20not%20keep_mps%0Afull_run_driver-%3EStudyOutputCleaner%3Aclean_benders_step%0Anote%20over%20StudyOutputCleaner%3Arm%20simulation-Xpansion%2Flp%2F*.mps%5Cnrm%20simulation-Xpansion%2Flp%2F*.lp%5Cnrm%20simulation-Xpansion%2Flp%2F*.zip%0Aend%0AXpansionDriver-%3EXpansionDriver%3Aclean_step%0Aactivate%20XpansionDriver%0Acreate%20antares_archive_updater_exe%0AXpansionDriver-%3Eantares_archive_updater_exe%3ARun%0Anote%20over%20antares_archive_updater_exe%3A%20add%20sumulation-Xpansion%20in%20simulation.zip%5Cnrm%20simulation-Xpansion%0Aantares_archive_updater_exe-%3Eantares_archive_updater_exe%3ACleanAntaresArchive%0Adestroyafter%20antares_archive_updater_exe%0Anote%20over%20XpansionDriver%3Arm%20output%2Fsimulation-Xpansion%0Adeactivateafter%20XpansionDriver%0A + title%20Xpansion%20folder%20and%20file%20management%20(full%20run)%0A%0Aparticipantgroup%20%23lightblue%20**Python**%0Aparticipant%20Launch.py%0Aparticipant%20XpansionDriver%0Aparticipant%20AntaresDriver%0Aparticipant%20StudyOutputCleaner%0Aparticipant%20config_loader%0Aparticipant%20problem_generator_driver%0Aparticipant%20full_run_driver%0Aend%0A%0ALaunch.py-%3EXpansionDriver%3Alaunch()%0AXpansionDriver-%3EXpansionDriver%3Alaunch_antares_step()%0Aactivate%20XpansionDriver%0AXpansionDriver-%3EXpansionDriver%3A_backup_general_data_ini%0AXpansionDriver-%3EXpansionDriver%3A_update_general_data_ini%0AXpansionDriver-%3EXpansionDriver%3Aend%20launch_antares_step%0Adeactivate%20XpansionDriver%0AXpansionDriver-%3EAntaresDriver%3ALaunch()%0Aactivate%20AntaresDriver%0AAntaresDriver-%3EAntaresDriver%3A_clear_old_log()%0Anote%20over%20AntaresDriver%3A%20remove%20antares-X.Y-solver.log%0AAntaresDriver-%3EAntaresDriver%3A_set_simulation_name%0Anote%20over%20AntaresDriver%3A%20Get%20last%20modified%20directory%5CnWARNING%20Wrong%20whith%20zipped%20results%0Acreate%20StudyOutputCleaner%0AAntaresDriver-%3EStudyOutputCleaner%3Aclean_antares_step%0AStudyOutputCleaner-%3EStudyOutputCleaner%3ARemove%20Criterion%20files%0Adeactivate%20AntaresDriver%0AAntaresDriver-%3EXpansionDriver%3A%20return%20status%0Aalt%20Antares%20ok%20(return%20false)%0AXpansionDriver-%3EXpansionDriver%3A%20_revert_general_data_ini%0Aelse%20case%20Antares%20NOK%20(return%20true)%0AXpansionDriver-%3EXpansionDriver%3A_backup_general_data_ini_on_error%0AXpansionDriver-%3EXpansionDriver%3A_revert_general_data_ini%0Aelse%20case%20Antares%20Axception%3A%0AXpansionDriver-%3EXpansionDriver%3A_backup_general_data_ini_on_error%0AXpansionDriver-%3EXpansionDriver%3A_revert_general_data_ini%0Aend%0AXpansionDriver-%3Eproblem_generator_driver%3Aset_output_path%0AXpansionDriver-%3Eproblem_generator_driver%3Acreate_lp_dir%0Anote%20over%20problem_generator_driver%3A%20output%2Fsimulation-Xpansion%2Flp%0Aproblem_generator_driver-%3Eproblem_generator_driver%3Aos.makedirs(self._lp_path)%0AXpansionDriver-%3Econfig_loader%3Abenders_pre_actions%0Aconfig_loader-%3Econfig_loader%3Asave_launcher_options%0Anote%20over%20config_loader%3ASave%20command%20line%20options%20in%20output%2Fsimulation-Xpansion%2Flp%2Flauncher_options.json%0Aalt%20Not%20Resume%0Aconfig_loader-%3Econfig_loader%3Acreate_expansion_dir%0Anote%20over%20config_loader%3Arm%2Bcreate%20output%2Fsimulation-Xpansion%2Fexpansion%0Aend%0Aconfig_loader-%3Econfig_loader%3A_set_options_for_benders_solver%0Anote%20over%20config_loader%3A%20write%20options%20in%20output%2Fsimulation-Xpansion%2Flp%2Foptions.json%0AXpansionDriver-%3Efull_run_driver%3Alaunch%0Acreate%20full_run%0Afull_run_driver-%3Efull_run%3Arun%0Aactivate%20full_run%0Anote%20over%20full_run%3A%20log_file_path%20%3D%20output%2Fsimulation-Xpansion%2Flp%2FProblemGenerationLog.txt%0Aparticipantgroup%20%23lightgreen%20**ProblemGeneration**%0Aparticipant%20ProblemGeneration%0Aparticipant%20LpFilesExtractor%0Aparticipant%20ZipProblemProviderAdapter%0Aparticipant%20FilesMapper%0Aend%0A%0Acreate%20ProblemGeneration%0Acreate%20LpFilesExtractor%0Acreate%20ZipProblemProviderAdapter%0Acreate%20FilesMapper%0Afull_run-%3EProblemGeneration%3ARunProblemGeneration%0Aactivate%20ProblemGeneration%0AProblemGeneration-%3EProblemGeneration%3ACreateDirectories%0Anote%20over%20ProblemGeneration%3A%20Create%20-Xpansion%20and%20-Xpansion%2Flp%20directories%0AProblemGeneration-%3EProblemGeneration%3AExtractUtilsFiles%0A%0AProblemGeneration-%3ELpFilesExtractor%3AExtractFiles%0AProblemGeneration-%3EFilesMapper%3AFillMap%0Anote%20over%20FilesMapper%3A%20read%20mps%20file%20list%20from%20zip%20%5Cn%20Fill%20Map%3CPair%3CYear%2C%20Week%3E%2C%20Tuple%3CMPSPAth%2C%20VariablePAth%2C%20ConstraintPath%3E%3E%20%0Adestroyafter%20FilesMapper%0A%0Aalt%20use%20zip%20implementation%20%3D%20true%20%7Bconst%7D%0Aloop%20mps%20files%0AProblemGeneration-%3EZipProblemProviderAdapter%3AProvideProblems%0Anote%20over%20ZipProblemProviderAdapter%3AExtract%20problem%20file%20from%20zip%5Cn%20create%20%22problem%22%2Fsolver%0Aend%0Aloop%20Problems%0AProblemGeneration-%3EProblemGeneration%3Atreat%0Anote%20over%20ProblemGeneration%3A%20write%20mps%0Aend%0Aend%0AProblemGeneration-%3EProblemGeneration%3AMasterGeneration()%0Adeactivateafter%20ProblemGeneration%0Adestroy%20LpFilesExtractor%0Adestroy%20ZipProblemProviderAdapter%0Adestroyafter%20ProblemGeneration%0A%0A%0Aparticipantgroup%20%23lightpink%20**Benders**%0Aparticipant%20BendersFactory%0Aparticipant%20Benders%0Aend%20%0Afull_run-%3EBendersFactory%3ARunBenders%0ABendersFactory-%3EBendersFactory%3ARead%20options%20file%0ABendersFactory-%3EBendersFactory%3ASet%20log%20destination%0Anote%20over%20BendersFactory%3A%20bendersLog-rank%3CN%3E.txt%0ABendersFactory-%3EBendersFactory%3AFileAndStdoutLoggerFactory%0Anote%20over%20BendersFactory%3Alog%20filename%3A%20reportbenders.txt%5Cndestination%3A%20%22.%22%20with%20previous%20chdir%20to%20lp%20dir%0ABendersFactory-%3EBendersFactory%3Aset%20writer%0Anote%20over%20BendersFactory%3A%20write%20in%20simulation-Xpansion%2Fexpansion%2Fout.json%0ABendersFactory-%3EBenders%3Alaunch%0A%0A%0Afull_run-%3Efull_run%3AupdateStudy%0A%0A%0Adeactivateafter%20full_run%0Adestroyafter%20full_run%0Aalt%20not%20keep_mps%0Afull_run_driver-%3EStudyOutputCleaner%3Aclean_benders_step%0Anote%20over%20StudyOutputCleaner%3Arm%20simulation-Xpansion%2Flp%2F*.mps%5Cnrm%20simulation-Xpansion%2Flp%2F*.lp%5Cnrm%20simulation-Xpansion%2Flp%2F*.zip%0Aend%0AXpansionDriver-%3EXpansionDriver%3Aclean_step%0Aactivate%20XpansionDriver%0Acreate%20antares_archive_updater_exe%0AXpansionDriver-%3Eantares_archive_updater_exe%3ARun%0Anote%20over%20antares_archive_updater_exe%3A%20add%20sumulation-Xpansion%20in%20simulation.zip%5Cnrm%20simulation-Xpansion%0Aantares_archive_updater_exe-%3Eantares_archive_updater_exe%3ACleanAntaresArchive%0Adestroyafter%20antares_archive_updater_exe%0Anote%20over%20XpansionDriver%3Arm%20output%2Fsimulation-Xpansion%0Adeactivateafter%20XpansionDriver%0A @@ -74,16 +74,18 @@ + + - + - Xpansion folder and file management (full run) + Xpansion folder and file management (full run) @@ -91,66 +93,66 @@ - ProblemGeneration + ProblemGeneration - Benders + Benders + d=" M 1400.33824394949 123.19256396400002 L 1400.33824394949 4771.789325832002" stroke-miterlimit="10" + stroke-width="1.2565541" stroke-dasharray="11.598960923076923,5.0262164"/> @@ -230,25 +232,25 @@ BendersFactory Benders @@ -260,13 +262,13 @@ d=" M 526.1165711588261 462.462170964 L 541.1952203588261 462.462170964 L 541.1952203588261 885.7198540080001 L 526.1165711588261 885.7198540080001 L 526.1165711588261 462.462170964" stroke-miterlimit="10" stroke-width="1.2565541" stroke-dasharray=""/> launch() @@ -652,38 +654,38 @@ LpFilesExtractor ZipProblemProviderAdapter FilesMapper @@ -699,7 +701,7 @@ - ExtractUtilsFiles + CreateDirectories + + + Create -Xpansion and -Xpansion/lp directories + + + ExtractUtilsFiles + + + + + + + - ExtractFiles + ExtractFiles - + + d=" M 2571.1704995140853 2778.5929502480008 L 2583.7360405140853 2784.8757207480007 L 2571.1704995140853 2791.1584912480007 Z"/> - FillMap + FillMap - + + d=" M 3095.459239666581 2820.813168008001 L 3108.024780666581 2827.095938508001 L 3095.459239666581 2833.378709008001 Z"/> - read mps file list from zip  + read mps file list from zip   Fill Map<Pair<Year, Week>, Tuple<MPSPAth, VariablePAth, ConstraintPath>>  - ProvideProblems + ProvideProblems - + + d=" M 2778.5564797705656 3066.595149968002 L 2791.1220207705655 3072.877920468002 L 2778.5564797705656 3079.1606909680017 Z"/> - Extract problem file from zip + Extract problem file from zip  create "problem"/solver - treat + treat - + + d=" M 2399.9654872674837 3264.125454488002 L 2387.3999462674838 3270.408224988002 L 2399.9654872674837 3276.690995488002 Z"/> - write mps + write mps - MasterGeneration() + MasterGeneration() - + + d=" M 2399.9654872674837 3428.482730768002 L 2387.3999462674838 3434.765501268002 L 2399.9654872674837 3441.048271768002 Z"/> - RunBenders + RunBenders - + + d=" M 3493.241658086833 3520.4624908880023 L 3505.807199086833 3526.7452613880023 L 3493.241658086833 3533.0280318880023 Z"/> - Read options file + Read options file - + + d=" M 3518.372740086833 3582.284952608002 L 3505.807199086833 3588.567723108002 L 3518.372740086833 3594.850493608002 Z"/> - Set log destination + Set log destination - + + d=" M 3518.372740086833 3644.107414328002 L 3505.807199086833 3650.390184828002 L 3518.372740086833 3656.672955328002 Z"/> - bendersLog-rank<N>.txt + bendersLog-rank<N>.txt - FileAndStdoutLoggerFactory + FileAndStdoutLoggerFactory - + + d=" M 3518.372740086833 3763.228743008002 L 3505.807199086833 3769.511513508002 L 3518.372740086833 3775.794284008002 Z"/> - log filename: reportbenders.txt + log filename: reportbenders.txt destination: "." with previous chdir to lp dir - set writer + set writer - + + d=" M 3518.372740086833 3897.428720888002 L 3505.807199086833 3903.711491388002 L 3518.372740086833 3909.994261888002 Z"/> - write in simulation-Xpansion/expansion/out.json + write in simulation-Xpansion/expansion/out.json - launch + launch - + + d=" M 3726.5925133258957 3996.947805608002 L 3739.1580543258956 4003.230576108002 L 3726.5925133258957 4009.513346608002 Z"/> - updateStudy + updateStudy - + + d=" M 2010.3097282405427 4058.770267328002 L 1997.7441872405427 4065.053037828002 L 2010.3097282405427 4071.335808328002 Z"/> - clean_benders_step + clean_benders_step - + + d=" M 743.8849472863652 4191.462380288001 L 731.3194062863652 4197.745150788001 L 743.8849472863652 4204.027921288001 Z"/> - rm simulation-Xpansion/lp/*.mps + rm simulation-Xpansion/lp/*.mps rm simulation-Xpansion/lp/*.lp rm simulation-Xpansion/lp/*.zip - clean_step + clean_step - + + d=" M 238.95927275882616 4363.358981168002 L 226.39373175882616 4369.641751668002 L 238.95927275882616 4375.924522168002 Z"/> antares_archive_updater_exe - Run + Run - + + d=" M 3964.868278914116 4471.472895932003 L 3977.433819914116 4477.755666432003 L 3964.868278914116 4484.038436932003 Z"/> - add sumulation-Xpansion in simulation.zip + add sumulation-Xpansion in simulation.zip rm simulation-Xpansion - CleanAntaresArchive + CleanAntaresArchive - + + d=" M 3989.999360914116 4605.672873812003 L 3977.433819914116 4611.955644312003 L 3989.999360914116 4618.238414812003 Z"/> - rm output/simulation-Xpansion + rm output/simulation-Xpansion @@ -1090,77 +1110,77 @@ alt - [use zip implementation = true {const}] loop - [mps files] loop - [Problems] alt - [not keep_mps] diff --git a/conception/SequenceDiagrams/FileManipulation/diagram.txt b/conception/SequenceDiagrams/FileManipulation/diagram.txt index 108693ed1..63ddf5ee7 100644 --- a/conception/SequenceDiagrams/FileManipulation/diagram.txt +++ b/conception/SequenceDiagrams/FileManipulation/diagram.txt @@ -68,6 +68,8 @@ create ZipProblemProviderAdapter create FilesMapper full_run->ProblemGeneration:RunProblemGeneration activate ProblemGeneration +ProblemGeneration->ProblemGeneration:CreateDirectories +note over ProblemGeneration: Create -Xpansion and -Xpansion/lp directories ProblemGeneration->ProblemGeneration:ExtractUtilsFiles ProblemGeneration->LpFilesExtractor:ExtractFiles From ab0a4e5132169abc0f9cf2d6f95c522ec7e30ad6 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 26 Oct 2023 11:45:56 +0200 Subject: [PATCH 12/13] Revert driver changes (out of scope) --- .../antares_xpansion/full_run_driver.py | 3 + .../problem_generator_driver.py | 10 +++ tests/python/test_problem_generator_driver.py | 68 +++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/src/python/antares_xpansion/full_run_driver.py b/src/python/antares_xpansion/full_run_driver.py index d10ef46bb..d88baa2fc 100644 --- a/src/python/antares_xpansion/full_run_driver.py +++ b/src/python/antares_xpansion/full_run_driver.py @@ -29,6 +29,9 @@ def prepare_drivers(self, output_path: Path, """ problem generation step : getnames + lp_namer """ + # Pb Gen pre-step + self.problem_generation_driver.clear_old_log() + self.problem_generation_driver.is_relaxed = problem_generation_is_relaxed self.keep_mps = benders_keep_mps diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index 09af5e0fd..1df3e4e3b 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -3,6 +3,7 @@ """ import os +import shutil import subprocess import sys from dataclasses import dataclass @@ -65,6 +66,7 @@ def launch(self, output_path: Path, is_relaxed: bool): """ problem generation step : getnames + lp_namer """ + self.clear_old_log() self.logger.info("Problem Generation") self.output_path = output_path @@ -79,6 +81,8 @@ def set_output_path(self, output_path): self._output_path = output_path self.xpansion_output_dir = output_path.parent / \ (output_path.stem + "-Xpansion") + if self.xpansion_output_dir.exists(): + shutil.rmtree(self.xpansion_output_dir) os.makedirs(self.xpansion_output_dir) self._lp_path = os.path.normpath( os.path.join(self.xpansion_output_dir, 'lp')) @@ -89,6 +93,10 @@ def set_output_path(self, output_path): def get_output_path(self): return self._output_path + def clear_old_log(self): + if (os.path.isfile(str(self.lp_namer_exe_path) + '.log')): + os.remove(str(self.lp_namer_exe_path) + '.log') + def _lp_step(self): """ copies area and interco files and launches the lp_namer @@ -104,6 +112,8 @@ def _lp_step(self): "ERROR: exited lpnamer with status %d" % returned_l.returncode) def create_lp_dir(self): + if os.path.isdir(self._lp_path): + shutil.rmtree(self._lp_path) os.makedirs(self._lp_path) def lp_namer_options(self): diff --git a/tests/python/test_problem_generator_driver.py b/tests/python/test_problem_generator_driver.py index 21ae640e4..7d7647407 100644 --- a/tests/python/test_problem_generator_driver.py +++ b/tests/python/test_problem_generator_driver.py @@ -1,6 +1,7 @@ import shutil from datetime import date, datetime from pathlib import Path +from unittest.mock import patch import pytest from antares_xpansion.problem_generator_driver import ProblemGeneratorData, ProblemGeneratorDriver @@ -39,6 +40,12 @@ def test_problem_generator_data(self): assert problem_generator_driver.user_weights_file_path == self.empty_pblm_gen_data.user_weights_file_path assert problem_generator_driver.lp_namer_exe_path == self.empty_pblm_gen_data.lp_namer_exe_path + def test_output_path(self, tmp_path): + problem_generator_driver = ProblemGeneratorDriver( + self.empty_pblm_gen_data) + with pytest.raises(ProblemGeneratorDriver.OutputPathError): + problem_generator_driver.launch(tmp_path / "i_don_t_exist", False) + def test_lp_namer_exe_does_not_exit(self, tmp_path): self._create_empty_area_file(tmp_path) @@ -52,6 +59,67 @@ def test_lp_namer_exe_does_not_exit(self, tmp_path): def xpansion_output(self, tmp_path): return tmp_path.parent / (tmp_path.name + '-Xpansion') + def test_clear_old_log(self, tmp_path): + + lp_namer_file = tmp_path / self.lp_exe + lp_namer_file.write_text("") + pblm_gen_data = ProblemGeneratorData(keep_mps=False, + additional_constraints="", + user_weights_file_path=Path(""), + weight_file_name_for_lp="", + lp_namer_exe_path=lp_namer_file, + active_years=[]) + self._create_empty_area_file(tmp_path) + self._create_empty_interco_file(tmp_path) + output_zipped = get_zipped_output(tmp_path) + log_file_name = self.lp_exe + ".log" + xpansion_dir = self.xpansion_output(tmp_path) + log_file = xpansion_dir / log_file_name + xpansion_dir.mkdir() + log_file.write_text("bla bla") + assert log_file.exists() + + pblm_gen = ProblemGeneratorDriver(pblm_gen_data) + with patch(SUBPROCESS_RUN, autospec=True): + with pytest.raises(ProblemGeneratorDriver.LPNamerExecutionError): + pblm_gen.launch(output_zipped, False) + + assert not log_file.exists() + + def test_clean_lp_dir_before_run(self, tmp_path): + + lp_namer_file = tmp_path / self.lp_exe + lp_namer_file.write_text("") + pblm_gen_data = ProblemGeneratorData(keep_mps=False, + additional_constraints="", + user_weights_file_path=Path(""), + weight_file_name_for_lp="", + lp_namer_exe_path=lp_namer_file, + active_years=[]) + self._create_empty_area_file(tmp_path) + self._create_empty_interco_file(tmp_path) + output_zipped = get_zipped_output(tmp_path) + log_file_name = self.lp_exe + ".log" + xpansion_dir = self.xpansion_output(tmp_path) + xpansion_dir.mkdir() + log_file = xpansion_dir / log_file_name + log_file.write_text("bla bla") + + lp_dir = xpansion_dir / "lp" + lp_dir.mkdir() + lp_dir_sub_file_1 = lp_dir / "file1" + lp_dir_sub_file_1.write_text("") + + assert lp_dir.exists() + assert lp_dir_sub_file_1.exists() + problem_generator_driver = ProblemGeneratorDriver(pblm_gen_data) + with patch(SUBPROCESS_RUN, autospec=True) as run_function: + run_function.return_value.returncode = 0 + problem_generator_driver.launch(output_zipped, False) + + assert lp_dir.exists() + assert not lp_dir_sub_file_1.exists() + def _get_expected_mps_txt(self, tmp_path): weeks = [1, 2, 3] From 8c0a7c7cd93e05ffff21c3914c67f3ac74b647ee Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 26 Oct 2023 16:09:20 +0200 Subject: [PATCH 13/13] "Fix" Test --- tests/cpp/lp_namer/ProblemGenerationTest.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp index 1fa3a3f36..2e8506965 100644 --- a/tests/cpp/lp_namer/ProblemGenerationTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationTest.cpp @@ -9,21 +9,6 @@ #include "RunProblemGeneration.h" #include "gtest/gtest.h" -TEST(InitializationTest, FoldersAreCreated) { - auto workingDirectory = - std::filesystem::temp_directory_path() / std::tmpnam(nullptr); - auto simulationDirectory = - workingDirectory / "output" / "simulation-Xpansion"; - auto logger = emptyLogger(); - std::filesystem::create_directories(simulationDirectory); - EXPECT_THROW( - RunProblemGeneration(simulationDirectory, "integer", "", "", logger, - simulationDirectory / "logs.txt", "", false), - ArchiveIOGeneralException); - auto lp = (simulationDirectory) / "lp"; - EXPECT_TRUE(std::filesystem::exists(lp)); -} - TEST(InitializationTest, FoldersAreEmpty) { auto workingDirectory = std::filesystem::temp_directory_path() / std::tmpnam(nullptr);