diff --git a/.github/workflows/build_ubuntu.yml b/.github/workflows/build_ubuntu.yml index ed60def35..ba64758a6 100644 --- a/.github/workflows/build_ubuntu.yml +++ b/.github/workflows/build_ubuntu.yml @@ -65,6 +65,16 @@ jobs: sudo apt-get install -y ccache cmake libgtest-dev libjsoncpp-dev libtbb-dev libopenmpi-dev sudo apt-get install -y g++-10 gcc-10 + - name: Update alternatives + #mpicxx uses "g++" so we need g++ to be symbolic link to g++-10 + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 + sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 + sudo update-alternatives --set cc /usr/bin/gcc + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 + sudo update-alternatives --set c++ /usr/bin/g++ + - name: Read antares-solver version id: antares-version uses: ./.github/actions/read-json-value diff --git a/.github/workflows/compile-boost/action.yml b/.github/workflows/compile-boost/action.yml index 72e116dc0..c8f9bbe88 100644 --- a/.github/workflows/compile-boost/action.yml +++ b/.github/workflows/compile-boost/action.yml @@ -4,9 +4,6 @@ inputs: prefix: description: 'extra qualifiers' required: true - load-toolset: - description: 'load devtoolset 10' - default: 'true' runs: using: "composite" steps: diff --git a/.github/workflows/compile-tbb/action.yml b/.github/workflows/compile-tbb/action.yml index 9deb17fdc..135c496b6 100644 --- a/.github/workflows/compile-tbb/action.yml +++ b/.github/workflows/compile-tbb/action.yml @@ -4,9 +4,7 @@ inputs: cmake: description: 'cmake version' default: cmake - load-toolset: - description: 'load devtoolset 10' - default: 'true' + runs: using: "composite" steps: diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 6e959664c..e1157359d 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -31,6 +31,11 @@ jobs: - name: Install sonar-scanner and build-wrapper uses: SonarSource/sonarcloud-github-c-cpp@v2 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: sonarcloud-${{ env.SONAR_SCANNER_VERSION }} + - name: Set up Python uses: actions/setup-python@v4 with: @@ -48,10 +53,21 @@ jobs: run: | sudo apt-get update --fix-missing sudo apt-get install libjsoncpp-dev libgtest-dev libboost-mpi-dev libboost-program-options-dev libtbb-dev + sudo apt-get install g++-10 gcc-10 cd /usr/src/googletest/ sudo cmake . sudo cmake --build . --target install + - name: Update alternatives + #mpicxx uses "g++" so we need g++ to be symbolic link to g++-10 + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 + sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 + sudo update-alternatives --set cc /usr/bin/gcc + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 + sudo update-alternatives --set c++ /usr/bin/g++ + - name: Read antares-solver version id: antares-version uses: ./.github/actions/read-json-value @@ -80,7 +96,6 @@ jobs: antares-version: ${{steps.antares-version.outputs.result}} os: ${{matrix.os}} os-full-name: Ubuntu-20.04 - buildtype: Debug - name: Compile Boost uses: ./.github/workflows/compile-boost @@ -94,11 +109,22 @@ jobs: git submodule update --init --recursive . - name: Configure + shell: bash run: | - cmake -B _build -S . -DDEPS_INSTALL_DIR=rte-antares-deps-Debug -DCODE_COVERAGE=ON -DBUILD_TESTING=ON -DBUILD_antares_solver=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=_install + cmake -B _build -S . \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER=/usr/bin/gcc-10 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++-10 \ + -DDEPS_INSTALL_DIR=rte-antares-deps-Release \ + -DCODE_COVERAGE=ON \ + -DBUILD_TESTING=ON \ + -DBUILD_antares_solver=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=_install - name: Build - run: build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/_build/output cmake --build _build --config Debug -j2 + run: build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/_build/output cmake --build _build --config Release -j2 - name: Test and generate coverage continue-on-error: true diff --git a/CMakeLists.txt b/CMakeLists.txt index e3fb6c5fb..8f60685df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,7 +98,6 @@ message ("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}") message ("CMAKE_C_COMPILER_VERSION : ${CMAKE_C_COMPILER_VERSION}") message ("CMAKE_CXX_COMPILER_VERSION : ${CMAKE_CXX_COMPILER_VERSION}") -set(CODE_COVERAGE FALSE CACHE BOOL "Enable code coverage") if (CODE_COVERAGE) # if code-coverage is ON, force tests build set(BUILD_TESTING ON) diff --git a/src/cpp/exe/full_run/main.cpp b/src/cpp/exe/full_run/main.cpp index dce49c82a..59203a50a 100644 --- a/src/cpp/exe/full_run/main.cpp +++ b/src/cpp/exe/full_run/main.cpp @@ -1,11 +1,10 @@ #include #include #include -#include #include "FullRunOptionsParser.h" +#include "ProblemGeneration.h" #include "ProblemGenerationLogger.h" -#include "RunProblemGeneration.h" #include "StudyUpdateRunner.h" #include "common_mpi.h" namespace po = boost::program_options; @@ -15,26 +14,11 @@ int main(int argc, char** argv) { mpi::communicator world; auto options_parser = FullRunOptionsParser(); std::filesystem::path xpansion_output_dir; - std::filesystem::path archive_path; options_parser.Parse(argc, argv); if (world.rank() == 0) { try { - xpansion_output_dir = options_parser.XpansionOutputDir(); - archive_path = options_parser.ArchivePath(); - - const auto log_file_path = - xpansion_output_dir / "lp" / "ProblemGenerationLog.txt"; - auto logger = ProblemGenerationLog::BuildLogger( - log_file_path, std::cout, "Full Run - Problem Generation"); - - auto master_formulation = options_parser.MasterFormulation(); - auto additionalConstraintFilename_l = - options_parser.AdditionalConstraintsFilename(); - auto weights_file = options_parser.WeightsFile(); - const auto unnamed_problems = options_parser.UnnamedProblems(); - RunProblemGeneration(xpansion_output_dir, master_formulation, - additionalConstraintFilename_l, archive_path, logger, - log_file_path, weights_file, unnamed_problems); + ProblemGeneration pbg(options_parser); + xpansion_output_dir = pbg.updateProblems(); } catch (std::exception& e) { std::cerr << "error: " << e.what() << std::endl; diff --git a/src/cpp/exe/lpnamer/main.cpp b/src/cpp/exe/lpnamer/main.cpp index dc940d881..4dd8fe136 100644 --- a/src/cpp/exe/lpnamer/main.cpp +++ b/src/cpp/exe/lpnamer/main.cpp @@ -1,45 +1,16 @@ #include +#include "ProblemGeneration.h" #include "ProblemGenerationExeOptions.h" -#include "ProblemGenerationLogger.h" -#include "RunProblemGeneration.h" - -static const std::string LP_DIRNAME = "lp"; - -void CreateDirectories(const std::filesystem::path& output_path) { - if (!std::filesystem::exists(output_path)) { - std::filesystem::create_directories(output_path); - } - auto lp_path = output_path / LP_DIRNAME; - if (!std::filesystem::exists(lp_path)) { - std::filesystem::create_directories(lp_path); - } -} int main(int argc, char** argv) { try { auto options_parser = ProblemGenerationExeOptions(); options_parser.Parse(argc, argv); - auto xpansion_output_dir = options_parser.XpansionOutputDir(); - auto archive_path = options_parser.ArchivePath(); - using namespace ProblemGenerationLog; - const auto log_file_path = - xpansion_output_dir / "lp" / "ProblemGenerationLog.txt"; - - CreateDirectories(xpansion_output_dir); - auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout, - "Problem Generation"); - - auto master_formulation = options_parser.MasterFormulation(); - auto additionalConstraintFilename_l = - options_parser.AdditionalConstraintsFilename(); - auto weights_file = options_parser.WeightsFile(); - auto unnamed_problems = options_parser.UnnamedProblems(); - RunProblemGeneration(xpansion_output_dir, master_formulation, - additionalConstraintFilename_l, archive_path, logger, - log_file_path, weights_file, unnamed_problems); + ProblemGeneration pbg(options_parser); + pbg.updateProblems(); return 0; } catch (std::exception& e) { diff --git a/src/cpp/full_run/FullRunOptionsParser.cpp b/src/cpp/full_run/FullRunOptionsParser.cpp index d88669b2d..aa0efe03c 100644 --- a/src/cpp/full_run/FullRunOptionsParser.cpp +++ b/src/cpp/full_run/FullRunOptionsParser.cpp @@ -13,5 +13,5 @@ FullRunOptionsParser::FullRunOptionsParser() : ProblemGenerationExeOptions() { "path to json solution file"); } void FullRunOptionsParser::Parse(unsigned int argc, const char* const* argv) { - OptionsParser::Parse(argc, argv); + ProblemGenerationExeOptions::Parse(argc, argv); } \ No newline at end of file diff --git a/src/cpp/full_run/include/FullRunOptionsParser.h b/src/cpp/full_run/include/FullRunOptionsParser.h index fe8068774..078562e82 100644 --- a/src/cpp/full_run/include/FullRunOptionsParser.h +++ b/src/cpp/full_run/include/FullRunOptionsParser.h @@ -8,7 +8,7 @@ class FullRunOptionsParser : public ProblemGenerationExeOptions { public: FullRunOptionsParser(); - virtual ~FullRunOptionsParser() = default; + ~FullRunOptionsParser() override = default; void Parse(unsigned int argc, const char* const* argv) override; std::filesystem::path BendersOptionsFile() const { diff --git a/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp b/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp index 07aca2dd4..2c0ebfa2f 100644 --- a/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp +++ b/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp @@ -29,9 +29,11 @@ std::vector CandidatesINIReader::ReadAntaresIntercoFile( const std::filesystem::path &antaresIntercoFile) const { std::ifstream interco_filestream(antaresIntercoFile); if (!interco_filestream.good()) { - (*logger_)(LogUtils::LOGLEVEL::FATAL) - << LOGLOCATION << "unable to open " << antaresIntercoFile.string(); - std::exit(1); + auto loglocation = LOGLOCATION; + using namespace std::string_literals; + auto message = "unable to open "s + antaresIntercoFile.string(); + (*logger_)(LogUtils::LOGLEVEL::FATAL) << LOGLOCATION << message; + throw InvalidIntercoFile(message, loglocation); } return ReadLineByLineInterco(interco_filestream); diff --git a/src/cpp/lpnamer/input_reader/CandidatesINIReader.h b/src/cpp/lpnamer/input_reader/CandidatesINIReader.h index 7ca622d33..2204c80ce 100644 --- a/src/cpp/lpnamer/input_reader/CandidatesINIReader.h +++ b/src/cpp/lpnamer/input_reader/CandidatesINIReader.h @@ -41,6 +41,11 @@ class CandidatesINIReader { std::vector readCandidateData( const std::filesystem::path& candidateFile) const; + class InvalidIntercoFile + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; + }; + private: bool checkArea(std::string const& areaName_p) const; CandidateData readCandidateSection(const std::filesystem::path& candidateFile, diff --git a/src/cpp/lpnamer/input_reader/MpsTxtWriter.cpp b/src/cpp/lpnamer/input_reader/MpsTxtWriter.cpp index 2c6e078e0..b7538047c 100644 --- a/src/cpp/lpnamer/input_reader/MpsTxtWriter.cpp +++ b/src/cpp/lpnamer/input_reader/MpsTxtWriter.cpp @@ -5,22 +5,69 @@ #include "ArchiveReader.h" +using namespace std::string_literals; + void FilesMapper::FillMap() { + if (antares_archive_path_.empty()) + FillMapFiles(); + else + FillMapZip(); +} + +auto FileWithExtension(std::string_view ext, + const std::filesystem::path& directory) { + std::vector vect_of_files_with_ext; + for (auto& p : std::filesystem::directory_iterator(directory)) { + if (p.path().extension().string() == ext) + vect_of_files_with_ext.push_back(p.path().filename()); + } + return vect_of_files_with_ext; +} + +void FilesMapper::FillMapFiles() { + year_week_and_files_dict_.clear(); + FillMapWithMpsFiles(FileWithExtension(".mps"s, simulation_dir_)); + std::vector variables_files; + std::vector constraints_files; + + auto vect_of_files_with_txt_ext = FileWithExtension(".txt"s, simulation_dir_); + + for (const auto& file : vect_of_files_with_txt_ext) { + auto filename_str = file.stem().string(); + auto is_it_variables_file = + filename_str.find("variables"s) != std::string::npos; + auto is_it_constraints_file = + filename_str.find("constraints"s) != std::string::npos; + if (is_it_variables_file && is_it_constraints_file) { + // case file : ***variables**constraints***.txt or + // ***constraints**variables***.txt + continue; + } else if (is_it_variables_file) { + variables_files.emplace_back(file); + } else if (is_it_constraints_file) { + constraints_files.emplace_back(file); + } + } + FillMapWithVariablesFiles(variables_files); + FillMapWithConstraintsFiles(constraints_files); +} + +void FilesMapper::FillMapZip() { auto zip_reader = ArchiveReader(antares_archive_path_); zip_reader.Open(); zip_reader.LoadEntriesPath(); year_week_and_files_dict_.clear(); - FillMapWithMpsFiles(zip_reader.GetEntriesPathWithExtension(".mps")); + FillMapWithMpsFiles(zip_reader.GetEntriesPathWithExtension(".mps"s)); std::vector variables_files; std::vector constraints_files; auto vect_of_files_with_txt_ext = - zip_reader.GetEntriesPathWithExtension(".txt"); + zip_reader.GetEntriesPathWithExtension(".txt"s); for (const auto& file : vect_of_files_with_txt_ext) { auto filename_str = file.stem().string(); auto is_it_variables_file = - filename_str.find("variables") != std::string::npos; + filename_str.find("variables"s) != std::string::npos; auto is_it_constraints_file = - filename_str.find("constraints") != std::string::npos; + filename_str.find("constraints"s) != std::string::npos; if (is_it_variables_file && is_it_constraints_file) { // case file : ***variables**constraints***.txt or // ***constraints**variables***.txt @@ -67,7 +114,7 @@ void FilesMapper::FillMapWithConstraintsFiles( YearAndWeek FilesMapper::YearAndWeekFromFileName( const std::filesystem::path& file_name) const { auto split_file_name = - StringManip::split(StringManip::trim(file_name.string()), '-'); + StringManip::split(StringManip::trim(file_name.filename().string()), '-'); return {std::stoi(split_file_name[1]), std::stoi(split_file_name[2])}; } diff --git a/src/cpp/lpnamer/input_reader/MpsTxtWriter.h b/src/cpp/lpnamer/input_reader/MpsTxtWriter.h index 974be3232..9f3a81c3d 100644 --- a/src/cpp/lpnamer/input_reader/MpsTxtWriter.h +++ b/src/cpp/lpnamer/input_reader/MpsTxtWriter.h @@ -28,8 +28,10 @@ struct ProblemData { class FilesMapper { public: - explicit FilesMapper(const std::filesystem::path& antares_archive_path) - : antares_archive_path_(antares_archive_path) {} + explicit FilesMapper(std::filesystem::path antares_archive_path, + std::filesystem::path simulation_dir = {}) + : antares_archive_path_(std::move(antares_archive_path)), + simulation_dir_(std::move(simulation_dir)) {} YearWeekAndFilesDict FilesMap() { FillMap(); return year_week_and_files_dict_; @@ -39,6 +41,7 @@ class FilesMapper { private: YearWeekAndFilesDict year_week_and_files_dict_; std::filesystem::path antares_archive_path_; + std::filesystem::path simulation_dir_; void FillMap(); void FillMapWithMpsFiles(const std::vector& mps_files); void FillMapWithVariablesFiles( @@ -47,5 +50,7 @@ class FilesMapper { const std::vector& variables_files); YearAndWeek YearAndWeekFromFileName( const std::filesystem::path& file_name) const; + void FillMapFiles(); + void FillMapZip(); }; #endif // SRC_CPP_LPNAMER_INPUTREADER_MPSTXTWRITER_H \ No newline at end of file diff --git a/src/cpp/lpnamer/main/CMakeLists.txt b/src/cpp/lpnamer/main/CMakeLists.txt index 6c01b7025..bfa797d0f 100644 --- a/src/cpp/lpnamer/main/CMakeLists.txt +++ b/src/cpp/lpnamer/main/CMakeLists.txt @@ -11,8 +11,12 @@ # --------------------------------------------------------------------------- add_library (problem_generation_main STATIC - ${CMAKE_CURRENT_SOURCE_DIR}/RunProblemGeneration.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ProblemGenerationExeOptions.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemGeneration.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/ProblemGeneration.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/ProblemGenerationOptions.h + ${CMAKE_CURRENT_SOURCE_DIR}/Version.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Version.h ) target_link_libraries (problem_generation_main @@ -26,11 +30,11 @@ target_link_libraries (problem_generation_main get_target_property(helpers_include helpers INTERFACE_INCLUDE_DIRECTORIES) - target_include_directories (problem_generation_main - PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${helpers_include} - ) +target_include_directories(problem_generation_main + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${helpers_include} +) add_library (${PROJECT_NAME}::problem_generation_main ALIAS problem_generation_main) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/ProblemGeneration.cpp similarity index 65% rename from src/cpp/lpnamer/main/RunProblemGeneration.cpp rename to src/cpp/lpnamer/main/ProblemGeneration.cpp index 527075c13..7380e08f4 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/ProblemGeneration.cpp @@ -1,11 +1,15 @@ +// +// Created by marechaljas on 27/10/23. +// -#include "RunProblemGeneration.h" +#include "include/ProblemGeneration.h" #include #include #include "ActiveLinks.h" #include "AdditionalConstraints.h" +#include "FileProblemsProviderAdapter.h" #include "GeneralDataReader.h" #include "LauncherHelpers.h" #include "LinkProblemsGenerator.h" @@ -15,50 +19,60 @@ #include "MasterGeneration.h" #include "MasterProblemBuilder.h" #include "MpsTxtWriter.h" +#include "ProblemGenerationLogger.h" +#include "ProblemVariablesFileAdapter.h" #include "ProblemVariablesFromProblemAdapter.h" #include "ProblemVariablesZipAdapter.h" #include "StringManip.h" #include "Timer.h" +#include "Version.h" #include "WeightsFileReader.h" #include "WeightsFileWriter.h" #include "ZipProblemsProviderAdapter.h" #include "config.h" -struct Version { - explicit Version(std::string_view version) { - auto split_version = StringManip::split(StringManip::trim(version), '.'); - major = std::stoi(split_version[0]); - minor = std::stoi(split_version[1]); - } +static const std::string LP_DIRNAME = "lp"; - bool operator<(const Version& another) const { - if (this->major == another.major) { - return this->minor < another.minor; - } else { - return this->major < another.major; - } +void CreateDirectories(const std::filesystem::path& output_path) { + if (!std::filesystem::exists(output_path)) { + std::filesystem::create_directories(output_path); + } + auto lp_path = output_path / LP_DIRNAME; + if (!std::filesystem::exists(lp_path)) { + std::filesystem::create_directories(lp_path); } +} - bool operator>(const Version& another) const { return !operator<(another); } +ProblemGeneration::ProblemGeneration(ProblemGenerationOptions& options) + : options_(options) {} +std::filesystem::path ProblemGeneration::updateProblems() { + const auto xpansion_output_dir = options_.XpansionOutputDir(); + const auto archive_path = options_.ArchivePath(); - bool operator==(const Version& another) const { - return (this->major == another.major && this->minor == another.minor); - } - bool operator<=(const Version& another) const { - return (operator<(another) || operator==(another)); - } - bool operator>=(const Version& another) const { - return (operator>(another) || operator==(another)); - } + auto deduced_xpansion_output_dir = + options_.deduceXpansionDirIfEmpty(xpansion_output_dir, archive_path); - private: - int major; - int minor; -}; + const auto log_file_path = + deduced_xpansion_output_dir / "lp" / "ProblemGenerationLog.txt"; + + CreateDirectories(deduced_xpansion_output_dir); + auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout, + "Problem Generation"); + + auto master_formulation = options_.MasterFormulation(); + auto additionalConstraintFilename_l = + options_.AdditionalConstraintsFilename(); + auto weights_file = options_.WeightsFile(); + auto unnamed_problems = options_.UnnamedProblems(); + + RunProblemGeneration(deduced_xpansion_output_dir, master_formulation, + additionalConstraintFilename_l, archive_path, logger, log_file_path, weights_file, unnamed_problems); + return deduced_xpansion_output_dir; +} std::shared_ptr InstantiateZipReader( const std::filesystem::path& antares_archive_path); -void ProcessWeights( +void ProblemGeneration::ProcessWeights( const std::filesystem::path& xpansion_output_dir, const std::filesystem::path& antares_archive_path, const std::filesystem::path& weights_file, @@ -77,7 +91,7 @@ void ProcessWeights( yearly_weight_writer.CreateWeightFile(); } -void ExtractUtilsFiles( +void ProblemGeneration::ExtractUtilsFiles( const std::filesystem::path& antares_archive_path, const std::filesystem::path& xpansion_output_dir, ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) { @@ -116,17 +130,23 @@ void validateMasterFormulation( std::vector> getXpansionProblems( SolverLogManager& solver_log_manager, const std::string& solver_name, const std::vector& mpsList, std::filesystem::path& lpDir_, - std::shared_ptr& reader) { + std::shared_ptr& reader, bool with_archive = true) { std::vector problem_names; std::transform(mpsList.begin(), mpsList.end(), std::back_inserter(problem_names), [](ProblemData const& data) { return data._problem_mps; }); - auto adapter = std::make_shared(lpDir_, reader, - problem_names); - return adapter->provideProblems(solver_name, solver_log_manager); + if (with_archive) { + auto adapter = std::make_shared(lpDir_, reader, + problem_names); + return adapter->provideProblems(solver_name, solver_log_manager); + } else { + auto adapter = + std::make_shared(lpDir_, problem_names); + return adapter->provideProblems(solver_name, solver_log_manager); + } } -void RunProblemGeneration( +void ProblemGeneration::RunProblemGeneration( const std::filesystem::path& xpansion_output_dir, const std::string& master_formulation, const std::string& additionalConstraintFilename_l, @@ -139,13 +159,16 @@ void RunProblemGeneration( validateMasterFormulation(master_formulation, logger); std::string solver_name = "CBC"; // TODO Use solver selected by user + SolverLoader::GetAvailableSolvers(); // Dirty fix to populate static value + // outside multi thread code Timer problem_generation_timer; if (!weights_file.empty()) { ProcessWeights(xpansion_output_dir, antares_archive_path, weights_file, logger); } - ExtractUtilsFiles(antares_archive_path, xpansion_output_dir, logger); + if (!antares_archive_path.empty()) + ExtractUtilsFiles(antares_archive_path, xpansion_output_dir, logger); std::vector links = getLinks(xpansion_output_dir, logger); @@ -164,20 +187,20 @@ void RunProblemGeneration( antares_version < first_version_without_variables_files; (*logger)(LogUtils::LOGLEVEL::INFO) << "rename problems: " << std::boolalpha << rename_problems << std::endl; - auto files_mapper = FilesMapper(antares_archive_path); + auto files_mapper = FilesMapper(antares_archive_path, xpansion_output_dir); auto mpsList = files_mapper.MpsAndVariablesFilesVect(); auto solver_log_manager = SolverLogManager(log_file_path); Couplings couplings; LinkProblemsGenerator linkProblemsGenerator( lpDir_, links, solver_name, logger, solver_log_manager, rename_problems); - std::shared_ptr reader = - InstantiateZipReader(antares_archive_path); + std::shared_ptr reader = + antares_archive_path.empty() ? std::make_shared() + : InstantiateZipReader(antares_archive_path); - /* Main stuff */ - std::vector> xpansion_problems = - getXpansionProblems(solver_log_manager, solver_name, mpsList, lpDir_, - reader); + /* Main stuff */ + std::vector> xpansion_problems = getXpansionProblems(solver_log_manager, solver_name, mpsList, lpDir_, + reader, !antares_archive_path.empty()); std::vector, ProblemData>> problems_and_data; @@ -191,13 +214,18 @@ void RunProblemGeneration( [&](const auto& problem_and_data) { const auto& [problem, data] = problem_and_data; std::shared_ptr variables_provider; - if (rename_problems) { - variables_provider = std::make_shared( - reader, data, links, logger); + if (antares_archive_path.empty()) { + variables_provider = std::make_shared( + data, links, logger, lpDir_); } else { - variables_provider = - std::make_shared( - problem, links, logger); + if (rename_problems) { + variables_provider = std::make_shared( + reader, data, links, logger); + } else { + variables_provider = + std::make_shared( + problem, links, logger); + } } linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(), variables_provider.get(), diff --git a/src/cpp/lpnamer/main/ProblemGenerationExeOptions.cpp b/src/cpp/lpnamer/main/ProblemGenerationExeOptions.cpp index 71efbef91..95c882271 100644 --- a/src/cpp/lpnamer/main/ProblemGenerationExeOptions.cpp +++ b/src/cpp/lpnamer/main/ProblemGenerationExeOptions.cpp @@ -4,10 +4,9 @@ namespace po = boost::program_options; ProblemGenerationExeOptions::ProblemGenerationExeOptions() : OptionsParser("Problem Generation exe") { AddOptions()("help,h", "produce help message")( - "output,o", - po::value(&xpansion_output_dir_)->required(), + "output,o", po::value(&xpansion_output_dir_), "antares-xpansion study output")( - "archive,a", po::value(&archive_path_)->required(), + "archive,a", po::value(&archive_path_), "antares-xpansion study zip")( "formulation,f", po::value(&master_formulation_)->default_value("relaxed"), @@ -20,4 +19,52 @@ ProblemGenerationExeOptions::ProblemGenerationExeOptions() "user weights file")("unnamed-problems,n", po::bool_switch(&unnamed_problems_), "use this option if unnamed problems are provided"); +} +void ProblemGenerationExeOptions::Parse(unsigned int argc, + const char *const *argv) { + OptionsParser::Parse(argc, argv); + auto log_location = LOGLOCATION; + using namespace std::string_literals; + if (!XpansionOutputDir().empty() && !ArchivePath().empty()) { + auto msg = "Giving both archive and output options is not permitted"s; + throw ProblemGenerationOptions::ConflictingParameters(msg, log_location); + } + if (XpansionOutputDir().empty() && ArchivePath().empty()) { + auto msg = "Both output directory and archive path are empty"s; + throw ProblemGenerationOptions::MissingParameters(msg, log_location); + } +} + +std::filesystem::path ProblemGenerationExeOptions::deduceArchivePathIfEmpty( + const std::filesystem::path& xpansion_output_dir, + const std::filesystem::path& archive_path) const { + if (archive_path.empty() && !xpansion_output_dir.empty()) { + if (xpansion_output_dir.string().find("-Xpansion") == std::string::npos) { + auto log_location = LOGLOCATION; + auto msg = + "Archive path is missing and output path does not contains" + " \"-Xpansion\" suffixe. Can't deduce archive file name."; + throw MismatchedParameters(msg, log_location); + } + auto deduced_archive_path = xpansion_output_dir; + auto dir_name = deduced_archive_path.stem().string(); + dir_name = dir_name.substr(0, dir_name.find("-Xpansion")); + deduced_archive_path = + deduced_archive_path.replace_filename(dir_name).replace_extension( + ".zip"); + return deduced_archive_path; + } + return archive_path; +} + +std::filesystem::path ProblemGenerationExeOptions::deduceXpansionDirIfEmpty( + std::filesystem::path xpansion_output_dir, + const std::filesystem::path& archive_path) const { + if (xpansion_output_dir.empty() && !archive_path.empty()) { + auto deduced_dir = archive_path; + deduced_dir = deduced_dir.replace_filename( + deduced_dir.stem().replace_extension("").string() + "-Xpansion"); + return deduced_dir; + } + return xpansion_output_dir; } \ No newline at end of file diff --git a/src/cpp/lpnamer/main/Version.cpp b/src/cpp/lpnamer/main/Version.cpp new file mode 100644 index 000000000..8560ee6b0 --- /dev/null +++ b/src/cpp/lpnamer/main/Version.cpp @@ -0,0 +1,32 @@ +// +// Created by marechaljas on 10/01/24. +// + +#include "Version.h" + +#include "StringManip.h" + +Version::Version(std::string_view version) { + auto split_version = StringManip::split(StringManip::trim(version), '.'); + major = std::stoi(split_version[0]); + minor = std::stoi(split_version[1]); +} +bool Version::operator<(const Version& another) const { + if (this->major == another.major) { + return this->minor < another.minor; + } else { + return this->major < another.major; + } +} +bool Version::operator>(const Version& another) const { + return !operator<(another); +} +bool Version::operator==(const Version& another) const { + return (this->major == another.major && this->minor == another.minor); +} +bool Version::operator<=(const Version& another) const { + return (operator<(another) || operator==(another)); +} +bool Version::operator>=(const Version& another) const { + return (operator>(another) || operator==(another)); +} diff --git a/src/cpp/lpnamer/main/Version.h b/src/cpp/lpnamer/main/Version.h new file mode 100644 index 000000000..2a626d19a --- /dev/null +++ b/src/cpp/lpnamer/main/Version.h @@ -0,0 +1,22 @@ +// +// Created by marechaljas on 10/01/24. +// + +#pragma once +#include + +struct Version { + explicit Version(std::string_view version); + + bool operator<(const Version& another) const; + + bool operator>(const Version& another) const; + + bool operator==(const Version& another) const; + bool operator<=(const Version& another) const; + bool operator>=(const Version& another) const; + + private: + int major; + int minor; +}; \ No newline at end of file diff --git a/src/cpp/lpnamer/main/include/ProblemGeneration.h b/src/cpp/lpnamer/main/include/ProblemGeneration.h new file mode 100644 index 000000000..f9ac5618f --- /dev/null +++ b/src/cpp/lpnamer/main/include/ProblemGeneration.h @@ -0,0 +1,39 @@ +// +// Created by marechaljas on 27/10/23. +// + +#pragma once + +#include +#include + +#include "ProblemGenerationExeOptions.h" +#include "ProblemGenerationLogger.h" +#include "ProblemGenerationOptions.h" + +class ProblemGeneration { + public: + explicit ProblemGeneration(ProblemGenerationOptions& options); + std::filesystem::path updateProblems(); + ProblemGenerationOptions& options_; + + virtual void RunProblemGeneration( + const std::filesystem::path& xpansion_output_dir, + const std::string& master_formulation, + const std::string& additionalConstraintFilename_l, + const std::filesystem::path& archive_path, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger, + const std::filesystem::path& log_file_path, + const std::filesystem::path& weights_file, bool unnamed_problems); + + private: + void ProcessWeights( + const std::filesystem::path& xpansion_output_dir, + const std::filesystem::path& antares_archive_path, + const std::filesystem::path& weights_file, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); + void ExtractUtilsFiles( + const std::filesystem::path& antares_archive_path, + const std::filesystem::path& xpansion_output_dir, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); +}; diff --git a/src/cpp/lpnamer/main/include/ProblemGenerationExeOptions.h b/src/cpp/lpnamer/main/include/ProblemGenerationExeOptions.h index 56345ae36..9f05cdc21 100644 --- a/src/cpp/lpnamer/main/include/ProblemGenerationExeOptions.h +++ b/src/cpp/lpnamer/main/include/ProblemGenerationExeOptions.h @@ -4,8 +4,10 @@ #include #include "OptionsParser.h" +#include "ProblemGenerationOptions.h" -class ProblemGenerationExeOptions : public OptionsParser { +class ProblemGenerationExeOptions : public OptionsParser, + public ProblemGenerationOptions { private: std::filesystem::path xpansion_output_dir_; std::string master_formulation_; @@ -30,5 +32,14 @@ class ProblemGenerationExeOptions : public OptionsParser { std::filesystem::path WeightsFile() const { return weights_file_; } std::vector ActiveYears() const { return active_years_; } bool UnnamedProblems() const { return unnamed_problems_; } + + void Parse(unsigned int argc, const char *const *argv) override; + + [[nodiscard]] std::filesystem::path deduceXpansionDirIfEmpty( + std::filesystem::path xpansion_output_dir, + const std::filesystem::path& archive_path) const override; + [[nodiscard]] std::filesystem::path deduceArchivePathIfEmpty( + const std::filesystem::path& xpansion_output_dir, + const std::filesystem::path& archive_path) const override; }; #endif // ANTARES_XPANSION_SRC_CPP_LPNAMER_MAIN_INCLUDE_PROBLEMGENERATIONEXEOPTIONS_H diff --git a/src/cpp/lpnamer/main/include/ProblemGenerationOptions.h b/src/cpp/lpnamer/main/include/ProblemGenerationOptions.h new file mode 100644 index 000000000..338ead38d --- /dev/null +++ b/src/cpp/lpnamer/main/include/ProblemGenerationOptions.h @@ -0,0 +1,55 @@ +// +// Created by marechaljas on 27/10/23. +// + +#pragma once + +#include +#include + +class ProblemGenerationOptions { + public: + virtual ~ProblemGenerationOptions() = default; + [[nodiscard]] virtual std::filesystem::path XpansionOutputDir() const = 0; + [[nodiscard]] virtual std::string MasterFormulation() const = 0; + [[nodiscard]] virtual std::string AdditionalConstraintsFilename() const = 0; + [[nodiscard]] virtual std::filesystem::path ArchivePath() const = 0; + [[nodiscard]] virtual std::filesystem::path WeightsFile() const = 0; + [[nodiscard]] virtual std::vector ActiveYears() const = 0; + [[nodiscard]] virtual bool UnnamedProblems() const = 0; + [[nodiscard]] virtual std::filesystem::path deduceXpansionDirIfEmpty( + std::filesystem::path xpansion_output_dir, + const std::filesystem::path& archive_path) const = 0; + [[nodiscard]] virtual std::filesystem::path deduceArchivePathIfEmpty( + const std::filesystem::path& xpansion_output_dir, + const std::filesystem::path& archive_path) const = 0; + + class ConflictingParameters + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; + + public: + explicit ConflictingParameters(const std::string& err_message, + const std::string& log_location) + : XpansionError(err_message, log_location) {} + }; + + class MismatchedParameters + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; + + public: + explicit MismatchedParameters(const std::string& err_message, + const std::string& log_location) + : XpansionError(err_message, log_location) {} + }; + + class MissingParameters : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; + + public: + explicit MissingParameters(const std::string& err_message, + const std::string& log_location) + : XpansionError(err_message, log_location) {} + }; +}; \ No newline at end of file diff --git a/src/cpp/lpnamer/main/include/RunProblemGeneration.h b/src/cpp/lpnamer/main/include/RunProblemGeneration.h deleted file mode 100644 index d4eac444f..000000000 --- a/src/cpp/lpnamer/main/include/RunProblemGeneration.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef ANTARES_XPANSION_SRC_CPP_LPNAMER_MAIN_INCLUDE_RUNPROBLEMGENERATION_H -#define ANTARES_XPANSION_SRC_CPP_LPNAMER_MAIN_INCLUDE_RUNPROBLEMGENERATION_H -#include -#include - -#include "ProblemGenerationExeOptions.h" -#include "ProblemGenerationLogger.h" - -void RunProblemGeneration( - const std::filesystem::path& xpansion_output_dir, - const std::string& master_formulation, - const std::string& additionalConstraintFilename_l, - const std::filesystem::path& archive_path, - ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger, - const std::filesystem::path& log_file_path, - const std::filesystem::path& weights_file, bool unnamed_problems); -void ProcessWeights( - const std::filesystem::path& xpansion_output_dir, - const std::filesystem::path& antares_archive_path, - const std::filesystem::path& weights_file, - ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); -void ExtractUtilsFiles( - const std::filesystem::path& antares_archive_path, - const std::filesystem::path& xpansion_output_dir, - ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); - -#endif // ANTARES_XPANSION_SRC_CPP_LPNAMER_MAIN_INCLUDE_RUNPROBLEMGENERATION_H diff --git a/src/cpp/lpnamer/problem_modifier/CMakeLists.txt b/src/cpp/lpnamer/problem_modifier/CMakeLists.txt index 291188a21..46df73a58 100644 --- a/src/cpp/lpnamer/problem_modifier/CMakeLists.txt +++ b/src/cpp/lpnamer/problem_modifier/CMakeLists.txt @@ -52,7 +52,11 @@ add_library(lp_namer_problem_modifier STATIC ${CMAKE_CURRENT_SOURCE_DIR}/ZipProblemsProviderAdapter.h ${CMAKE_CURRENT_SOURCE_DIR}/ProblemVariablesFromProblemAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ProblemVariablesFromProblemAdapter.h - ) + FileProblemsProviderAdapter.cpp + FileProblemsProviderAdapter.h + FileProblemProviderAdapter.cpp + FileProblemProviderAdapter.h +) target_include_directories (lp_namer_problem_modifier PUBLIC diff --git a/src/cpp/lpnamer/problem_modifier/FileProblemProviderAdapter.cpp b/src/cpp/lpnamer/problem_modifier/FileProblemProviderAdapter.cpp new file mode 100644 index 000000000..3be4ba3e8 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/FileProblemProviderAdapter.cpp @@ -0,0 +1,22 @@ +// +// Created by marechaljas on 09/01/24. +// + +#include "FileProblemProviderAdapter.h" + +#include + +#include "multisolver_interface/SolverFactory.h" +std::shared_ptr FileProblemProviderAdapter::provide_problem( + const std::string& solver_name, + SolverLogManager& solver_log_manager) const { + SolverFactory factory; + auto in_prblm = std::make_shared( + factory.create_solver(solver_name, solver_log_manager)); + + in_prblm->read_prob_mps(lp_dir_.parent_path() / problem_name_); + return in_prblm; +} +FileProblemProviderAdapter::FileProblemProviderAdapter( + std::filesystem::path lp_dir, std::string problem_name) + : lp_dir_(std::move(lp_dir)), problem_name_(std::move(problem_name)) {} diff --git a/src/cpp/lpnamer/problem_modifier/FileProblemProviderAdapter.h b/src/cpp/lpnamer/problem_modifier/FileProblemProviderAdapter.h new file mode 100644 index 000000000..c3b221764 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/FileProblemProviderAdapter.h @@ -0,0 +1,18 @@ +// +// Created by marechaljas on 09/01/24. +// + +#pragma once + +#include "IProblemProviderPort.h" +class FileProblemProviderAdapter : public IProblemProviderPort { + public: + explicit FileProblemProviderAdapter(std::filesystem::path lp_dir, + std::string problem_name); + virtual ~FileProblemProviderAdapter() = default; + std::shared_ptr provide_problem( + const std::string& solver_name, + SolverLogManager& solver_log_manager) const override; + std::filesystem::path lp_dir_; + std::string problem_name_; +}; diff --git a/src/cpp/lpnamer/problem_modifier/FileProblemsProviderAdapter.cpp b/src/cpp/lpnamer/problem_modifier/FileProblemsProviderAdapter.cpp new file mode 100644 index 000000000..8bb83d967 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/FileProblemsProviderAdapter.cpp @@ -0,0 +1,29 @@ +// +// Created by marechaljas on 09/01/24. +// + +#include "FileProblemsProviderAdapter.h" + +#include + +#include "FileProblemProviderAdapter.h" + +std::vector> +FileProblemsProviderAdapter::provideProblems( + const std::string& solver_name, + SolverLogManager& solver_log_manager) const { + std::vector> problems(problem_names_.size()); + // Order is important. Problems need to be in the same order as names + std::transform(std::execution::par, + /*std::transform preserves order of element*/ + problem_names_.begin(), problem_names_.end(), problems.begin(), + [&](auto name) { + FileProblemProviderAdapter problem_provider(lp_dir_, name); + return problem_provider.provide_problem(solver_name, + solver_log_manager); + }); + return problems; +} +FileProblemsProviderAdapter::FileProblemsProviderAdapter( + std::filesystem::path lp_dir, std::vector problem_names) + : lp_dir_(lp_dir), problem_names_(problem_names) {} diff --git a/src/cpp/lpnamer/problem_modifier/FileProblemsProviderAdapter.h b/src/cpp/lpnamer/problem_modifier/FileProblemsProviderAdapter.h new file mode 100644 index 000000000..1631401cf --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/FileProblemsProviderAdapter.h @@ -0,0 +1,19 @@ +// +// Created by marechaljas on 09/01/24. +// + +#pragma once + +#include "IXpansionProblemsProvider.h" + +class FileProblemsProviderAdapter : public IXpansionProblemsProvider { + public: + FileProblemsProviderAdapter(std::filesystem::path lp_dir, + std::vector problem_names); + ~FileProblemsProviderAdapter() = default; + std::vector> provideProblems( + const std::string& solver_name, + SolverLogManager& solver_log_manager) const override; + std::filesystem::path lp_dir_; + std::vector problem_names_; +}; diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp index 2f741eae4..8750a71f5 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp @@ -60,6 +60,7 @@ void LinkProblemsGenerator::treat( } } auto const lp_mps_name = lpDir_ / problem->_name; + problem->_name = lp_mps_name.string(); writer->Write_problem(problem); } diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h index cb75ed2b0..431d6bf22 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h @@ -37,8 +37,8 @@ class LinkProblemsGenerator { _solver_name(std::move(solver_name)), lpDir_(lpDir), logger_(std::move(logger)), - solver_log_manager_(solver_log_manager), - rename_problems_(rename_problems) {} + rename_problems_(rename_problems), + solver_log_manager_(solver_log_manager) {} void treatloop(const std::filesystem::path& root, Couplings& couplings, const std::vector& mps_list, diff --git a/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp b/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp index 5ca2be7f9..ee26cd729 100644 --- a/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp +++ b/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp @@ -31,7 +31,7 @@ void ProblemVariablesFileAdapter::extract_variables( ProblemVariables ProblemVariablesFileAdapter::Provide() { ProblemVariables result; - extract_variables(lpdir_ / problem_data_._variables_txt, + extract_variables(lpdir_.parent_path() / problem_data_._variables_txt, result.variable_names, result.ntc_columns, result.direct_cost_columns, result.indirect_cost_columns); return result; diff --git a/src/cpp/multisolver_interface/SolverCbc.cpp b/src/cpp/multisolver_interface/SolverCbc.cpp index 6cb19eec9..058968170 100644 --- a/src/cpp/multisolver_interface/SolverCbc.cpp +++ b/src/cpp/multisolver_interface/SolverCbc.cpp @@ -120,12 +120,16 @@ void SolverCbc::write_prob_mps(const std::filesystem::path &filename) { } } - writer.setMpsData( - *(_clp_inner_solver.getMatrixByCol()), _clp_inner_solver.getInfinity(), - _clp_inner_solver.getColLower(), _clp_inner_solver.getColUpper(), - _clp_inner_solver.getObjCoefficients(), - hasInteger ? integrality : nullptr, _clp_inner_solver.getRowLower(), - _clp_inner_solver.getRowUpper(), colNames, rowNames); + { + auto mcol = _clp_inner_solver.getMatrixByCol(); + auto col = *(_clp_inner_solver.getMatrixByCol()); + auto infinity = _clp_inner_solver.getInfinity(); + writer.setMpsData( + col, infinity, _clp_inner_solver.getColLower(), + _clp_inner_solver.getColUpper(), _clp_inner_solver.getObjCoefficients(), + hasInteger ? integrality : nullptr, _clp_inner_solver.getRowLower(), + _clp_inner_solver.getRowUpper(), colNames, rowNames); + } std::string probName = ""; _clp_inner_solver.getStrParam(OsiProbName, probName); diff --git a/src/python/antares_xpansion/full_run_driver.py b/src/python/antares_xpansion/full_run_driver.py index 725e02cce..7a978fd9f 100644 --- a/src/python/antares_xpansion/full_run_driver.py +++ b/src/python/antares_xpansion/full_run_driver.py @@ -64,10 +64,11 @@ def run(self): os.chdir(lp_path) self.logger.info(f"Current directory is now: {os.getcwd()}") + self.logger.info(f"Command is {self.full_command()}") + print(self.full_command()) ret = subprocess.run( self.full_command(), shell=False, stdout=sys.stdout, stderr=sys.stderr, encoding='utf-8') - if ret.returncode != 0: raise FullRunDriver.FullRunExecutionError( f"ERROR: exited {self.full_exe} with status {ret.returncode}" diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index 1df3e4e3b..1c76cded8 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -118,8 +118,7 @@ def create_lp_dir(self): def lp_namer_options(self): is_relaxed = 'relaxed' if self.is_relaxed else 'integer' - ret = ["-o", str(self.xpansion_output_dir), "-a", - str(self.output_path), "-f", is_relaxed] + ret = ["-a", str(self.output_path), "-f", is_relaxed] if self.weight_file_name_for_lp: ret.extend(["-w", str(self.user_weights_file_path)]) diff --git a/tests/cpp/full_run/FullRunTest.cpp b/tests/cpp/full_run/FullRunTest.cpp index 3b370d9c3..f7f12ed2a 100644 --- a/tests/cpp/full_run/FullRunTest.cpp +++ b/tests/cpp/full_run/FullRunTest.cpp @@ -5,35 +5,32 @@ namespace po = boost::program_options; -class FullRunOptionsParserTest : public ::testing::Test { +class FullRunOptionsParserTest + : public ::testing::TestWithParam> { protected: FullRunOptionsParser full_run_options_options_parser_; }; -class FullRunOptionsParserTestParameterizedMethod +class FullRunOptionsParserTestFullPath : public FullRunOptionsParserTest {}; + +class FullRunOptionsParserTestParameterizedMethod_output : public ::testing::TestWithParam> { protected: FullRunOptionsParser full_run_options_options_parser_; }; -TEST_F(FullRunOptionsParserTest, ThatArchiveFileIsRequired) { - const char argv0[] = "full_run.exe"; - std::vector ppargv = {argv0}; - try { - full_run_options_options_parser_.Parse(1, ppargv.data()); - } catch (const std::exception& e) { - EXPECT_EQ(e.what(), - std::string("the option '--archive' is required but missing")); - } +auto params() { + return ::testing::ValuesIn(std::vector>{ + {"full_run.exe", "--archive", "something"}, + {"full_run.exe", "--output", "something"}, + }); } -TEST_F(FullRunOptionsParserTest, ThatBendersOptionFileIsRequired) { - const char argv0[] = "full_run.exe"; - const char argv1[] = "--archive"; - const char argv2[] = "something"; - const char argv3[] = "--output"; - const char argv4[] = "something"; - std::vector ppargv = {argv0, argv1, argv2, argv3, argv4}; +TEST_P(FullRunOptionsParserTest, ThatBendersOptionFileIsRequired) { + auto params = GetParam(); + std::vector pargs; + std::ranges::transform(params, std::back_inserter(pargs), + [](const std::string& s) { return s.data(); }); try { - full_run_options_options_parser_.Parse(5, ppargv.data()); + full_run_options_options_parser_.Parse(params.size(), pargs.data()); } catch (const std::exception& e) { EXPECT_EQ( e.what(), @@ -41,39 +38,56 @@ TEST_F(FullRunOptionsParserTest, ThatBendersOptionFileIsRequired) { } } -TEST_F(FullRunOptionsParserTest, ThatSolutionOptionIsRequired) { - const char argv0[] = "full_run.exe"; - const char argv1[] = "--archive"; - const char argv2[] = "something"; - const char argv3[] = "--output"; - const char argv4[] = "something"; +TEST_P(FullRunOptionsParserTest, ThatMethodOptionIsRequired) { + auto params = GetParam(); + std::vector pargs; + std::ranges::transform(params, std::back_inserter(pargs), + [](const std::string& s) { return s.data(); }); + const char argv5[] = "--benders_options"; + const char argv6[] = "something"; + pargs.push_back(argv5); + pargs.push_back(argv6); + try { + full_run_options_options_parser_.Parse(pargs.size(), pargs.data()); + } catch (const std::exception& e) { + EXPECT_EQ(e.what(), + std::string("the option '--method' is required but missing")); + } +} + +TEST_P(FullRunOptionsParserTest, ThatSolutionOptionIsRequired) { + auto params = GetParam(); + std::vector pargs; + std::ranges::transform(params, std::back_inserter(pargs), + [](const std::string& s) { return s.data(); }); const char argv5[] = "--benders_options"; const char argv6[] = "something"; - std::vector ppargv = {argv0, argv1, argv2, argv3, - argv4, argv5, argv6}; + pargs.push_back(argv5); + pargs.push_back(argv6); try { - full_run_options_options_parser_.Parse(7, ppargv.data()); + full_run_options_options_parser_.Parse(pargs.size(), pargs.data()); } catch (const std::exception& e) { EXPECT_EQ(e.what(), std::string("the option '--solution' is required but missing")); } } +INSTANTIATE_TEST_SUITE_P(args, FullRunOptionsParserTest, params()); -TEST_F(FullRunOptionsParserTest, OptionsParsing) { - const char argv0[] = "full_run.exe"; - const char argv1[] = "--archive"; - const char argv2[] = "/path/to/output.zip"; - const char argv3[] = "--output"; - const char argv4[] = "/path/to/output"; +TEST_P(FullRunOptionsParserTestFullPath, OptionsParsing) { + auto params = GetParam(); + std::vector pargs; + std::ranges::transform(params, std::back_inserter(pargs), + [](const std::string& s) { return s.data(); }); const char argv5[] = "--benders_options"; - const char argv6[] = "options.json"; + const char argv6[] = "something"; + pargs.push_back(argv5); + pargs.push_back(argv6); const char argv7[] = "-s"; const char argv8[] = "/path/to/solution.json"; - std::vector ppargv = {argv0, argv1, argv2, argv3, argv4, - argv5, argv6, argv7, argv8}; - full_run_options_options_parser_.Parse(9, ppargv.data()); - ASSERT_EQ(full_run_options_options_parser_.ArchivePath(), - std::filesystem::path(argv2)); + pargs.push_back(argv7); + pargs.push_back(argv8); + + full_run_options_options_parser_.Parse(pargs.size(), pargs.data()); ASSERT_EQ(full_run_options_options_parser_.BendersOptionsFile(), std::filesystem::path(argv6)); ASSERT_EQ(full_run_options_options_parser_.SolutionFile(), diff --git a/tests/cpp/lp_namer/CMakeLists.txt b/tests/cpp/lp_namer/CMakeLists.txt index 473efc505..989289c3d 100644 --- a/tests/cpp/lp_namer/CMakeLists.txt +++ b/tests/cpp/lp_namer/CMakeLists.txt @@ -36,7 +36,6 @@ add_executable (lp_namer_tests WeightsFileReaderTest.cpp LpFilesExtractorTest.cpp MpsTxtWriterTest.cpp - ProblemGenerationTest.cpp ) target_link_libraries (lp_namer_tests PRIVATE diff --git a/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp b/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp index 4b8316237..2340bfc17 100644 --- a/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp +++ b/tests/cpp/lp_namer/ProblemGenerationExeOptionsTest.cpp @@ -1,60 +1,131 @@ +#include +#include + +#include "ProblemGeneration.h" #include "ProblemGenerationExeOptions.h" #include "gtest/gtest.h" namespace po = boost::program_options; class ProblemGenerationExeOptionsTest : public ::testing::Test { - protected: + public: ProblemGenerationExeOptions problem_generation_options_parser_; -}; -TEST_F(ProblemGenerationExeOptionsTest, WithOutArchiveOption) { - char argv = 'c'; - char* pargv = &argv; - char** ppargv = &pargv; - try { - problem_generation_options_parser_.Parse(1, ppargv); - } catch (const std::exception& e) { - EXPECT_EQ(e.what(), - std::string("the option '--archive' is required but missing")); + auto parseOptions(std::convertible_to auto&&... s) { + const char argv0[] = "lp_namer.exe"; + std::vector ppargv; + ppargv.push_back(argv0); + for (auto v : std::initializer_list{s...}) { + ppargv.push_back(v.data()); + } + return problem_generation_options_parser_.Parse(ppargv.size(), + ppargv.data()); } -} -TEST_F(ProblemGenerationExeOptionsTest, WithOutOutputOption) { - const char argv0[] = "lp_namer.exe"; - const char argv1[] = "--archive"; - const char argv2[] = "something"; - std::vector ppargv = {argv0, argv1, argv2}; - try { - problem_generation_options_parser_.Parse(3, ppargv.data()); - } catch (const std::exception& e) { - EXPECT_EQ(e.what(), - std::string("the option '--output' is required but missing")); +}; + +class ProblemGenerationSpyAndMock : public ProblemGeneration { + public: + virtual ~ProblemGenerationSpyAndMock() = default; + explicit ProblemGenerationSpyAndMock(ProblemGenerationExeOptions& options) + : ProblemGeneration(options) {} + void RunProblemGeneration( + const std::filesystem::path& xpansion_output_dir, + const std::string& master_formulation, + const std::string& additionalConstraintFilename_l, + const std::filesystem::path& archive_path, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger, + const std::filesystem::path& log_file_path, + const std::filesystem::path& weights_file, + bool unnamed_problems) override { + xpansion_output_dir_ = xpansion_output_dir; + master_formulation_ = master_formulation; + additionalConstraintFilename_l_ = additionalConstraintFilename_l; + archive_path_ = archive_path; + log_file_path_ = log_file_path; + weights_file_ = weights_file; + unnamed_problems_ = unnamed_problems; } + std::filesystem::path xpansion_output_dir_; + std::string master_formulation_; + std::string additionalConstraintFilename_l_; + std::filesystem::path archive_path_; + std::filesystem::path log_file_path_; + std::filesystem::path weights_file_; + bool unnamed_problems_ = false; +}; + +TEST_F(ProblemGenerationExeOptionsTest, WithoutOutputOption) { + EXPECT_NO_THROW(parseOptions("--archive", "something");); } +// OK TEST_F(ProblemGenerationExeOptionsTest, MasterFormulationDefaultValue) { - const char argv0[] = "lp.exe "; - const char argv1[] = "--archive"; - const char argv2[] = "something"; - const char argv3[] = "--output"; - const char argv4[] = "something"; - std::vector ppargv = {argv0, argv1, argv2, argv3, argv4}; - problem_generation_options_parser_.Parse(5, ppargv.data()); + parseOptions("--output", "something"); ASSERT_EQ(problem_generation_options_parser_.MasterFormulation(), std::string("relaxed")); } -// TEST_F(ProblemGenerationExeOptionsTest, -// AdditionalConstraintsFilenameIsNotEmptyIfGiven) { -// const char argv0[] = "lp.exe "; -// const char argv1[] = "--output"; -// const char argv2[] = "something"; -// const char argv3[] = "-e"; -// std::vector ppargv = {argv0, argv1, argv2, argv3}; -// try { -// problem_generation_options_parser_.Parse(4, ppargv.data()); -// } catch (const std::exception& e) { -// EXPECT_EQ(e.what(), -// std::string("the required argument for option -// '--exclusion-files' is missing")); -// } -// } + +TEST_F(ProblemGenerationExeOptionsTest, + OutputAndArchiveParameters_mutually_exclusives) { + auto test_root = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + auto archive = std::string(tmpnam(nullptr)) + "study.zip"; + auto output_path = test_root / "study-Xpansion"; + + EXPECT_THROW( + parseOptions("--archive", archive, "--output", output_path.string()), + ProblemGenerationOptions::ConflictingParameters); + + ProblemGenerationSpyAndMock pbg(problem_generation_options_parser_); + pbg.updateProblems(); + + EXPECT_EQ(pbg.archive_path_, archive); + EXPECT_EQ(pbg.xpansion_output_dir_, output_path); + EXPECT_TRUE(std::filesystem::exists(output_path)); + EXPECT_TRUE(std::filesystem::exists(output_path / "lp")); +} + +TEST_F(ProblemGenerationExeOptionsTest, + OutputAndArchiveParameters_deduceOuputFromArchive) { + auto test_root = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + auto archive = test_root / "study.zip"; + auto output_path = test_root / "study-Xpansion"; + + parseOptions("--archive", archive.string()); + + ProblemGenerationSpyAndMock pbg(problem_generation_options_parser_); + pbg.updateProblems(); + + EXPECT_TRUE(problem_generation_options_parser_.XpansionOutputDir().empty()); + EXPECT_EQ(pbg.archive_path_, archive); + EXPECT_EQ(pbg.xpansion_output_dir_, output_path); + EXPECT_TRUE(std::filesystem::exists(output_path)); + EXPECT_TRUE(std::filesystem::exists(output_path / "lp")); +} + +TEST_F(ProblemGenerationExeOptionsTest, use_only_output_option) { + auto test_root = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + auto archive = test_root / "study.zip"; + auto output_path = test_root / "study-Xpansion"; + + parseOptions("--output", output_path.string()); + + ProblemGenerationSpyAndMock pbg(problem_generation_options_parser_); + pbg.updateProblems(); + + EXPECT_TRUE(problem_generation_options_parser_.ArchivePath().empty()); + EXPECT_TRUE(pbg.archive_path_.empty()); + EXPECT_EQ(pbg.xpansion_output_dir_, output_path); + EXPECT_TRUE(std::filesystem::exists(output_path)); + EXPECT_TRUE(std::filesystem::exists(output_path / "lp")); +} + +TEST_F(ProblemGenerationExeOptionsTest, + OutputAndArchiveParameters_ErrorIfBothArchiveAndOutputAreMissing) { + auto test_root = + std::filesystem::temp_directory_path() / std::tmpnam(nullptr); + + EXPECT_THROW(parseOptions(), ProblemGenerationOptions::MissingParameters); +} \ No newline at end of file diff --git a/tests/cpp/lp_namer/ProblemGenerationTest.cpp b/tests/cpp/lp_namer/ProblemGenerationTest.cpp deleted file mode 100644 index 2e8506965..000000000 --- a/tests/cpp/lp_namer/ProblemGenerationTest.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by marechaljas on 10/10/23. -// - -#include - -#include "ArchiveIO.h" -#include "LoggerBuilder.h" -#include "RunProblemGeneration.h" -#include "gtest/gtest.h" - -TEST(InitializationTest, FoldersAreEmpty) { - auto workingDirectory = - std::filesystem::temp_directory_path() / std::tmpnam(nullptr); - auto simulationDirectory = workingDirectory / "output" / "simulation"; - auto logger = emptyLogger(); - std::filesystem::create_directories(simulationDirectory); - - auto xpansionDirectory = - (simulationDirectory.parent_path() / - (simulationDirectory.filename().string() + "-Xpansion")); - std::filesystem::create_directories(xpansionDirectory); - std::ofstream emptyfile(xpansionDirectory / "garbage.txt"); - - EXPECT_TRUE(std::filesystem::exists(xpansionDirectory / "garbage.txt")); - EXPECT_THROW( - RunProblemGeneration(simulationDirectory, "integer", "", "", logger, - simulationDirectory / "logs.txt", "", false), - ArchiveIOGeneralException); - - EXPECT_FALSE(std::filesystem::exists(simulationDirectory / "garbage.txt")); -} \ No newline at end of file diff --git a/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py b/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py index 016e70d8a..36cc758b5 100644 --- a/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py +++ b/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py @@ -1,13 +1,13 @@ import filecmp -import glob import os import shutil import subprocess -import pytest import zipfile from pathlib import Path -MPS_ZIP = "AntaresStudyOutput.zip" +import pytest + +MPS_ZIP = "economy.zip" DATA_TEST = Path("../../../data_test/") DATA_TEST_INTEGER = DATA_TEST / "tests_lpnamer" / "tests_integer" DATA_TEST_RELAXED = DATA_TEST / "tests_lpnamer" / "tests_relaxed" @@ -30,65 +30,98 @@ TEST_LP_RELAXED_02 = DATA_TEST_RELAXED / "SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed" / "output" \ / "economy" test_data = [ - (TEST_LP_INTEGER_01, "integer"), - (TEST_LP_INTEGER_02, "integer"), - (TEST_LP_RELAXED_01, "relaxed"), - (TEST_LP_RELAXED_02, "relaxed") + (TEST_LP_INTEGER_01, "integer", True), + (TEST_LP_INTEGER_01, "integer", False), + (TEST_LP_INTEGER_02, "integer", True), + (TEST_LP_INTEGER_02, "integer", False), + (TEST_LP_RELAXED_01, "relaxed", True), + (TEST_LP_RELAXED_01, "relaxed", False), + (TEST_LP_RELAXED_02, "relaxed", True), + (TEST_LP_RELAXED_02, "relaxed", False), ] test_data_multiple_candidates = [ - (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB, "integer"), - (TEST_LP_INTEGER_MULTIPLE_CANDIDATES, "integer"), - (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_HURDLES, "integer"), - (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_NULL_PROFILE, "integer") + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB, "integer", True), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB, "integer", False), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES, "integer", True), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES, "integer", False), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_HURDLES, "integer", True), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_HURDLES, "integer", False), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_NULL_PROFILE, "integer", True), + (TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_NULL_PROFILE, "integer", False), ] @pytest.fixture def setup_and_teardown_lp_directory(request): test_dir = request.getfixturevalue('test_dir') - lp_dir = test_dir / "lp" + lp_dir = test_dir.parent / (test_dir.stem + "-Xpansion") / "lp" if Path(lp_dir).is_dir(): shutil.rmtree(lp_dir) - Path(lp_dir).mkdir(exist_ok=True) + Path(lp_dir).mkdir(parents=True, exist_ok=True) list_files = list(Path(test_dir).glob("*.mps")) list_files.extend(list(Path(test_dir).glob("variables*.txt"))) list_files.extend(list(Path(test_dir).glob("area*.txt"))) list_files.extend(list(Path(test_dir).glob("interco*.txt"))) - with zipfile.ZipFile(lp_dir/MPS_ZIP, "w") as write_mps_zip: + with zipfile.ZipFile(test_dir.parent / MPS_ZIP, "w") as write_mps_zip: for file in list_files: write_mps_zip.write( file, file.name, compress_type=zipfile.ZIP_DEFLATED) yield -@ pytest.mark.parametrize("test_dir,master_mode", test_data) -def test_lp_directory_files(install_dir, test_dir, master_mode, setup_and_teardown_lp_directory): +@pytest.mark.parametrize("test_dir,master_mode, use_archive", test_data) +def test_lp_directory_files(install_dir, test_dir, master_mode, use_archive, setup_and_teardown_lp_directory): # given - launch_and_compare_lp_with_reference(install_dir, master_mode, test_dir) + if use_archive: + launch_and_compare_lp_with_reference_archive(install_dir, master_mode, test_dir) + else: + launch_and_compare_lp_with_reference_output(install_dir, master_mode, test_dir) -@ pytest.mark.parametrize("test_dir,master_mode", test_data_multiple_candidates) -def test_lp_multiple_candidates(install_dir, test_dir, master_mode, setup_and_teardown_lp_directory): - launch_and_compare_lp_with_reference(install_dir, master_mode, test_dir) +@pytest.mark.parametrize("test_dir,master_mode, use_archive", test_data_multiple_candidates) +def test_lp_multiple_candidates(install_dir, test_dir, master_mode, use_archive, setup_and_teardown_lp_directory): + if use_archive: + launch_and_compare_lp_with_reference_archive(install_dir, master_mode, test_dir) + else: + launch_and_compare_lp_with_reference_output(install_dir, master_mode, test_dir) -def launch_and_compare_lp_with_reference(install_dir, master_mode, test_dir): +def launch_and_compare_lp_with_reference_output(install_dir, master_mode, test_dir): old_path = os.getcwd() reference_lp_dir = test_dir / "reference_lp" lp_dir = test_dir / "lp" lp_namer_exe = Path(install_dir) / "lp_namer" - zip_path = (lp_dir/MPS_ZIP).resolve() study_dir = test_dir.resolve() os.chdir(test_dir.parent) - launch_command = [str(lp_namer_exe), "-o", str(study_dir), "-a", str(zip_path), + launch_command = [str(lp_namer_exe), "-o", str(study_dir), "-e", "contraintes.txt", "-f", master_mode, "--unnamed-problems"] # when returned_l = subprocess.run(launch_command, shell=False) # then os.chdir(old_path) + then(lp_dir, old_path, reference_lp_dir, returned_l) + + +def launch_and_compare_lp_with_reference_archive(install_dir, master_mode, test_dir): + old_path = os.getcwd() + reference_lp_dir = test_dir / "reference_lp" + lp_dir = test_dir.parent / (test_dir.stem + "-Xpansion") / "lp" + lp_namer_exe = Path(install_dir) / "lp_namer" + zip_path = (test_dir.parent / MPS_ZIP).resolve() + os.chdir(test_dir.parent) + launch_command = [str(lp_namer_exe), "-a", str(zip_path), + "-e", "contraintes.txt", "-f", master_mode, "--unnamed-problems"] + # when + returned_l = subprocess.run(launch_command, shell=False) + # then + then(lp_dir, old_path, reference_lp_dir, returned_l) + + +def then(lp_dir, old_path, reference_lp_dir, returned_l): + os.chdir(old_path) files_to_compare = os.listdir(reference_lp_dir) match, mismatch, errors = filecmp.cmpfiles( reference_lp_dir, lp_dir, files_to_compare) diff --git a/tests/python/test_full_run_driver.py b/tests/python/test_full_run_driver.py index 55e7fc926..30c742f37 100644 --- a/tests/python/test_full_run_driver.py +++ b/tests/python/test_full_run_driver.py @@ -1,10 +1,10 @@ +from pathlib import Path + +from antares_xpansion.benders_driver import BendersDriver from antares_xpansion.full_run_driver import FullRunDriver from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver, ProblemGeneratorData -from antares_xpansion.benders_driver import BendersDriver + from test_problem_generator_driver import get_zipped_output -from unittest.mock import patch -import pytest -from pathlib import Path SUBPROCESS_RUN = "antares_xpansion.full_run_driver.subprocess.run" @@ -46,7 +46,7 @@ def test_sequential_command(self, tmp_path): xpansion_output_dir = output_path.parent / \ (output_path.stem+"-Xpansion") expected_command = [self.full_run_exe, "--benders_options", self.benders_driver_options_file, - "-s", str(json_file_path), "-a", str(output_path), "-o", str(xpansion_output_dir), "-f", "integer", "-e", self.pb_gen_data.additional_constraints] + "-s", str(json_file_path), "-a", str(output_path), "-f", "integer", "-e", self.pb_gen_data.additional_constraints] command = full_run_driver.full_command() assert len(expected_command) == len(command) @@ -80,7 +80,7 @@ def test_mpi_command(self, tmp_path): xpansion_output_dir = output_path.parent / \ (output_path.stem+"-Xpansion") expected_command = [benders_driver.MPI_LAUNCHER, "-n", str(benders_n_mpi), self.full_run_exe, "--benders_options", self.benders_driver_options_file, - "-s", str(json_file_path), "-a", str(output_path), "-o", str(xpansion_output_dir), "-f", "integer", "-e", self.pb_gen_data.additional_constraints] + "-s", str(json_file_path), "-a", str(output_path), "-f", "integer", "-e", self.pb_gen_data.additional_constraints] command = full_run_driver.full_command()