Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create lp directories in cpp #710

Merged
merged 13 commits into from
Nov 2, 2023
Merged
332 changes: 176 additions & 156 deletions conception/SequenceDiagrams/FileManipulation/diagram.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 2 additions & 0 deletions conception/SequenceDiagrams/FileManipulation/diagram.txt
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ create ZipProblemProviderAdapter
create FilesMapper
full_run->ProblemGeneration:RunProblemGeneration
activate ProblemGeneration
ProblemGeneration->ProblemGeneration:CreateDirectories
note over ProblemGeneration: Create -Xpansion and -Xpansion/lp directories
ProblemGeneration->ProblemGeneration:ExtractUtilsFiles

ProblemGeneration->LpFilesExtractor:ExtractFiles
Expand Down
14 changes: 13 additions & 1 deletion src/cpp/exe/lpnamer/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,18 @@
#include "ProblemGenerationLogger.h"
#include "RunProblemGeneration.h"

static const std::string LP_DIRNAME = "lp";

void CreateDirectories(const std::filesystem::path& output_path) {
if (!std::filesystem::exists(output_path)) {
std::filesystem::create_directories(output_path);
}
auto lp_path = output_path / LP_DIRNAME;
if (!std::filesystem::exists(lp_path)) {
std::filesystem::create_directories(lp_path);
}
}

int main(int argc, char** argv) {
try {
auto options_parser = ProblemGenerationExeOptions();
Expand All @@ -16,9 +28,9 @@ int main(int argc, char** argv) {
const auto log_file_path =
xpansion_output_dir / "lp" / "ProblemGenerationLog.txt";

CreateDirectories(xpansion_output_dir);
auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout,
"Problem Generation");
auto& loggerRef = (*logger);

auto master_formulation = options_parser.MasterFormulation();
auto additionalConstraintFilename_l =
Expand Down
5 changes: 3 additions & 2 deletions src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ ProblemGenerationFileLogger::ProblemGenerationFileLogger(
const std::filesystem::path& logFilePath)
: logFilePath_(logFilePath) {
SetType(LogUtils::LOGGERTYPE::FILE);
logFile_.open(logFilePath_, std::ofstream::out | std::ofstream::app);
auto logPath = std::filesystem::absolute(logFilePath_);
logFile_.open(logPath, std::ofstream::out | std::ofstream::app);
if (logFile_.fail()) {
std::cerr << "ProblemGenerationFileLogger: Invalid file ("
<< logFilePath_.string() << ") passed as parameter" << std::endl;
<< logPath.string() << ") passed as parameter" << std::endl;
}
}
void ProblemGenerationFileLogger::DisplayMessage(const std::string& message) {
Expand Down
106 changes: 27 additions & 79 deletions src/cpp/lpnamer/main/RunProblemGeneration.cpp
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@

#include "RunProblemGeneration.h"

#include <boost/archive/text_iarchive.hpp>
#include <execution>
#include <iostream>

#include "ActiveLinks.h"
#include "AdditionalConstraints.h"
#include "Clock.h"
#include "GeneralDataReader.h"
#include "LauncherHelpers.h"
#include "LinkProblemsGenerator.h"
Expand All @@ -17,14 +15,12 @@
#include "MasterGeneration.h"
#include "MasterProblemBuilder.h"
#include "MpsTxtWriter.h"
#include "ProblemGenerationExeOptions.h"
#include "ProblemVariablesFromProblemAdapter.h"
#include "ProblemVariablesZipAdapter.h"
#include "StringManip.h"
#include "Timer.h"
#include "WeightsFileReader.h"
#include "WeightsFileWriter.h"
#include "XpansionProblemsFromAntaresProvider.h"
#include "ZipProblemsProviderAdapter.h"
#include "config.h"

Expand Down Expand Up @@ -59,6 +55,7 @@ struct Version {
int major;
int minor;
};

std::shared_ptr<ArchiveReader> InstantiateZipReader(
const std::filesystem::path& antares_archive_path);
void ProcessWeights(
Expand Down Expand Up @@ -128,6 +125,7 @@ std::vector<std::shared_ptr<Problem>> getXpansionProblems(
problem_names);
return adapter->provideProblems(solver_name, solver_log_manager);
}

void RunProblemGeneration(
const std::filesystem::path& xpansion_output_dir,
const std::string& master_formulation,
Expand Down Expand Up @@ -169,14 +167,10 @@ void RunProblemGeneration(
auto files_mapper = FilesMapper(antares_archive_path);
auto mpsList = files_mapper.MpsAndVariablesFilesVect();

bool use_zip_implementation = true;
bool use_file_implementation = false;

auto solver_log_manager = SolverLogManager(log_file_path);
Couplings couplings;
LinkProblemsGenerator linkProblemsGenerator(
lpDir_, links, solver_name, logger, solver_log_manager, rename_problems);
if (use_zip_implementation) {
std::shared_ptr<ArchiveReader> reader =
InstantiateZipReader(antares_archive_path);

Expand All @@ -185,79 +179,33 @@ void RunProblemGeneration(
getXpansionProblems(solver_log_manager, solver_name, mpsList, lpDir_,
reader);

std::vector<std::pair<std::shared_ptr<Problem>, ProblemData>>
problems_and_data;
for (int i = 0; i < xpansion_problems.size(); ++i) {
xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps;
problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i));
}
auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
std::for_each(
std::execution::par, problems_and_data.begin(), problems_and_data.end(),
[&](const auto& problem_and_data) {
const auto& [problem, data] = problem_and_data;
std::shared_ptr<IProblemVariablesProviderPort> variables_provider;
if (rename_problems) {
variables_provider = std::make_shared<ProblemVariablesZipAdapter>(
reader, data, links, logger);
} else {
variables_provider =
std::make_shared<ProblemVariablesFromProblemAdapter>(
problem, links, logger);
}
linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(),
variables_provider.get(), mps_file_writer.get());
});

reader->Close();
reader->Delete();
} else if (use_file_implementation) {
/* Main stuff */
auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
linkProblemsGenerator.treatloop(xpansion_output_dir, couplings, mpsList,
std::vector<std::pair<std::shared_ptr<Problem>, ProblemData>>
problems_and_data;
for (int i = 0; i < xpansion_problems.size(); ++i) {
xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps;
problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i));
}
auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
std::for_each(
std::execution::par, problems_and_data.begin(), problems_and_data.end(),
[&](const auto& problem_and_data) {
const auto& [problem, data] = problem_and_data;
std::shared_ptr<IProblemVariablesProviderPort> variables_provider;
if (rename_problems) {
variables_provider = std::make_shared<ProblemVariablesZipAdapter>(
reader, data, links, logger);
} else {
variables_provider =
std::make_shared<ProblemVariablesFromProblemAdapter>(
problem, links, logger);
}
linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(),
variables_provider.get(),
mps_file_writer.get());
} else {
std::filesystem::path path =
xpansion_output_dir.parent_path().parent_path() /
"fichierDeSerialisation";
std::ifstream ifs(xpansion_output_dir.parent_path().parent_path() /
"fichierDeSerialisation");
boost::archive::text_iarchive ia(ifs);

LpsFromAntares lps;
ia >> lps;
lps._constant->Mdeb.push_back(lps._constant->NombreDeCoefficients);

XpansionProblemsFromAntaresProvider adapter(lps);
auto xpansion_problems =
adapter.provideProblems(solver_name, solver_log_manager);
std::vector<std::pair<std::shared_ptr<Problem>, ProblemData>>
problems_and_data;
for (int i = 0; i < xpansion_problems.size(); ++i) {
xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps;
problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i));
}

auto reader = InstantiateZipReader(antares_archive_path);
auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
});

std::for_each(
std::execution::par, problems_and_data.begin(), problems_and_data.end(),
[&](const auto& problem_and_data) {
const auto& [problem, data] = problem_and_data;
std::shared_ptr<IProblemVariablesProviderPort> variables_provider;
if (rename_problems) {
variables_provider = std::make_shared<ProblemVariablesZipAdapter>(
reader, data, links, logger);
} else {
variables_provider =
std::make_shared<ProblemVariablesFromProblemAdapter>(
problem, links, logger);
}
linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(),
variables_provider.get(), mps_file_writer.get());
});
}
reader->Close();
reader->Delete();

MasterGeneration master_generation(
xpansion_output_dir, links, additionalConstraints, couplings,
Expand Down
10 changes: 5 additions & 5 deletions src/python/antares_xpansion/full_run_driver.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@

from typing import List
import os
import subprocess
import sys
from pathlib import Path
from typing import List

from antares_xpansion.benders_driver import BendersDriver
from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver
from antares_xpansion.logger import step_logger
from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver
from antares_xpansion.study_output_cleaner import StudyOutputCleaner
import os
from pathlib import Path


class FullRunDriver:
Expand Down
2 changes: 1 addition & 1 deletion src/python/antares_xpansion/problem_generator_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def set_output_path(self, output_path):
if output_path.exists():
self._output_path = output_path
self.xpansion_output_dir = output_path.parent / \
(output_path.stem+"-Xpansion")
(output_path.stem + "-Xpansion")
JasonMarechal25 marked this conversation as resolved.
Show resolved Hide resolved
if self.xpansion_output_dir.exists():
shutil.rmtree(self.xpansion_output_dir)
os.makedirs(self.xpansion_output_dir)
Expand Down
3 changes: 2 additions & 1 deletion tests/cpp/lp_namer/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ add_executable (lp_namer_tests
WeightsFileReaderTest.cpp
LpFilesExtractorTest.cpp
MpsTxtWriterTest
)
ProblemGenerationTest.cpp
)

target_link_libraries (lp_namer_tests PRIVATE
GTest::Main
Expand Down
32 changes: 32 additions & 0 deletions tests/cpp/lp_namer/ProblemGenerationTest.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
//
// Created by marechaljas on 10/10/23.
//

#include <filesystem>

#include "ArchiveIO.h"
#include "LoggerBuilder.h"
#include "RunProblemGeneration.h"
#include "gtest/gtest.h"

TEST(InitializationTest, FoldersAreEmpty) {
auto workingDirectory =
std::filesystem::temp_directory_path() / std::tmpnam(nullptr);
auto simulationDirectory = workingDirectory / "output" / "simulation";
auto logger = emptyLogger();
std::filesystem::create_directories(simulationDirectory);

auto xpansionDirectory =
(simulationDirectory.parent_path() /
(simulationDirectory.filename().string() + "-Xpansion"));
std::filesystem::create_directories(xpansionDirectory);
std::ofstream emptyfile(xpansionDirectory / "garbage.txt");

EXPECT_TRUE(std::filesystem::exists(xpansionDirectory / "garbage.txt"));
EXPECT_THROW(
RunProblemGeneration(simulationDirectory, "integer", "", "", logger,
simulationDirectory / "logs.txt", "", false),
ArchiveIOGeneralException);

EXPECT_FALSE(std::filesystem::exists(simulationDirectory / "garbage.txt"));
}
14 changes: 4 additions & 10 deletions tests/python/test_problem_generator_driver.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,10 @@
from typing import Text
import pytest
import os
from pathlib import Path
from datetime import date, datetime
from unittest.mock import ANY, mock_open, patch
import shutil
from datetime import date, datetime
from pathlib import Path
from unittest.mock import patch

from file_creation import _create_weight_file
import pytest
from antares_xpansion.problem_generator_driver import ProblemGeneratorData, ProblemGeneratorDriver
from antares_xpansion.antares_driver import AntaresDriver
from antares_xpansion.xpansion_study_reader import XpansionStudyReader
from antares_xpansion.general_data_reader import GeneralDataIniReader

SUBPROCESS_RUN = "antares_xpansion.problem_generator_driver.subprocess.run"
zipfile_ZipFile = "antares_xpansion.problem_generator_driver.zipfile.ZipFile"
Expand Down
Loading