diff --git a/.github/workflows/build_centos7.yml b/.github/workflows/build_centos7.yml index 2fa536992..022dd5029 100644 --- a/.github/workflows/build_centos7.yml +++ b/.github/workflows/build_centos7.yml @@ -176,10 +176,10 @@ jobs: cmake --build _build --config Release -j$(nproc) - - name: Run cucumber on outer_loop tests + - name: Tests with Cucumber uses: ./.github/workflows/cucumber-tests with: - feature: "features/outer_loop_tests.feature" + # feature: "features/outer_loop_tests.feature" mpi_path: ${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin - name: Cache vcpkg binary dir diff --git a/.github/workflows/build_oracle8.yml b/.github/workflows/build_oracle8.yml index 583fb57d7..0ab49eac4 100644 --- a/.github/workflows/build_oracle8.yml +++ b/.github/workflows/build_oracle8.yml @@ -146,10 +146,10 @@ jobs: cmake --build _build --config Release -j$(nproc) - - name: Run cucumber on outer_loop tests + - name: Tests with Cucumber uses: ./.github/workflows/cucumber-tests with: - feature: "features/outer_loop_tests.feature" + # feature: "features/outer_loop_tests.feature" mpi_path: ${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin - name: Running unit tests diff --git a/.github/workflows/build_ubuntu.yml b/.github/workflows/build_ubuntu.yml index f6e4bafd1..0c7dbf26d 100644 --- a/.github/workflows/build_ubuntu.yml +++ b/.github/workflows/build_ubuntu.yml @@ -150,13 +150,12 @@ jobs: run: | cmake --build _build --config Release -j$(nproc) - - name: Run cucumber on outer_loop tests + - name: Tests with Cucumber uses: ./.github/workflows/cucumber-tests with: - feature: "features/outer_loop_tests.feature" + # feature: "features/outer_loop_tests.feature" mpi_path: ${{ github.workspace }}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin - - name: Test run: | export PATH=${GITHUB_WORKSPACE}/_build/vcpkg_installed/x64-linux-release/tools/openmpi/bin:$PATH diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml index f5e3225aa..6339c17dd 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/build_windows.yml @@ -128,10 +128,10 @@ jobs: cmake --build _build --config Release -j4 - - name: Run cucumber on outer_loop tests + - name: Tests with Cucumber uses: ./.github/workflows/cucumber-tests with: - feature: "features/outer_loop_tests.feature" + # feature: "features/outer_loop_tests.feature" mpi_path: /c/Program Files/Microsoft MPI/Bin diff --git a/CMakeGraphVizOptions.cmake b/CMakeGraphVizOptions.cmake index e69de29bb..4fd047468 100644 --- a/CMakeGraphVizOptions.cmake +++ b/CMakeGraphVizOptions.cmake @@ -0,0 +1,3 @@ +set(GRAPHVIZ_GENERATE_PER_TARGET FALSE) +set(GRAPHVIZ_IGNORE_TARGETS ".*test.*") +set(GRAPHVIZ_EXTERNAL_LIBS FALSE) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 6faf48e68..800819773 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,3 +407,10 @@ endif () #version file to define Xpansion and Antares-simulator versions configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/python/antares_xpansion/__version__.py.in ${CMAKE_CURRENT_SOURCE_DIR}/src/python/antares_xpansion/__version__.py) include(CPack) + +add_custom_target(graphviz + COMMAND ${CMAKE_COMMAND} "--graphviz=xpansion.dot" . + COMMAND dot -Tsvg xpansion.dot -o xpansion.svg + COMMAND dot -Tpng xpansion.dot -o xpansion.png + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" +) diff --git a/tests/end_to_end/cucumber/steps/__init__.py b/examples/SmallTestFiveCandidatesWithWeights/output/.gitkeep similarity index 100% rename from tests/end_to_end/cucumber/steps/__init__.py rename to examples/SmallTestFiveCandidatesWithWeights/output/.gitkeep diff --git a/src/cpp/benders/benders_core/BendersMathLogger.cpp b/src/cpp/benders/benders_core/BendersMathLogger.cpp index b0aa1c1a3..143c9885f 100644 --- a/src/cpp/benders/benders_core/BendersMathLogger.cpp +++ b/src/cpp/benders/benders_core/BendersMathLogger.cpp @@ -82,11 +82,16 @@ LogDestination::LogDestination(const std::filesystem::path& file_path, std::cerr << err_msg.str(); } } +void LogDestination::setDelimiter(const std::string& delimiter) { + delimiter_ = delimiter; +} void MathLoggerBehaviour::write_header() { setHeadersList(); + LogsDestination().InsertDelimiter(); for (const auto& header : Headers()) { LogsDestination() << header; + LogsDestination().InsertDelimiter(); } LogsDestination() << std::endl; } @@ -154,36 +159,54 @@ double getDurationNotSolving(double iteration, double master, void PrintBendersData(LogDestination& log_destination, const CurrentIterationData& data, const HEADERSTYPE& type, const BENDERSMETHOD& method) { + log_destination.InsertDelimiter(); log_destination << data.it; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.lb; + log_destination.InsertDelimiter(); if (method == BENDERSMETHOD::BENDERS) { log_destination << std::scientific << std::setprecision(10) << data.ub; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.best_ub; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(2) << data.best_ub - data.lb; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(2) << (data.best_ub - data.lb) / data.best_ub; + log_destination.InsertDelimiter(); } log_destination << data.min_simplexiter; + log_destination.InsertDelimiter(); log_destination << data.max_simplexiter; + log_destination.InsertDelimiter(); if (type == HEADERSTYPE::LONG || method == BENDERSMETHOD::BENDERS_BY_BATCH) { log_destination << data.number_of_subproblem_solved; + log_destination.InsertDelimiter(); } if (type == HEADERSTYPE::LONG) { log_destination << data.cumulative_number_of_subproblem_solved; + log_destination.InsertDelimiter(); } log_destination << std::setprecision(2) << data.iteration_time; + log_destination.InsertDelimiter(); + log_destination << std::setprecision(2) << data.timer_master; + log_destination.InsertDelimiter(); + log_destination << std::setprecision(2) << data.subproblems_walltime; + log_destination.InsertDelimiter(); if (type == HEADERSTYPE::LONG) { log_destination << std::setprecision(2) << data.subproblems_cumulative_cputime; + log_destination.InsertDelimiter(); log_destination << std::setprecision(2) << getDurationNotSolving(data.iteration_time, data.timer_master, data.subproblems_walltime); + log_destination.InsertDelimiter(); } log_destination << std::endl; } @@ -192,20 +215,27 @@ void PrintExternalLoopData(LogDestination& log_destination, const CurrentIterationData& data, const HEADERSTYPE& type, const BENDERSMETHOD& method) { + log_destination.InsertDelimiter(); log_destination << data.outer_loop_current_iteration_data.benders_num_run; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.outer_loop_current_iteration_data.max_criterion; + log_destination.InsertDelimiter(); log_destination << data.outer_loop_current_iteration_data.max_criterion_area; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.outer_loop_current_iteration_data.external_loop_lambda; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.outer_loop_current_iteration_data.external_loop_lambda_min; + log_destination.InsertDelimiter(); log_destination << std::scientific << std::setprecision(10) << data.outer_loop_current_iteration_data.external_loop_lambda_max; diff --git a/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersMathLogger.h b/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersMathLogger.h index 861128678..c1a62c16f 100644 --- a/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersMathLogger.h +++ b/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersMathLogger.h @@ -37,11 +37,16 @@ class LogDestination { template std::ostream& operator<<(const T& obj); + std::ostream& InsertDelimiter() { return *stream_ << delimiter_; } private: std::ofstream file_stream_; std::ostream* stream_; std::streamsize width_ = 40; + std::string delimiter_ = "\t"; + + public: + void setDelimiter(const std::string& delimiter); }; template std::ostream& LogDestination::operator<<(const T& obj) { @@ -214,10 +219,14 @@ void MathLoggerExternalLoopSpecific::setHeadersList() { template void MathLoggerExternalLoopSpecific::Print( const CurrentIterationData& data) { + LogsDestination().InsertDelimiter(); LogsDestination() << data.outer_loop_current_iteration_data.benders_num_run; + LogsDestination().InsertDelimiter(); LogsDestination() << data.it; + LogsDestination().InsertDelimiter(); for (const auto& t : data.outer_loop_current_iteration_data.*ptr_) { LogsDestination() << std::scientific << std::setprecision(10) << t; + LogsDestination().InsertDelimiter(); } LogsDestination() << std::endl; } diff --git a/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersStructsDatas.h b/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersStructsDatas.h index 5aba0ec6e..d11659950 100644 --- a/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersStructsDatas.h +++ b/src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersStructsDatas.h @@ -15,8 +15,8 @@ struct OuterLoopCurrentIterationData{ double external_loop_lambda = 0.; double external_loop_lambda_min = 0.; double external_loop_lambda_max = 0.; - std::string max_criterion_area; - std::string max_criterion_area_best_it; + std::string max_criterion_area = "N/A"; + std::string max_criterion_area_best_it = "N/A"; }; /*! \struct * struct that hold current Benders iteration diff --git a/src/cpp/benders/factories/BendersFactory.cpp b/src/cpp/benders/factories/BendersFactory.cpp index b776abc1c..b416dbe03 100644 --- a/src/cpp/benders/factories/BendersFactory.cpp +++ b/src/cpp/benders/factories/BendersFactory.cpp @@ -183,7 +183,9 @@ BendersMainFactory::ProcessCriterionInput() { const auto fpath = std::filesystem::path(options_.INPUTROOT) / options_.OUTER_LOOP_OPTION_FILE; // if adequacy_criterion.yml is provided read it - if (std::filesystem::exists(fpath)) { + if ((method_ == BENDERSMETHOD::BENDERS_OUTERLOOP || + method_ == BENDERSMETHOD::BENDERS_BY_BATCH_OUTERLOOP) && + std::filesystem::exists(fpath)) { return Benders::Criterion::OuterLoopInputFromYaml().Read(fpath); } // else compute criterion for all areas! diff --git a/tests/cpp/logger/logger_test.cpp b/tests/cpp/logger/logger_test.cpp index 662c1220e..779d553a0 100644 --- a/tests/cpp/logger/logger_test.cpp +++ b/tests/cpp/logger/logger_test.cpp @@ -743,6 +743,7 @@ TEST_F(MasterLoggerTest, LogSwitchToInteger) { ASSERT_TRUE(_logger->_switchToIntegerCall); ASSERT_TRUE(_logger2->_switchToIntegerCall); } +static constexpr const char* const DELIMITER = "\t"; TEST(LogDestinationTest, WithInvalidEmptyFilePath) { const std::filesystem::path invalid_file_path(""); @@ -867,8 +868,10 @@ TEST(MathLoggerBendersByBatchTest, HeadersListStdOutShort) { std::streamsize width = 25; std::ostringstream expected_msg; + expected_msg << DELIMITER; for (const auto& header : headers_manager.HeadersList()) { expected_msg << std::setw(width) << std::left << header; + expected_msg << DELIMITER; } expected_msg << std::endl; std::stringstream redirectedStdout; @@ -886,8 +889,10 @@ TEST(MathLoggerBendersByBatchTest, HeadersListFileLong) { std::streamsize width = 25; std::ostringstream expected_msg; + expected_msg << DELIMITER; for (const auto& header : headers_manager.HeadersList()) { expected_msg << std::setw(width) << std::left << header; + expected_msg << DELIMITER; } expected_msg << std::endl; auto log_file = @@ -919,27 +924,39 @@ TEST(MathLoggerBendersByBatchTest, DataInFileLong) { data.iteration_time - data.timer_master - data.subproblems_walltime; std::ostringstream expected_msg; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.it; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.lb; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.min_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.max_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.number_of_subproblem_solved; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.cumulative_number_of_subproblem_solved; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.iteration_time; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.timer_master; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.subproblems_walltime; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.subproblems_cumulative_cputime; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << time_not_solving; + expected_msg << DELIMITER; expected_msg << std::endl; auto log_file = CreateRandomSubDir(std::filesystem::temp_directory_path()) / "log.txt"; @@ -970,20 +987,29 @@ TEST(MathLoggerBendersByBatchTest, DataInStdOutShort) { data.iteration_time - data.timer_master - data.subproblems_walltime; std::ostringstream expected_msg; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.it; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.lb; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.min_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.max_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.number_of_subproblem_solved; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.iteration_time; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.timer_master; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.subproblems_walltime; + expected_msg << DELIMITER; expected_msg << std::endl; std::stringstream redirectedStdout; @@ -1016,36 +1042,52 @@ TEST(MathLoggerBendersBaseTest, DataInFileLong) { data.iteration_time - data.timer_master - data.subproblems_walltime; std::ostringstream expected_msg; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.it; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.lb; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.ub; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.best_ub; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(2) << data.best_ub - data.lb; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(2) << (data.best_ub - data.lb) / data.best_ub; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.min_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.max_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.number_of_subproblem_solved; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.cumulative_number_of_subproblem_solved; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.iteration_time; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.timer_master; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.subproblems_walltime; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.subproblems_cumulative_cputime; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << time_not_solving; + expected_msg << DELIMITER; expected_msg << std::endl; auto log_file = CreateRandomSubDir(std::filesystem::temp_directory_path()) / "log.txt"; @@ -1076,28 +1118,40 @@ TEST(MathLoggerBendersBaseTest, DataInStdOutShort) { data.iteration_time - data.timer_master - data.subproblems_walltime; std::ostringstream expected_msg; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.it; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.lb; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.ub; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(10) << data.best_ub; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(2) << data.best_ub - data.lb; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::scientific << std::setprecision(2) << (data.best_ub - data.lb) / data.best_ub; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.min_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << data.max_simplexiter; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.iteration_time; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.timer_master; + expected_msg << DELIMITER; expected_msg << std::left << std::setw(width) << std::setprecision(2) << data.subproblems_walltime; + expected_msg << DELIMITER; expected_msg << std::endl; std::stringstream redirectedStdout; diff --git a/tests/end_to_end/cucumber/__init__.py b/tests/end_to_end/cucumber/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/end_to_end/cucumber/features/Benders_criterion_output_tests.feature b/tests/end_to_end/cucumber/features/Benders_criterion_output_tests.feature new file mode 100644 index 000000000..02320906e --- /dev/null +++ b/tests/end_to_end/cucumber/features/Benders_criterion_output_tests.feature @@ -0,0 +1,59 @@ +Feature: Benders Criterion files + + @fast @short @Benders + Scenario: xpansion-test-01 + Given the study path is "data_test/examples/xpansion-test-01" + When I run antares-xpansion with the benders method and 1 proc(s) + Then the simulation takes less than 300 seconds + And the simulation succeeds + And the expected positive unsupplied energy is + | Outer loop | Ite | area1 | area2 | flex | peak | pv | semibase | store_in | store_out | + | 0 | 1 | 5.3771400000e+05 | 4.3137090000e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.6208000000e+07 | 0.0000000000e+00 | + | 0 | 2 | 3.5308500000e+04 | 3.1096971069e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.5156337018e+07 | 1.9520000000e-02 | + | 0 | 3 | 1.6400000000e+03 | 2.5707979522e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4800616055e+07 | 2.0230000000e-02 | + | 0 | 4 | 1.9800000000e+02 | 2.3400869580e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4641367990e+07 | 2.1180000000e-02 | + | 0 | 5 | 1.5000000000e+01 | 2.2326080240e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4563004605e+07 | 2.1210000000e-02 | + | 0 | 6 | 5.2566603689e+01 | 2.1822119631e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4526012326e+07 | 2.1190000000e-02 | + | 0 | 7 | 1.3160579423e+02 | 2.1580236056e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4515297112e+07 | 2.1410000000e-02 | + | 0 | 8 | 1.1480289711e+02 | 2.1459011556e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4510396117e+07 | 2.1350000000e-02 | + | 0 | 9 | 1.0640144856e+02 | 2.1399579801e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4510156683e+07 | 2.1600000000e-02 | + | 0 | 10 | 1.0220072428e+02 | 2.1369066777e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4507996161e+07 | 2.1630000000e-02 | + | 0 | 11 | 1.0010036214e+02 | 2.1353880070e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4506928012e+07 | 2.1620000000e-02 | + | 0 | 12 | 9.9050181069e+01 | 2.1346301325e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4506394365e+07 | 2.1610000000e-02 | + | 0 | 13 | 9.8525090534e+01 | 2.1342512929e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4506126957e+07 | 2.1610000000e-02 | + | 0 | 14 | 9.8262545267e+01 | 2.1340618754e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4505993242e+07 | 2.1610000000e-02 | + | 0 | 15 | 9.8131272633e+01 | 2.1339671667e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4505926384e+07 | 2.1610000000e-02 | + | 0 | 16 | 9.8065636316e+01 | 2.1339198123e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4505892955e+07 | 2.1610000000e-02 | + | 0 | 17 | 9.8032818157e+01 | 2.1338961351e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4505876240e+07 | 2.1610000000e-02 | + | 0 | 18 | 9.8016409078e+01 | 2.1338842965e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4505867883e+07 | 2.1610000000e-02 | + | 0 | 19 | 1.1900000000e+02 | 3.2167972574e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4866362811e+07 | 2.2450000000e-02 | + | 0 | 20 | 3.2080000000e+03 | 3.2167972574e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4866362811e+07 | 2.2450000000e-02 | + | 0 | 21 | 1.4810000000e+03 | 3.2167972574e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4866362811e+07 | 2.2450000000e-02 | + | 0 | 22 | 1.4810000000e+03 | 3.2167972574e+06 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 2.4866362811e+07 | 2.2450000000e-02 | + And the expected loss of load is + | Outer loop | Ite | area1 | area2 | flex | peak | pv | semibase | store_in | store_out | + | 0 | 1 | 3.6366666667e+02 | 1.8480000000e+03 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 2 | 4.6333333333e+01 | 1.2103333333e+03 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 3 | 5.0000000000e+00 | 9.8900000000e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 4 | 6.6666666667e-01 | 8.8600000000e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 5 | 3.3333333333e-01 | 8.3900000000e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 6 | 3.3333333333e-01 | 8.2300000000e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 7 | 6.6666666667e-01 | 8.1633333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 8 | 6.6666666667e-01 | 8.1333333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 9 | 6.6666666667e-01 | 8.1066666667e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 10 | 6.6666666667e-01 | 8.0933333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 11 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 12 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 13 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 14 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 15 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 16 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 17 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 18 | 6.6666666667e-01 | 8.0733333333e+02 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7360000000e+03 | 0.0000000000e+00 | + | 0 | 19 | 1.0000000000e+00 | 1.1003333333e+03 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7150000000e+03 | 0.0000000000e+00 | + | 0 | 20 | 7.3333333333e+00 | 1.1003333333e+03 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7150000000e+03 | 0.0000000000e+00 | + | 0 | 21 | 3.0000000000e+00 | 1.1003333333e+03 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7150000000e+03 | 0.0000000000e+00 | + | 0 | 22 | 3.0000000000e+00 | 1.1003333333e+03 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 0.0000000000e+00 | 8.7150000000e+03 | 0.0000000000e+00 | + + + diff --git a/tests/end_to_end/cucumber/features/__init__.py b/tests/end_to_end/cucumber/features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/end_to_end/cucumber/features/environment.py b/tests/end_to_end/cucumber/features/environment.py new file mode 100644 index 000000000..4dd9a9a57 --- /dev/null +++ b/tests/end_to_end/cucumber/features/environment.py @@ -0,0 +1,6 @@ +import tempfile +from pathlib import Path + + +def before_scenario(context, scenario): + context.temp_dir = Path(tempfile.TemporaryDirectory().name) diff --git a/tests/end_to_end/cucumber/features/steps/__init__.py b/tests/end_to_end/cucumber/features/steps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/end_to_end/cucumber/features/steps/steps.py b/tests/end_to_end/cucumber/features/steps/steps.py new file mode 100644 index 000000000..85b876945 --- /dev/null +++ b/tests/end_to_end/cucumber/features/steps/steps.py @@ -0,0 +1,223 @@ +import csv +import io +import json +import math +import os +import shutil +import subprocess +import sys +from pathlib import Path +import numpy as np +from behave import * + +from utils_functions import get_mpi_command, get_conf, read_outputs, remove_outputs + + +@given('the study path is "{string}"') +def study_path_is(context, string): + # context.study_path + context.study_path = Path() / "../../" / string + context.tmp_study = context.temp_dir / context.study_path.name + shutil.copytree(context.study_path, context.tmp_study) + + +def build_outer_loop_command(context, n: int, option_file: str = "options.json"): + command = get_mpi_command(allow_run_as_root=context.allow_run_as_root, nproc=n) + exe_path = Path(get_conf("DEFAULT_INSTALL_DIR")) / get_conf("OUTER_LOOP") + command.append(str(exe_path)) + command.append(option_file) + return command + + +def build_launch_command(study_dir: Path, method: str, nproc: int, in_memory: bool, allow_run_as_root: bool = False): + command = [ + sys.executable, + "../../src/python/launch.py", "--installDir", str(get_conf('DEFAULT_INSTALL_DIR')), "--dataDir", + str(study_dir), "--method", + method, "-n", str(nproc), "--oversubscribe"] + if in_memory: + command.append("--memory") + print(command) + if allow_run_as_root: + command.append("--allow-run-as-root") + return command + + +def read_json_file(output_path): + with open(output_path, 'r') as file: + outputs = json.load(file) + return outputs + + +@when('I run outer loop with {n:d} proc(s) and "{option_file}" as option file') +@when('I run outer loop with {n:d} proc(s)') +def run_outer_loop(context, n, option_file: str = "options.json"): + context.allow_run_as_root = get_conf("allow_run_as_root") + command = build_outer_loop_command(context, n, option_file) + print(f"Running command: {command}") + old_cwd = os.getcwd() + + lp_path = Path(context.tmp_study) / "lp" if (Path(context.tmp_study) / "lp").exists() else Path( + context.tmp_study) + + os.chdir(lp_path) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + process.communicate() + context.return_code = process.returncode + options = read_json_file(option_file) + output_file_path = options["JSON_FILE"] + context.outputs = read_json_file(output_file_path) + context.loss_of_load_file = (Path(options["OUTPUTROOT"]) / "LOLD.txt").absolute() + context.positive_unsupplied_energy_file = (Path(options["OUTPUTROOT"]) / "PositiveUnsuppliedEnergy.txt").absolute() + + os.chdir(old_cwd) + + +@when('I run antares-xpansion with the {method} method and {n:d} proc(s)') +@when('I run antares-xpansion in {memory} with the {method} method and {n:d} proc(s)') +def run_antares_xpansion(context, method, memory=None, n: int = 1): + memory = True if memory is not None else False + # Clean study output + remove_outputs(context.tmp_study) + + context.return_code = run_command(context.tmp_study, memory=memory, method=method, n_mpi=n, + allow_run_as_root=get_conf("allow_run_as_root")) + + output_path = context.tmp_study / "output" + outputs = read_outputs(output_path, use_archive=not memory, lold=True, positive_unsupplied_energy=True) + context.outputs = outputs.out_json + context.options_data = outputs.options_json + context.lold = outputs.lold + context.positive_unsupplied_energy = outputs.positive_unsupplied_energy + + +def run_command(study_path, memory, method, n_mpi, allow_run_as_root=False): + command = build_launch_command(study_path, method, nproc=n_mpi, in_memory=memory, + allow_run_as_root=allow_run_as_root) + print(f"Running command: {command}") + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + out, err = process.communicate() + if process.returncode != 0: + print("*********************** Begin stdout ***********************") + print(out) + print("*********************** End stdout ***********************") + + print("*********************** Begin stderr ***********************") + print(err) + print("*********************** End stderr ***********************") + + return process.returncode + + +@then("the simulation takes less than {seconds:d} seconds") +def check_simu_time(context, seconds): + assert context.outputs["run_duration"] <= seconds + + +@then("the simulation succeeds") +def simu_success(context): + assert context.return_code == 0 + + +@then("the expected overall cost is {value:g}") +def check_overall_cost(context, value): + np.testing.assert_allclose(value, context.outputs["solution"]["overall_cost"], rtol=1e-6, atol=0) + + +@then("the expected investment cost is {value:g}") +def check_overall_cost(context, value): + np.testing.assert_allclose(value, context.outputs["solution"]["investment_cost"], rtol=1e-6, atol=0) + + +def assert_dict_allclose(actual, expected, rtol=1e-06, atol=0): + for key in expected: + np.testing.assert_allclose( + actual[key], + expected[key], + rtol=rtol, + atol=atol, + err_msg=f"Mismatch found at key '{key}'" + ) + + +@then("the solution is") +def check_solution(context): + expected_solution = {row['variable']: float(row['value']) for row in context.table} + assert_dict_allclose(context.outputs["solution"]["values"], expected_solution) + + +def read_table_from_string(raw_data): + reader = csv.reader(io.StringIO(raw_data), delimiter='\t') + header = [item.strip() for item in + next(reader)[1:-1]] # Store the header row, ignoring the first and last columns + current_results = [{header[index]: item.strip() for index, item in enumerate(row[1:-1])} for row in + reader] + + return current_results + + +def read_cucumber_table_from_file(filename): + with open(filename, 'r') as file: + reader = csv.reader(file, delimiter='\t') + header = [item.strip() for item in + next(reader)[1:-1]] # Store the header row, ignoring the first and last columns + current_results = [{header[index]: item.strip() for index, item in enumerate(row[1:-1])} for row in + reader] + return current_results + + +def is_file_full_of_zeros(filename, abs_tol=1e-9): + data = read_cucumber_table_from_file(filename) + + for line_number, line in enumerate(data): + + for key, value in line.items(): + if key in ["Outer loop", "Ite"]: + continue + try: + value = float(value) + except (ValueError, IndexError): + print(f"Error parsing line: {line_number} at column {key}") + return False + + # Use math.isclose to compare to zero with tolerance + if not math.isclose(value, 0.0, abs_tol=abs_tol): + print(f"Error {value} is not close to 0") + return False + + return True + +@then("LOLD.txt and PositiveUnsuppliedEnergy.txt files are full of zeros") +def check_other_outputs(context): + assert (is_file_full_of_zeros(context.loss_of_load_file)) + assert (is_file_full_of_zeros(context.positive_unsupplied_energy_file)) + + +@then("the expected positive unsupplied energy is") +def check_positive_unsupplied_energy(context): + results = read_table_from_string(context.positive_unsupplied_energy) + check_cucumber_table(context, results) + + +@then("the expected loss of load is") +def check_loss_of_load_is(context): + results = read_table_from_string(context.lold) + check_cucumber_table(context, results) + + +def check_cucumber_table(context, results): + headers = context.table.headings + for i, row in enumerate(context.table): + for header in headers: + expected_value = float(row[header]) + actual_value = float(results[i][header]) + + np.testing.assert_allclose(actual_value, expected_value, rtol=1e-6, atol=0, + err_msg=f"Mismatch in row {i + 1}, column '{header}': expected {expected_value}, got {actual_value}") + + +def get_results_file_path_from_logs(logs: bytes) -> str: + for line in logs.splitlines(): + if b'Optimization results available in : ' in line: + return line.split(b'Optimization results available in : ')[1].decode('ascii') + raise LookupError("Could not find results file path in output logs") diff --git a/tests/end_to_end/cucumber/features/weights.feature b/tests/end_to_end/cucumber/features/weights.feature index 68b00ab30..ca396e126 100644 --- a/tests/end_to_end/cucumber/features/weights.feature +++ b/tests/end_to_end/cucumber/features/weights.feature @@ -7,3 +7,12 @@ Feature: add weights on MC years Given the study path is "examples/SmallTestFiveCandidatesWithWeights" When I run antares-xpansion in memory with the benders method and 1 proc(s) Then the simulation succeeds + And the expected overall cost is 24232177891.450203 + And the expected investment cost is 230600000.0 + And the solution is + | variable | value | + | battery | 1000.0 | + | peak | 1500.0000 | + | pv | 1000.0000 | + | semibase | 200.0 | + | transmission_line | 0.0 | diff --git a/tests/end_to_end/cucumber/steps/steps.py b/tests/end_to_end/cucumber/steps/steps.py deleted file mode 100644 index 429d99261..000000000 --- a/tests/end_to_end/cucumber/steps/steps.py +++ /dev/null @@ -1,149 +0,0 @@ -import json -import math -import os -import subprocess -from pathlib import Path - -import numpy as np -from behave import * - -from utils_functions import get_mpi_command, get_conf - - -@given('the study path is "{string}"') -def study_path_is(context, string): - context.study_path = os.path.join(Path() / "../../", - string.replace("/", os.sep)) - - -def build_outer_loop_command(context, n: int, option_file: str = "options.json"): - command = get_mpi_command(allow_run_as_root=context.allow_run_as_root, nproc=n) - exe_path = Path(get_conf("DEFAULT_INSTALL_DIR")) / get_conf("OUTER_LOOP") - command.append(str(exe_path)) - command.append(option_file) - return command - - -def build_launch_command(study_dir: str, method: str, nproc: int, in_memory: bool): - command = f"python ../../src/python/launch.py --installDir {get_conf('DEFAULT_INSTALL_DIR')} --dataDir {study_dir} --method {method} -n {nproc} --oversubscribe" - if in_memory: - command += " --memory" - return command - - -def read_json_file(output_path): - with open(output_path, 'r') as file: - outputs = json.load(file) - return outputs - - -def read_file(output_path): - with open(output_path, 'r') as file: - outputs = file.readlines() - return outputs - - -@when('I run outer loop with {n:d} proc(s) and "{option_file}" as option file') -@when('I run outer loop with {n:d} proc(s)') -def run_outer_loop(context, n, option_file: str = "options.json"): - context.allow_run_as_root = get_conf("allow_run_as_root") - command = build_outer_loop_command(context, n, option_file) - print(f"Running command: {command}") - old_cwd = os.getcwd() - - lp_path = Path(context.study_path) / "lp" if (Path(context.study_path) / "lp").exists() else Path( - context.study_path) - - os.chdir(lp_path) - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - process.communicate() - context.return_code = process.returncode - options = read_json_file(option_file) - output_file_path = options["JSON_FILE"] - context.outputs = read_json_file(output_file_path) - context.loss_of_load_file = (Path(options["OUTPUTROOT"]) / "LOLD.txt").absolute() - context.positive_unsupplied_energy_file = (Path(options["OUTPUTROOT"]) / "PositiveUnsuppliedEnergy.txt").absolute() - - os.chdir(old_cwd) - - -@when('I run antares-xpansion in memory with the {method} method and {n:d} proc(s)') -def run_antares_xpansion(context, method, n): - command = build_launch_command(context.study_path, method, n, True) - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, shell=True) - out, err = process.communicate() - context.return_code = process.returncode - context.outputs = read_json_file(Path(get_results_file_path_from_logs(out))) - - -@then("the simulation takes less than {seconds:d} seconds") -def check_simu_time(context, seconds): - assert context.outputs["run_duration"] <= seconds - - -@then("the simulation succeeds") -def simu_success(context): - assert context.return_code == 0 - - -@then("the expected overall cost is {value:g}") -def check_overall_cost(context, value): - np.testing.assert_allclose(value, context.outputs["solution"]["overall_cost"], rtol=1e-6, atol=0) - - -def assert_dict_allclose(actual, expected, rtol=1e-06, atol=0): - for key in expected: - np.testing.assert_allclose( - actual[key], - expected[key], - rtol=rtol, - atol=atol, - err_msg=f"Mismatch found at key '{key}'" - ) - - -@then("the solution is") -def check_solution(context): - expected_solution = {row['variable']: float(row['value']) for row in context.table} - assert_dict_allclose(context.outputs["solution"]["values"], expected_solution) - - -def is_column_full_of_zeros(filename, column_index, abs_tol=1e-9): - with open(filename, 'r') as file: - # Skip the header - next(file) - - # Check each line in the file - for line in file: - columns = line.split() - - # Ensure column exists - if column_index >= len(columns): - print(f"Error: Missing column at index {column_index} in line: {line.strip()}") - return False - - try: - value = float(columns[column_index]) - except (ValueError, IndexError): - print(f"Error parsing line: {line.strip()}") - return False - - # Use math.isclose to compare to zero with tolerance - if not math.isclose(value, 0.0, abs_tol=abs_tol): - print(f"Error {value} is not close to 0") - return False - - return True - - -@then("LOLD.txt and PositiveUnsuppliedEnergy.txt files are full of zeros") -def check_other_outputs(context): - assert (is_column_full_of_zeros(context.loss_of_load_file, 2)) - assert (is_column_full_of_zeros(context.positive_unsupplied_energy_file, 2)) - - -def get_results_file_path_from_logs(logs: bytes) -> str: - for line in logs.splitlines(): - if b'Optimization results available in : ' in line: - return line.split(b'Optimization results available in : ')[1].decode('ascii') - raise LookupError("Could not find results file path in output logs") diff --git a/tests/end_to_end/examples/example_test.py b/tests/end_to_end/examples/example_test.py index 343298e6e..7ff314504 100644 --- a/tests/end_to_end/examples/example_test.py +++ b/tests/end_to_end/examples/example_test.py @@ -1,9 +1,6 @@ -import json -import os import shutil import subprocess import sys -import zipfile from enum import Enum from pathlib import Path @@ -11,6 +8,7 @@ import pytest from src.python.antares_xpansion.candidates_reader import CandidatesReader +from tests.end_to_end.utils_functions import read_outputs, remove_outputs ALL_STUDIES_PATH = Path("../../../data_test/examples") RELATIVE_TOLERANCE = 1e-4 @@ -22,31 +20,6 @@ class BendersMethod(Enum): BENDERS_BY_BATCH = "benders_by_batch" -def get_json_filepath(output_dir, folder, filename): - op = [] - for path in Path(output_dir).iterdir(): - for jsonpath in Path(path / folder).rglob(filename): - op.append(jsonpath) - assert len(op) == 1 - return op[0] - -def get_json_file_data(output_dir, folder, filename): - data = None - for path in Path(output_dir).iterdir(): - if path.suffix == ".zip": - with zipfile.ZipFile(path, "r") as archive: - data = json.loads(archive.read(folder+"/"+filename)) - return data - - -def remove_outputs(study_path): - output_path = study_path / "output" - if os.path.isdir(output_path): - for f in Path(output_path).iterdir(): - if f.is_dir(): - shutil.rmtree(f) - - def launch_xpansion(install_dir, study_path, allow_run_as_root=False, nproc: int = 4): # Clean study output remove_outputs(study_path) @@ -119,19 +92,9 @@ def assert_convergence(solution, options_data, method: BendersMethod): def verify_solution(study_path, expected_values, expected_investment_solution, method: BendersMethod = BendersMethod.BENDERS, use_archive=True): output_path = study_path / "output" - - if use_archive: - json_data = get_json_file_data(output_path, "expansion", "out.json") - options_data = get_json_file_data(output_path, "lp", "options.json") - else: - json_path = get_json_filepath(output_path, "expansion", "out.json") - options_path = get_json_filepath(output_path, "lp", "options.json") - - with open(str(json_path), "r") as json_file: - json_data = json.load(json_file) - - with open(str(options_path), "r") as options_file: - options_data = json.load(options_file) + outputs = read_outputs(output_path, use_archive) + json_data = outputs.out_json + options_data = outputs.options_json solution = json_data["solution"] investment_solution = solution["values"] diff --git a/tests/end_to_end/utils_functions.py b/tests/end_to_end/utils_functions.py index 856567248..7958f4953 100644 --- a/tests/end_to_end/utils_functions.py +++ b/tests/end_to_end/utils_functions.py @@ -1,8 +1,12 @@ +import json import os +import shutil import sys +import zipfile from pathlib import Path import yaml +from dataclasses import dataclass # File CONFIG_FILE_PATH # yaml file containing executable name @@ -24,13 +28,9 @@ def get_conf(key: str): def get_mpi_command(allow_run_as_root=False, nproc: int = 1): - MPI_LAUNCHER = "" - MPI_N = "" nproc_str = str(nproc) if sys.platform.startswith("win32"): - MPI_LAUNCHER = "mpiexec" - MPI_N = "-n" - return [MPI_LAUNCHER, MPI_N, nproc_str] + return ["mpiexec", "-n", nproc_str] elif sys.platform.startswith("linux"): MPI_LAUNCHER = "mpirun" MPI_N = "-np" @@ -38,3 +38,88 @@ def get_mpi_command(allow_run_as_root=False, nproc: int = 1): return [MPI_LAUNCHER, "--allow-run-as-root", MPI_N, nproc_str, "--oversubscribe"] else: return [MPI_LAUNCHER, MPI_N, nproc_str, "--oversubscribe"] + + +def remove_outputs(study_path): + output_path = study_path / "output" + if os.path.isdir(output_path): + shutil.rmtree(output_path) + os.makedirs(output_path) + +def get_filepath(output_dir, folder, filename): + op = [] + for path in Path(output_dir).iterdir(): + for jsonpath in Path(path / folder).rglob(filename): + op.append(jsonpath) + assert len(op) == 1 + return op[0] + + +def read_file(output_path): + with open(output_path, 'r') as file: + outputs = file.readlines() + return outputs + + +class FilesToRead: + out_json: Path + options_json: Path + lold: Path = None + positive_unsupplied_energy: Path = None + + +class Outputs: + out_json: str + options_json: str + lold: str + positive_unsupplied_energy: str + + +def get_out_data(output_dir, files_to_read: FilesToRead) -> Outputs: + for path in Path(output_dir).iterdir(): + if path.suffix == ".zip": + with zipfile.ZipFile(path, "r") as archive: + out = Outputs() + out.out_json = json.loads(archive.read(files_to_read.out_json.as_posix())) + out.options_json = json.loads(archive.read(files_to_read.options_json.as_posix())) + if files_to_read.lold: + out.lold = archive.read(files_to_read.lold.as_posix()).decode('utf-8') + if files_to_read.positive_unsupplied_energy: + out.positive_unsupplied_energy = archive.read( + files_to_read.positive_unsupplied_energy.as_posix()).decode( + 'utf-8') + + return out + return None + + +def read_outputs(output_path, use_archive=True, lold=False, positive_unsupplied_energy=False): + files_to_read = FilesToRead() + files_to_read.out_json = Path("expansion") / "out.json" + files_to_read.options_json = Path("lp") / "options.json" + + if lold: + files_to_read.lold = Path("lp") / "LOLD.txt" + if positive_unsupplied_energy: + files_to_read.positive_unsupplied_energy = Path("lp") / "PositiveUnsuppliedEnergy.txt" + + if use_archive: + return get_out_data(output_path, files_to_read) + else: + out = Outputs() + json_path = get_filepath(output_path, "expansion", "out.json") + options_path = get_filepath(output_path, "lp", "options.json") + + with open(str(json_path), "r") as json_file: + out.out_json = json.load(json_file) + + with open(str(options_path), "r") as options_file: + out.options_json = json.load(options_file) + + if lold: + out.lold = read_file(get_filepath(output_path, "lp", "LOLD.txt")) + if positive_unsupplied_energy: + out.positive_unsupplied_energy = read_file( + get_filepath(output_path, "lp", "PositiveUnsuppliedEnergy.txt")) + + return out