diff --git a/antarest/study/service.py b/antarest/study/service.py index 7766a1e6dc..42c0486fbd 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -97,6 +97,7 @@ StudySimResultDTO, ) from antarest.study.repository import StudyFilter, StudyMetadataRepository, StudyPagination, StudySortBy +from antarest.study.storage.matrix_profile import get_matrix_profiles_by_version from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.folder_node import ChildNotFoundError from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode @@ -113,14 +114,7 @@ should_study_be_denormalized, upgrade_study, ) -from antarest.study.storage.utils import ( - MatrixProfile, - assert_permission, - get_matrix_profile_by_version, - get_start_date, - is_managed, - remove_from_cache, -) +from antarest.study.storage.utils import assert_permission, get_start_date, is_managed, remove_from_cache from antarest.study.storage.variantstudy.model.command.icommand import ICommand from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_comments import UpdateComments @@ -151,40 +145,6 @@ def get_disk_usage(path: t.Union[str, Path]) -> int: return total_size -def _handle_specific_matrices( - df: pd.DataFrame, - matrix_profile: MatrixProfile, - matrix_path: str, - *, - with_index: bool, - with_header: bool, -) -> pd.DataFrame: - if with_header: - if Path(matrix_path).parts[1] == "links": - cols = _handle_links_columns(matrix_path, matrix_profile) - else: - cols = matrix_profile.cols - if cols: - df.columns = pd.Index(cols) - rows = matrix_profile.rows - if with_index and rows: - df.index = rows # type: ignore - return df - - -def _handle_links_columns(matrix_path: str, matrix_profile: MatrixProfile) -> t.List[str]: - path_parts = Path(matrix_path).parts - area_id_1 = path_parts[2] - area_id_2 = path_parts[3] - result = matrix_profile.cols - for k, col in enumerate(result): - if col == "Hurdle costs direct": - result[k] = f"{col} ({area_id_1}->{area_id_2})" - elif col == "Hurdle costs indirect": - result[k] = f"{col} ({area_id_2}->{area_id_1})" - return result - - class StudyUpgraderTask: """ Task to perform a study upgrade. @@ -2452,12 +2412,11 @@ def get_matrix_with_index_and_header( ) df_matrix.index = time_column - matrix_profiles = get_matrix_profile_by_version(int(study.version)) + matrix_profiles = get_matrix_profiles_by_version(int(study.version)) for pattern, matrix_profile in matrix_profiles.items(): if fnmatch.fnmatch(path, pattern): - return _handle_specific_matrices( + return matrix_profile.handle_specific_matrices( df_matrix, - matrix_profile, path, with_index=with_index, with_header=with_header, diff --git a/antarest/study/storage/matrix_profile.py b/antarest/study/storage/matrix_profile.py new file mode 100644 index 0000000000..55833d0382 --- /dev/null +++ b/antarest/study/storage/matrix_profile.py @@ -0,0 +1,160 @@ +import copy +import typing as t +from pathlib import Path + +import pandas as pd + + +class MatrixProfile(t.NamedTuple): + """ + Matrix profile for time series or classic tables. + """ + + cols: t.Sequence[str] + rows: t.Sequence[str] + stats: bool + + def handle_specific_matrices( + self, + df: pd.DataFrame, + matrix_path: str, + *, + with_index: bool, + with_header: bool, + ) -> pd.DataFrame: + if with_header: + if Path(matrix_path).parts[1] == "links": + cols = self.handle_links_columns(matrix_path) + else: + cols = self.cols + if cols: + df.columns = pd.Index(cols) + rows = self.rows + if with_index and rows: + df.index = rows # type: ignore + return df + + def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: + path_parts = Path(matrix_path).parts + area_id_1 = path_parts[2] + area_id_2 = path_parts[3] + result = list(self.cols) + for k, col in enumerate(result): + if col == "Hurdle costs direct": + result[k] = f"{col} ({area_id_1}->{area_id_2})" + elif col == "Hurdle costs indirect": + result[k] = f"{col} ({area_id_2}->{area_id_1})" + return result + + +# noinspection SpellCheckingInspection +SPECIFIC_MATRICES = { + "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( + cols=[str(i) for i in range(101)], + rows=["Generating Power", "Pumping Power"], + stats=False, + ), + "input/hydro/common/capacity/maxpower_*": MatrixProfile( + cols=[ + "Generating Max Power (MW)", + "Generating Max Energy (Hours at Pmax)", + "Pumping Max Power (MW)", + "Pumping Max Energy (Hours at Pmax)", + ], + rows=[], + stats=False, + ), + "input/hydro/common/capacity/reservoir_*": MatrixProfile( + cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], + rows=[], + stats=False, + ), + "input/hydro/common/capacity/waterValues_*": MatrixProfile( + cols=[f"{i}%" for i in range(101)], + rows=[], + stats=False, + ), + "input/hydro/series/*/mod": MatrixProfile(cols=[], rows=[], stats=True), + "input/hydro/series/*/ror": MatrixProfile(cols=[], rows=[], stats=True), + "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), + "input/hydro/prepro/*/energy": MatrixProfile( + cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], + rows=[ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ], + stats=False, + ), + "input/thermal/prepro/*/*/modulation": MatrixProfile( + cols=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], + rows=[], + stats=False, + ), + "input/thermal/prepro/*/*/data": MatrixProfile( + cols=["FO Duration", "PO Duration", "FO Rate", "PO Rate", "NPO Min", "NPO Max"], + rows=[], + stats=False, + ), + "input/reserves/*": MatrixProfile( + cols=["Primary Res. (draft)", "Strategic Res. (draft)", "DSM", "Day Ahead"], + rows=[], + stats=False, + ), + "input/misc-gen/miscgen-*": MatrixProfile( + cols=["CHP", "Bio Mass", "Bio Gaz", "Waste", "GeoThermal", "Other", "PSP", "ROW Balance"], + rows=[], + stats=False, + ), + "input/bindingconstraints/*": MatrixProfile(cols=["<", ">", "="], rows=[], stats=False), + "input/links/*/*": MatrixProfile( + cols=[ + "Capacités de transmission directes", + "Capacités de transmission indirectes", + "Hurdle costs direct", + "Hurdle costs indirect", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ], + rows=[], + stats=False, + ), +} + +SPECIFIC_MATRICES_820 = copy.deepcopy(SPECIFIC_MATRICES) +SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( + cols=[ + "Hurdle costs direct", + "Hurdle costs indirect", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ], + rows=[], + stats=False, +) + +SPECIFIC_MATRICES_870 = copy.deepcopy(SPECIFIC_MATRICES_820) +# noinspection SpellCheckingInspection +SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) + + +def get_matrix_profiles_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: + if study_version < 820: + return SPECIFIC_MATRICES + elif study_version < 870: + return SPECIFIC_MATRICES_820 + else: + return SPECIFIC_MATRICES_870 diff --git a/antarest/study/storage/utils.py b/antarest/study/storage/utils.py index f3eca35ad1..da939a3125 100644 --- a/antarest/study/storage/utils.py +++ b/antarest/study/storage/utils.py @@ -1,5 +1,4 @@ import calendar -import copy import logging import math import os @@ -244,128 +243,6 @@ def assert_permission( DAY_NAMES = calendar.day_name[:] -def _generate_columns(column_suffix: str) -> t.List[str]: - return [f"{i}{column_suffix}" for i in range(101)] - - -class MatrixProfile(t.NamedTuple): - """ - Matrix profile for time series or classic tables. - """ - - cols: t.List[str] - rows: t.List[str] - stats: bool - - -SPECIFIC_MATRICES = { - "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( - cols=_generate_columns(""), - rows=["Generating Power", "Pumping Power"], - stats=False, - ), - "input/hydro/common/capacity/maxpower_*": MatrixProfile( - cols=[ - "Generating Max Power (MW)", - "Generating Max Energy (Hours at Pmax)", - "Pumping Max Power (MW)", - "Pumping Max Energy (Hours at Pmax)", - ], - rows=[], - stats=False, - ), - "input/hydro/common/capacity/reservoir_*": MatrixProfile( - cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], - rows=[], - stats=False, - ), - "input/hydro/common/capacity/waterValues_*": MatrixProfile(cols=_generate_columns("%"), rows=[], stats=False), - "input/hydro/series/*/mod": MatrixProfile(cols=[], rows=[], stats=True), - "input/hydro/series/*/ror": MatrixProfile(cols=[], rows=[], stats=True), - "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), - "input/hydro/prepro/*/energy": MatrixProfile( - cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], - rows=[ - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", - ], - stats=False, - ), - "input/thermal/prepro/*/*/modulation": MatrixProfile( - cols=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], - rows=[], - stats=False, - ), - "input/thermal/prepro/*/*/data": MatrixProfile( - cols=["FO Duration", "PO Duration", "FO Rate", "PO Rate", "NPO Min", "NPO Max"], - rows=[], - stats=False, - ), - "input/reserves/*": MatrixProfile( - cols=["Primary Res. (draft)", "Strategic Res. (draft)", "DSM", "Day Ahead"], - rows=[], - stats=False, - ), - "input/misc-gen/miscgen-*": MatrixProfile( - cols=["CHP", "Bio Mass", "Bio Gaz", "Waste", "GeoThermal", "Other", "PSP", "ROW Balance"], - rows=[], - stats=False, - ), - "input/bindingconstraints/*": MatrixProfile(cols=["<", ">", "="], rows=[], stats=False), - "input/links/*/*": MatrixProfile( - cols=[ - "Capacités de transmission directes", - "Capacités de transmission indirectes", - "Hurdle costs direct", - "Hurdle costs indirect", - "Impedances", - "Loop flow", - "P.Shift Min", - "P.Shift Max", - ], - rows=[], - stats=False, - ), -} - - -SPECIFIC_MATRICES_820 = copy.deepcopy(SPECIFIC_MATRICES) -SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( - cols=[ - "Hurdle costs direct", - "Hurdle costs indirect", - "Impedances", - "Loop flow", - "P.Shift Min", - "P.Shift Max", - ], - rows=[], - stats=False, -) - -SPECIFIC_MATRICES_870 = copy.deepcopy(SPECIFIC_MATRICES_820) -SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) - - -def get_matrix_profile_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: - if study_version < 820: - return SPECIFIC_MATRICES - elif study_version < 870: - return SPECIFIC_MATRICES_820 - else: - return SPECIFIC_MATRICES_870 - - def get_start_date( file_study: FileStudy, output_id: t.Optional[str] = None,