diff --git a/src/antares/study/version/model/study_antares.py b/src/antares/study/version/model/study_antares.py index d59297c..c27b4a2 100644 --- a/src/antares/study/version/model/study_antares.py +++ b/src/antares/study/version/model/study_antares.py @@ -23,7 +23,7 @@ class StudyAntares: [antares] caption = Thermal fleet optimization - version = 9.1 + version = 9.2 created = 1246524135 lastsave = 1686128483 author = John Doe @@ -34,7 +34,7 @@ class StudyAntares: >>> data = { ... "caption": "Thermal fleet optimization", - ... "version": "9.1", + ... "version": "9.2", ... "created_date": 1246524135, ... "last_save_date": 1686128483, ... "author": "John Doe", @@ -45,7 +45,7 @@ class StudyAntares: >>> study_antares.caption 'Thermal fleet optimization' >>> study_antares.version - StudyVersion(major=9, minor=1, patch=0) + StudyVersion(major=9, minor=2, patch=0) >>> study_antares.created_date datetime.datetime(2009, 7, 2, 10, 42, 15) >>> study_antares.last_save_date diff --git a/src/antares/study/version/upgrade_app/exceptions.py b/src/antares/study/version/upgrade_app/exceptions.py index 0cff54c..c635346 100644 --- a/src/antares/study/version/upgrade_app/exceptions.py +++ b/src/antares/study/version/upgrade_app/exceptions.py @@ -1,3 +1,6 @@ +from typing import List + + class UpgradeError(Exception): """ Base class for exceptions in this module. @@ -22,3 +25,17 @@ def __init__(self, link_path: str): f" that allows to replace the matrix links by valid TSV matrices." ) super().__init__(message) + + +class UnexpectedThematicTrimmingFieldsError(UpgradeError): + """ + Exception raised when there are unexpected thematic trimming fields in the generaldata.ini file. + """ + + def __init__(self, enabled_fields: List[str], disabled_fields: List[str]): + message = ( + f"Found these enabled fields {enabled_fields} with these disabled fields {disabled_fields} in the" + f" generaldata.ini. We cannot determine if the new variable `STS by group` should be enabled or disabled." + f" Choose one before upgrading your study." + ) + super().__init__(message) diff --git a/src/antares/study/version/upgrade_app/scenario_mapping.py b/src/antares/study/version/upgrade_app/scenario_mapping.py index 8101d73..9af6820 100644 --- a/src/antares/study/version/upgrade_app/scenario_mapping.py +++ b/src/antares/study/version/upgrade_app/scenario_mapping.py @@ -16,6 +16,7 @@ from .upgrader_0807 import UpgradeTo0807 from .upgrader_0808 import UpgradeTo0808 from .upgrader_0900 import UpgradeTo0900 +from .upgrader_0902 import UpgradeTo0902 ALL_UPGRADE_METHODS = ( UpgradeTo0701(), @@ -30,6 +31,7 @@ UpgradeTo0807(), UpgradeTo0808(), UpgradeTo0900(), + UpgradeTo0902(), ) diff --git a/src/antares/study/version/upgrade_app/upgrader_0804.py b/src/antares/study/version/upgrade_app/upgrader_0804.py index 41070ee..4be8ffc 100644 --- a/src/antares/study/version/upgrade_app/upgrader_0804.py +++ b/src/antares/study/version/upgrade_app/upgrader_0804.py @@ -32,5 +32,5 @@ def upgrade(cls, study_dir: Path) -> None: data = GeneralData.from_ini_file(study_dir) actual_capacities = data["optimization"]["transmission-capacities"] data["optimization"]["transmission-capacities"] = _TRANSMISSION_CAPACITIES[actual_capacities] - del data["optimization"]["include-split-exported-mps"] + data["optimization"].pop("include-split-exported-mps", None) data.to_ini_file(study_dir) diff --git a/src/antares/study/version/upgrade_app/upgrader_0902.py b/src/antares/study/version/upgrade_app/upgrader_0902.py new file mode 100644 index 0000000..2b06df1 --- /dev/null +++ b/src/antares/study/version/upgrade_app/upgrader_0902.py @@ -0,0 +1,139 @@ +from itertools import product +from pathlib import Path + +import numpy as np +import typing as t + +from antares.study.version.ini_reader import IniReader +from antares.study.version.ini_writer import IniWriter +from antares.study.version.model.study_version import StudyVersion +from .exceptions import UnexpectedThematicTrimmingFieldsError + +from .upgrade_method import UpgradeMethod +from ..model.general_data import GENERAL_DATA_PATH, GeneralData + + +def _upgrade_thematic_trimming(data: GeneralData) -> None: + def _get_possible_variables() -> t.Set[str]: + groups = ["psp_open", "psp_closed", "pondage", "battery", "other1", "other2", "other3", "other4", "other5"] + outputs = ["injection", "withdrawal", "level"] + return {f"{group}_{output}" for group, output in product(groups, outputs)} + + variables_selection = data["variables selection"] + possible_variables = _get_possible_variables() + d: t.Dict[str, t.Dict[str, t.List[str]]] = {} + for sign in ["+", "-"]: + select_var = f"select_var {sign}" + d[select_var] = {"keep": [], "remove": []} + # The 'remove' list gathers all fields that should not be kept after the upgrade. + # It applies to any field inside the 27 listed by the `_get_possible_variables` method. + # The 'keep' list gathers all fields that have nothing to do with the upgrade and therefore should be kept. + # We check these fields for enabled and disabled variables (symbolized by +/-) as we can have both. + # In the end, we remove all legacy fields and replace them by one field only: 'STS by group'. + # For more information, see https://antares-simulator.readthedocs.io/en/latest/user-guide/04-migration-guides/#short-term-storage-groups + for var in variables_selection.get(select_var, []): + key = "remove" if var.lower() in possible_variables else "keep" + d[select_var][key].append(var) + + if d["select_var +"]["remove"] and d["select_var -"]["remove"]: + raise UnexpectedThematicTrimmingFieldsError(d["select_var +"]["remove"], d["select_var -"]["remove"]) + for sign in ["+", "-"]: + select_var = f"select_var {sign}" + if d[select_var]["keep"]: + d[select_var]["keep"].append("STS by group") + variables_selection[select_var] = d[select_var]["keep"] + + +class UpgradeTo0902(UpgradeMethod): + """ + This class upgrades the study from version 9.0 to version 9.2. + """ + + old = StudyVersion(9, 0) + new = StudyVersion(9, 2) + files = ["input/st-storage", GENERAL_DATA_PATH, "input/links"] + + @staticmethod + def _upgrade_general_data(study_dir: Path) -> None: + data = GeneralData.from_ini_file(study_dir) + adq_patch = data["adequacy patch"] + adq_patch.pop("enable-first-step", None) + adq_patch.pop("set-to-null-ntc-between-physical-out-for-first-step", None) + other_preferences = data["other preferences"] + other_preferences.pop("initial-reservoir-levels", None) + other_preferences["hydro-pmax-format"] = "daily" + data["general"]["nbtimeserieslinks"] = 1 + + if "variables selection" in data: + _upgrade_thematic_trimming(data) + + data.to_ini_file(study_dir) + + @staticmethod + def _upgrade_links(study_dir: Path) -> None: + links_path = study_dir / "input" / "links" + default_prepro = np.tile([1, 1, 0, 0, 0, 0], (365, 1)) + default_modulation = np.ones(dtype=int, shape=(8760, 1)) + for area in links_path.iterdir(): + area_path = links_path / area + capacity_folder = area_path / "capacities" + if not capacity_folder.exists(): + # the folder doesn't contain any existing link + continue + + ini_path = area_path / "properties.ini" + reader = IniReader() + writer = IniWriter() + sections = reader.read(ini_path) + area_names = [] + for area_name, section in sections.items(): + area_names.append(area_name) + section["unitcount"] = 1 + section["nominalcapacity"] = 0 + section["law.planned"] = "uniform" + section["law.forced"] = "uniform" + section["volatility.planned"] = 0 + section["volatility.forced"] = 0 + section["force-no-generation"] = True + writer.write(sections, ini_path) + + prepro_path = area_path / "prepro" + prepro_path.mkdir() + for area_name in area_names: + np.savetxt(prepro_path / f"{area_name}_direct.txt", default_prepro, delimiter="\t", fmt="%.6f") + np.savetxt(prepro_path / f"{area_name}_indirect.txt", default_prepro, delimiter="\t", fmt="%.6f") + np.savetxt(prepro_path / f"{area_name}_mod.txt", default_modulation, delimiter="\t", fmt="%.6f") + + @staticmethod + def _upgrade_storages(study_dir: Path) -> None: + st_storage_dir = study_dir / "input" / "st-storage" + reader = IniReader() + writer = IniWriter() + cluster_files = (st_storage_dir / "clusters").glob("*/list.ini") + for file_path in cluster_files: + sections = reader.read(file_path) + for section in sections.values(): + section["efficiencywithdrawal"] = 1 + writer.write(sections, file_path) + + matrices_to_create = ["cost-injection.txt", "cost-withdrawal.txt", "cost-level.txt"] + series_path = st_storage_dir / "series" + for area in series_path.iterdir(): + area_dir = st_storage_dir / "series" / area + for storage in area_dir.iterdir(): + final_dir = area_dir / storage + for matrix in matrices_to_create: + (final_dir / matrix).touch() + + @classmethod + def upgrade(cls, study_dir: Path) -> None: + """ + Upgrades the study to version 9.2. + + Args: + study_dir: The study directory. + """ + + cls._upgrade_general_data(study_dir) + cls._upgrade_links(study_dir) + cls._upgrade_storages(study_dir) diff --git a/tests/conftest.py b/tests/conftest.py index b7d1a9c..23c8571 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ STUDY_ANTARES_FILE = """\ [antares] caption = Thermal fleet optimization -version = 9.1 +version = 9.2 created = 1246524135 lastsave = 1686128483 author = John Doe diff --git a/tests/show_app/test_show_app.py b/tests/show_app/test_show_app.py index 1476b0f..87b5cf8 100644 --- a/tests/show_app/test_show_app.py +++ b/tests/show_app/test_show_app.py @@ -28,7 +28,7 @@ def test_study_antares(self, study_dir: Path) -> None: "caption": "Thermal fleet optimization", "created_date": datetime.datetime(2009, 7, 2, 8, 42, 15), "last_save_date": datetime.datetime(2023, 6, 7, 9, 1, 23), - "version": {"major": 9, "minor": 1, "patch": 0}, + "version": {"major": 9, "minor": 2, "patch": 0}, } assert dataclasses.asdict(actual) == expected diff --git a/tests/test_cli.py b/tests/test_cli.py index 66e434d..52c536b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,7 +33,7 @@ def test_cli__show(self, study_dir: Path) -> None: assert result.exit_code == 0 show_str = result.output.strip() assert "Caption: Thermal fleet optimization" in show_str - assert "Version: v9.1" in show_str + assert "Version: v9.2" in show_str assert "Created: 2009-07-02 08:42:15" in show_str assert "Last Save: 2023-06-07 09:01:23" in show_str assert "Author: John Doe" in show_str diff --git a/tests/upgrade_app/test_upgrade_0902.py b/tests/upgrade_app/test_upgrade_0902.py new file mode 100644 index 0000000..02ed064 --- /dev/null +++ b/tests/upgrade_app/test_upgrade_0902.py @@ -0,0 +1,28 @@ +from antares.study.version.model.general_data import GeneralData +from antares.study.version.upgrade_app.upgrader_0902 import UpgradeTo0902 +from tests.conftest import StudyAssets +from tests.helpers import are_same_dir + + +def test_hydro_format_legacy(study_assets: StudyAssets): + """ + Check that the files are correctly modified + """ + + # upgrade the study + UpgradeTo0902.upgrade(study_assets.study_dir) + + # compare generaldata.ini + actual = GeneralData.from_ini_file(study_assets.study_dir) + expected = GeneralData.from_ini_file(study_assets.expected_dir) + assert actual == expected + + # compare st-storage folders (st-storage) + actual_input_path = study_assets.study_dir / "input" / "st-storage" + expected_input_path = study_assets.expected_dir / "input" / "st-storage" + assert are_same_dir(actual_input_path, expected_input_path) + + # compare links folders + actual_input_path = study_assets.study_dir / "input" / "links" + expected_input_path = study_assets.expected_dir / "input" / "links" + assert are_same_dir(actual_input_path, expected_input_path) diff --git a/tests/upgrade_app/upgrade_0902/hydro_format_legacy/little_study_0900.expected.zip b/tests/upgrade_app/upgrade_0902/hydro_format_legacy/little_study_0900.expected.zip new file mode 100644 index 0000000..0b1bc46 Binary files /dev/null and b/tests/upgrade_app/upgrade_0902/hydro_format_legacy/little_study_0900.expected.zip differ diff --git a/tests/upgrade_app/upgrade_0902/hydro_format_legacy/little_study_0900.zip b/tests/upgrade_app/upgrade_0902/hydro_format_legacy/little_study_0900.zip new file mode 100644 index 0000000..de94bdb Binary files /dev/null and b/tests/upgrade_app/upgrade_0902/hydro_format_legacy/little_study_0900.zip differ