feat(upgrade): add v9.2 upgrader (#20)

src/antares/study/version/model/study_antares.py

@@ -23,7 +23,7 @@ class StudyAntares:
 
         [antares]
         caption = Thermal fleet optimization
-        version = 9.1
+        version = 9.2
         created = 1246524135
         lastsave = 1686128483
         author = John Doe
@@ -34,7 +34,7 @@ class StudyAntares:
 
     >>> data = {
     ...     "caption": "Thermal fleet optimization",
-    ...     "version": "9.1",
+    ...     "version": "9.2",
     ...     "created_date": 1246524135,
     ...     "last_save_date": 1686128483,
     ...     "author": "John Doe",
@@ -45,7 +45,7 @@ class StudyAntares:
     >>> study_antares.caption
     'Thermal fleet optimization'
     >>> study_antares.version
-    StudyVersion(major=9, minor=1, patch=0)
+    StudyVersion(major=9, minor=2, patch=0)
     >>> study_antares.created_date
     datetime.datetime(2009, 7, 2, 10, 42, 15)
     >>> study_antares.last_save_date

src/antares/study/version/upgrade_app/exceptions.py

@@ -1,3 +1,6 @@
+from typing import List
+
+
 class UpgradeError(Exception):
     """
     Base class for exceptions in this module.
@@ -22,3 +25,17 @@ def __init__(self, link_path: str):
             f" that allows to replace the matrix links by valid TSV matrices."
         )
         super().__init__(message)
+
+
+class UnexpectedThematicTrimmingFieldsError(UpgradeError):
+    """
+    Exception raised when there are unexpected thematic trimming fields in the generaldata.ini file.
+    """
+
+    def __init__(self, enabled_fields: List[str], disabled_fields: List[str]):
+        message = (
+            f"Found these enabled fields {enabled_fields} with these disabled fields {disabled_fields} in the"
+            f" generaldata.ini. We cannot determine if the new variable `STS by group` should be enabled or disabled."
+            f" Choose one before upgrading your study."
+        )
+        super().__init__(message)

src/antares/study/version/upgrade_app/scenario_mapping.py

@@ -16,6 +16,7 @@
 from .upgrader_0807 import UpgradeTo0807
 from .upgrader_0808 import UpgradeTo0808
 from .upgrader_0900 import UpgradeTo0900
+from .upgrader_0902 import UpgradeTo0902
 
 ALL_UPGRADE_METHODS = (
     UpgradeTo0701(),
@@ -30,6 +31,7 @@
     UpgradeTo0807(),
     UpgradeTo0808(),
     UpgradeTo0900(),
+    UpgradeTo0902(),
 )
 
 

src/antares/study/version/upgrade_app/upgrader_0804.py

@@ -32,5 +32,5 @@ def upgrade(cls, study_dir: Path) -> None:
         data = GeneralData.from_ini_file(study_dir)
         actual_capacities = data["optimization"]["transmission-capacities"]
         data["optimization"]["transmission-capacities"] = _TRANSMISSION_CAPACITIES[actual_capacities]
-        del data["optimization"]["include-split-exported-mps"]
+        data["optimization"].pop("include-split-exported-mps", None)
         data.to_ini_file(study_dir)

src/antares/study/version/upgrade_app/upgrader_0902.py

@@ -0,0 +1,139 @@
+from itertools import product
+from pathlib import Path
+
+import numpy as np
+import typing as t
+
+from antares.study.version.ini_reader import IniReader
+from antares.study.version.ini_writer import IniWriter
+from antares.study.version.model.study_version import StudyVersion
+from .exceptions import UnexpectedThematicTrimmingFieldsError
+
+from .upgrade_method import UpgradeMethod
+from ..model.general_data import GENERAL_DATA_PATH, GeneralData
+
+
+def _upgrade_thematic_trimming(data: GeneralData) -> None:
+    def _get_possible_variables() -> t.Set[str]:
+        groups = ["psp_open", "psp_closed", "pondage", "battery", "other1", "other2", "other3", "other4", "other5"]
+        outputs = ["injection", "withdrawal", "level"]
+        return {f"{group}_{output}" for group, output in product(groups, outputs)}
+
+    variables_selection = data["variables selection"]
+    possible_variables = _get_possible_variables()
+    d: t.Dict[str, t.Dict[str, t.List[str]]] = {}
+    for sign in ["+", "-"]:
+        select_var = f"select_var {sign}"
+        d[select_var] = {"keep": [], "remove": []}
+        # The 'remove' list gathers all fields that should not be kept after the upgrade.
+        # It applies to any field inside the 27 listed by the `_get_possible_variables` method.
+        # The 'keep' list gathers all fields that have nothing to do with the upgrade and therefore should be kept.
+        # We check these fields for enabled and disabled variables (symbolized by +/-) as we can have both.
+        # In the end, we remove all legacy fields and replace them by one field only: 'STS by group'.
+        # For more information, see https://antares-simulator.readthedocs.io/en/latest/user-guide/04-migration-guides/#short-term-storage-groups
+        for var in variables_selection.get(select_var, []):
+            key = "remove" if var.lower() in possible_variables else "keep"
+            d[select_var][key].append(var)
+
+    if d["select_var +"]["remove"] and d["select_var -"]["remove"]:
+        raise UnexpectedThematicTrimmingFieldsError(d["select_var +"]["remove"], d["select_var -"]["remove"])
+    for sign in ["+", "-"]:
+        select_var = f"select_var {sign}"
+        if d[select_var]["keep"]:
+            d[select_var]["keep"].append("STS by group")
+            variables_selection[select_var] = d[select_var]["keep"]
+
+
+class UpgradeTo0902(UpgradeMethod):
+    """
+    This class upgrades the study from version 9.0 to version 9.2.
+    """
+
+    old = StudyVersion(9, 0)
+    new = StudyVersion(9, 2)
+    files = ["input/st-storage", GENERAL_DATA_PATH, "input/links"]
+
+    @staticmethod
+    def _upgrade_general_data(study_dir: Path) -> None:
+        data = GeneralData.from_ini_file(study_dir)
+        adq_patch = data["adequacy patch"]
+        adq_patch.pop("enable-first-step", None)
+        adq_patch.pop("set-to-null-ntc-between-physical-out-for-first-step", None)
+        other_preferences = data["other preferences"]
+        other_preferences.pop("initial-reservoir-levels", None)
+        other_preferences["hydro-pmax-format"] = "daily"
+        data["general"]["nbtimeserieslinks"] = 1
+
+        if "variables selection" in data:
+            _upgrade_thematic_trimming(data)
+
+        data.to_ini_file(study_dir)
+
+    @staticmethod
+    def _upgrade_links(study_dir: Path) -> None:
+        links_path = study_dir / "input" / "links"
+        default_prepro = np.tile([1, 1, 0, 0, 0, 0], (365, 1))
+        default_modulation = np.ones(dtype=int, shape=(8760, 1))
+        for area in links_path.iterdir():
+            area_path = links_path / area
+            capacity_folder = area_path / "capacities"
+            if not capacity_folder.exists():
+                # the folder doesn't contain any existing link
+                continue
+
+            ini_path = area_path / "properties.ini"
+            reader = IniReader()
+            writer = IniWriter()
+            sections = reader.read(ini_path)
+            area_names = []
+            for area_name, section in sections.items():
+                area_names.append(area_name)
+                section["unitcount"] = 1
+                section["nominalcapacity"] = 0
+                section["law.planned"] = "uniform"
+                section["law.forced"] = "uniform"
+                section["volatility.planned"] = 0
+                section["volatility.forced"] = 0
+                section["force-no-generation"] = True
+            writer.write(sections, ini_path)
+
+            prepro_path = area_path / "prepro"
+            prepro_path.mkdir()
+            for area_name in area_names:
+                np.savetxt(prepro_path / f"{area_name}_direct.txt", default_prepro, delimiter="\t", fmt="%.6f")
+                np.savetxt(prepro_path / f"{area_name}_indirect.txt", default_prepro, delimiter="\t", fmt="%.6f")
+                np.savetxt(prepro_path / f"{area_name}_mod.txt", default_modulation, delimiter="\t", fmt="%.6f")
+
+    @staticmethod
+    def _upgrade_storages(study_dir: Path) -> None:
+        st_storage_dir = study_dir / "input" / "st-storage"
+        reader = IniReader()
+        writer = IniWriter()
+        cluster_files = (st_storage_dir / "clusters").glob("*/list.ini")
+        for file_path in cluster_files:
+            sections = reader.read(file_path)
+            for section in sections.values():
+                section["efficiencywithdrawal"] = 1
+            writer.write(sections, file_path)
+
+        matrices_to_create = ["cost-injection.txt", "cost-withdrawal.txt", "cost-level.txt"]
+        series_path = st_storage_dir / "series"
+        for area in series_path.iterdir():
+            area_dir = st_storage_dir / "series" / area
+            for storage in area_dir.iterdir():
+                final_dir = area_dir / storage
+                for matrix in matrices_to_create:
+                    (final_dir / matrix).touch()
+
+    @classmethod
+    def upgrade(cls, study_dir: Path) -> None:
+        """
+        Upgrades the study to version 9.2.
+
+        Args:
+            study_dir: The study directory.
+        """
+
+        cls._upgrade_general_data(study_dir)
+        cls._upgrade_links(study_dir)
+        cls._upgrade_storages(study_dir)

tests/conftest.py

@@ -7,7 +7,7 @@
 STUDY_ANTARES_FILE = """\
 [antares]
 caption = Thermal fleet optimization
-version = 9.1
+version = 9.2
 created = 1246524135
 lastsave = 1686128483
 author = John Doe

tests/show_app/test_show_app.py

@@ -28,7 +28,7 @@ def test_study_antares(self, study_dir: Path) -> None:
             "caption": "Thermal fleet optimization",
             "created_date": datetime.datetime(2009, 7, 2, 8, 42, 15),
             "last_save_date": datetime.datetime(2023, 6, 7, 9, 1, 23),
-            "version": {"major": 9, "minor": 1, "patch": 0},
+            "version": {"major": 9, "minor": 2, "patch": 0},
         }
         assert dataclasses.asdict(actual) == expected
 

tests/test_cli.py

@@ -33,7 +33,7 @@ def test_cli__show(self, study_dir: Path) -> None:
         assert result.exit_code == 0
         show_str = result.output.strip()
         assert "Caption: Thermal fleet optimization" in show_str
-        assert "Version: v9.1" in show_str
+        assert "Version: v9.2" in show_str
         assert "Created: 2009-07-02 08:42:15" in show_str
         assert "Last Save: 2023-06-07 09:01:23" in show_str
         assert "Author: John Doe" in show_str

tests/upgrade_app/test_upgrade_0902.py

@@ -0,0 +1,28 @@
+from antares.study.version.model.general_data import GeneralData
+from antares.study.version.upgrade_app.upgrader_0902 import UpgradeTo0902
+from tests.conftest import StudyAssets
+from tests.helpers import are_same_dir
+
+
+def test_hydro_format_legacy(study_assets: StudyAssets):
+    """
+    Check that the files are correctly modified
+    """
+
+    # upgrade the study
+    UpgradeTo0902.upgrade(study_assets.study_dir)
+
+    # compare generaldata.ini
+    actual = GeneralData.from_ini_file(study_assets.study_dir)
+    expected = GeneralData.from_ini_file(study_assets.expected_dir)
+    assert actual == expected
+
+    # compare st-storage folders (st-storage)
+    actual_input_path = study_assets.study_dir / "input" / "st-storage"
+    expected_input_path = study_assets.expected_dir / "input" / "st-storage"
+    assert are_same_dir(actual_input_path, expected_input_path)
+
+    # compare links folders
+    actual_input_path = study_assets.study_dir / "input" / "links"
+    expected_input_path = study_assets.expected_dir / "input" / "links"
+    assert are_same_dir(actual_input_path, expected_input_path)