From c60a318e0b1dc0e1489612d856000d8806f9ac84 Mon Sep 17 00:00:00 2001 From: MartinBelthle <102529366+MartinBelthle@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:40:16 +0100 Subject: [PATCH] fix(xpansion): catch Exception when no sensitvity folder in xpansion (#1961) Resolves [ANT-1216] --- .../study/business/xpansion_management.py | 2 +- antarest/study/storage/rawstudy/ini_reader.py | 2 +- .../rawstudy/model/filesystem/bucket_node.py | 46 ++++--- .../rawstudy/model/filesystem/folder_node.py | 32 ++--- .../model/filesystem/json_file_node.py | 40 ++++++- .../root/user/expansion/expansion.py | 6 +- .../study/storage/study_upgrader/__init__.py | 47 +++----- .../storage/business/test_xpansion_manager.py | 101 ++++++---------- .../repository/filesystem/test_folder_node.py | 113 ++++++++++++++++-- 9 files changed, 230 insertions(+), 159 deletions(-) diff --git a/antarest/study/business/xpansion_management.py b/antarest/study/business/xpansion_management.py index f3adadad32..1bb80cfbaf 100644 --- a/antarest/study/business/xpansion_management.py +++ b/antarest/study/business/xpansion_management.py @@ -365,7 +365,7 @@ def get_xpansion_settings(self, study: Study) -> GetXpansionSettings: logger.info(f"Getting xpansion settings for study '{study.id}'") file_study = self.study_storage_service.get_storage(study).get_raw(study) config_obj = file_study.tree.get(["user", "expansion", "settings"]) - with contextlib.suppress(KeyError): + with contextlib.suppress(ChildNotFoundError): config_obj["sensitivity_config"] = file_study.tree.get( ["user", "expansion", "sensitivity", "sensitivity_in"] ) diff --git a/antarest/study/storage/rawstudy/ini_reader.py b/antarest/study/storage/rawstudy/ini_reader.py index f145b948a9..84be7c099b 100644 --- a/antarest/study/storage/rawstudy/ini_reader.py +++ b/antarest/study/storage/rawstudy/ini_reader.py @@ -109,7 +109,7 @@ def read(self, path: t.Any) -> JSON: sections = self._parse_ini_file(f) except FileNotFoundError: # If the file is missing, an empty dictionary is returned. - # This is required tp mimic the behavior of `configparser.ConfigParser`. + # This is required to mimic the behavior of `configparser.ConfigParser`. return {} elif hasattr(path, "read"): diff --git a/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py b/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py index 3415edd1c3..6106326d2f 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Dict, List, Optional +import typing as t from antarest.core.model import JSON, SUB_JSON from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig @@ -12,7 +12,7 @@ class RegisteredFile: def __init__( self, key: str, - node: Optional[Callable[[ContextServer, FileStudyTreeConfig], INode[Any, Any, Any]]], + node: t.Optional[t.Callable[[ContextServer, FileStudyTreeConfig], INode[t.Any, t.Any, t.Any]]], filename: str = "", ): self.key = key @@ -29,42 +29,36 @@ def __init__( self, context: ContextServer, config: FileStudyTreeConfig, - registered_files: Optional[List[RegisteredFile]] = None, - default_file_node: Callable[..., INode[Any, Any, Any]] = RawFileNode, + registered_files: t.Optional[t.List[RegisteredFile]] = None, + default_file_node: t.Callable[..., INode[t.Any, t.Any, t.Any]] = RawFileNode, ): super().__init__(context, config) - self.registered_files: List[RegisteredFile] = registered_files or [] - self.default_file_node: Callable[..., INode[Any, Any, Any]] = default_file_node + self.registered_files: t.List[RegisteredFile] = registered_files or [] + self.default_file_node: t.Callable[..., INode[t.Any, t.Any, t.Any]] = default_file_node - def _get_registered_file(self, key: str) -> Optional[RegisteredFile]: - for registered_file in self.registered_files: - if registered_file.key == key: - return registered_file - return None + def _get_registered_file_by_key(self, key: str) -> t.Optional[RegisteredFile]: + return next((rf for rf in self.registered_files if rf.key == key), None) - def _get_registered_file_from_filename(self, filename: str) -> Optional[RegisteredFile]: - for registered_file in self.registered_files: - if registered_file.filename == filename: - return registered_file - return None + def _get_registered_file_by_filename(self, filename: str) -> t.Optional[RegisteredFile]: + return next((rf for rf in self.registered_files if rf.filename == filename), None) def save( self, data: SUB_JSON, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, ) -> None: self._assert_not_in_zipped_file() if not self.config.path.exists(): self.config.path.mkdir() - if url is None or len(url) == 0: - assert isinstance(data, Dict) + if not url: + assert isinstance(data, dict) for key, value in data.items(): self._save(value, key) else: key = url[0] if len(url) > 1: - registered_file = self._get_registered_file(key) + registered_file = self._get_registered_file_by_key(key) if registered_file: node = registered_file.node or self.default_file_node node(self.context, self.config.next_file(key)).save(data, url[1:]) @@ -74,7 +68,7 @@ def save( self._save(data, key) def _save(self, data: SUB_JSON, key: str) -> None: - registered_file = self._get_registered_file(key) + registered_file = self._get_registered_file_by_key(key) if registered_file: node, filename = ( registered_file.node or self.default_file_node, @@ -88,12 +82,12 @@ def _save(self, data: SUB_JSON, key: str) -> None: BucketNode(self.context, self.config.next_file(key)).save(data) def build(self) -> TREE: - if not self.config.path.exists(): - return dict() + if not self.config.path.is_dir(): + return {} children: TREE = {} for item in sorted(self.config.path.iterdir()): - registered_file = self._get_registered_file_from_filename(item.name) + registered_file = self._get_registered_file_by_filename(item.name) if registered_file: node = registered_file.node or self.default_file_node children[registered_file.key] = node(self.context, self.config.next_file(item.name)) @@ -107,7 +101,7 @@ def build(self) -> TREE: def check_errors( self, data: JSON, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, raising: bool = False, - ) -> List[str]: + ) -> t.List[str]: return [] diff --git a/antarest/study/storage/rawstudy/model/filesystem/folder_node.py b/antarest/study/storage/rawstudy/model/filesystem/folder_node.py index 7d174cfbc6..3ea51c098d 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/folder_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/folder_node.py @@ -1,7 +1,7 @@ import shutil +import typing as t from abc import ABC, abstractmethod from http import HTTPStatus -from typing import Dict, List, Optional, Tuple, Union from fastapi import HTTPException @@ -38,7 +38,7 @@ def __init__( self, context: ContextServer, config: FileStudyTreeConfig, - children_glob_exceptions: Optional[List[str]] = None, + children_glob_exceptions: t.Optional[t.List[str]] = None, ) -> None: super().__init__(config) self.context = context @@ -50,11 +50,11 @@ def build(self) -> TREE: def _forward_get( self, - url: List[str], + url: t.List[str], depth: int = -1, formatted: bool = True, get_node: bool = False, - ) -> Union[JSON, INode[JSON, SUB_JSON, JSON]]: + ) -> t.Union[JSON, INode[JSON, SUB_JSON, JSON]]: children = self.build() names, sub_url = self.extract_child(children, url) @@ -84,7 +84,7 @@ def _forward_get( def _expand_get( self, depth: int = -1, formatted: bool = True, get_node: bool = False - ) -> Union[JSON, INode[JSON, SUB_JSON, JSON]]: + ) -> t.Union[JSON, INode[JSON, SUB_JSON, JSON]]: if get_node: return self @@ -99,11 +99,11 @@ def _expand_get( def _get( self, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, depth: int = -1, formatted: bool = True, get_node: bool = False, - ) -> Union[JSON, INode[JSON, SUB_JSON, JSON]]: + ) -> t.Union[JSON, INode[JSON, SUB_JSON, JSON]]: if url and url != [""]: return self._forward_get(url, depth, formatted, get_node) else: @@ -111,7 +111,7 @@ def _get( def get( self, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, depth: int = -1, expanded: bool = False, formatted: bool = True, @@ -122,7 +122,7 @@ def get( def get_node( self, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, ) -> INode[JSON, SUB_JSON, JSON]: output = self._get(url=url, get_node=True) assert isinstance(output, INode) @@ -131,7 +131,7 @@ def get_node( def save( self, data: SUB_JSON, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, ) -> None: self._assert_not_in_zipped_file() children = self.build() @@ -146,7 +146,7 @@ def save( for key in data: children[key].save(data[key]) - def delete(self, url: Optional[List[str]] = None) -> None: + def delete(self, url: t.Optional[t.List[str]] = None) -> None: if url and url != [""]: children = self.build() names, sub_url = self.extract_child(children, url) @@ -158,16 +158,16 @@ def delete(self, url: Optional[List[str]] = None) -> None: def check_errors( self, data: JSON, - url: Optional[List[str]] = None, + url: t.Optional[t.List[str]] = None, raising: bool = False, - ) -> List[str]: + ) -> t.List[str]: children = self.build() if url and url != [""]: (name,), sub_url = self.extract_child(children, url) return children[name].check_errors(data, sub_url, raising) else: - errors: List[str] = [] + errors: t.List[str] = [] for key in data: if key not in children: msg = f"key={key} not in {list(children.keys())} for {self.__class__.__name__}" @@ -186,7 +186,7 @@ def denormalize(self) -> None: for child in self.build().values(): child.denormalize() - def extract_child(self, children: TREE, url: List[str]) -> Tuple[List[str], List[str]]: + def extract_child(self, children: TREE, url: t.List[str]) -> t.Tuple[t.List[str], t.List[str]]: names, sub_url = url[0].split(","), url[1:] names = ( list( @@ -208,6 +208,6 @@ def extract_child(self, children: TREE, url: List[str]) -> Tuple[List[str], List for name in names: if name not in children: raise ChildNotFoundError(f"'{name}' not a child of {self.__class__.__name__}") - if type(children[name]) != child_class: + if not isinstance(children[name], child_class): raise FilterError("Filter selection has different classes") return names, sub_url diff --git a/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py b/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py index ee14da91ea..80f607485d 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py @@ -1,6 +1,6 @@ import json +import typing as t from pathlib import Path -from typing import Any, Dict, Optional, cast from antarest.core.model import JSON from antarest.study.storage.rawstudy.ini_reader import IReader @@ -11,13 +11,41 @@ class JsonReader(IReader): - def read(self, path: Any) -> JSON: - if isinstance(path, Path): - return cast(JSON, json.loads(path.read_text(encoding="utf-8"))) - return cast(JSON, json.loads(path)) + """ + JSON file reader. + """ + + def read(self, path: t.Any) -> JSON: + content: t.Union[str, bytes] + + if isinstance(path, (Path, str)): + try: + with open(path, mode="r", encoding="utf-8") as f: + content = f.read() + except FileNotFoundError: + # If the file is missing, an empty dictionary is returned, + # to mimic the behavior of `configparser.ConfigParser`. + return {} + + elif hasattr(path, "read"): + with path: + content = path.read() + + else: # pragma: no cover + raise TypeError(repr(type(path))) + + try: + return t.cast(JSON, json.loads(content)) + except json.JSONDecodeError as exc: + err_msg = f"Failed to parse JSON file '{path}'" + raise ValueError(err_msg) from exc class JsonWriter(IniWriter): + """ + JSON file writer. + """ + def write(self, data: JSON, path: Path) -> None: with open(path, "w") as fh: json.dump(data, fh) @@ -28,6 +56,6 @@ def __init__( self, context: ContextServer, config: FileStudyTreeConfig, - types: Optional[Dict[str, Any]] = None, + types: t.Optional[t.Dict[str, t.Any]] = None, ) -> None: super().__init__(context, config, types, JsonReader(), JsonWriter()) diff --git a/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/expansion.py b/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/expansion.py index 2b7414234f..c38246f6cf 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/expansion.py +++ b/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/expansion.py @@ -14,11 +14,7 @@ class Expansion(BucketNode): registered_files = [ - RegisteredFile( - key="candidates", - node=ExpansionCandidates, - filename="candidates.ini", - ), + RegisteredFile(key="candidates", node=ExpansionCandidates, filename="candidates.ini"), RegisteredFile(key="settings", node=ExpansionSettings, filename="settings.ini"), RegisteredFile(key="capa", node=ExpansionMatrixResources), RegisteredFile(key="weights", node=ExpansionMatrixResources), diff --git a/antarest/study/storage/study_upgrader/__init__.py b/antarest/study/storage/study_upgrader/__init__.py index 1993b4a0c3..3f53def629 100644 --- a/antarest/study/storage/study_upgrader/__init__.py +++ b/antarest/study/storage/study_upgrader/__init__.py @@ -3,10 +3,10 @@ import shutil import tempfile import time +import typing as t from http import HTTPStatus from http.client import HTTPException from pathlib import Path -from typing import Callable, List, NamedTuple from antarest.core.exceptions import StudyValidationError @@ -23,40 +23,27 @@ logger = logging.getLogger(__name__) -class UpgradeMethod(NamedTuple): +class UpgradeMethod(t.NamedTuple): """Raw study upgrade method (old version, new version, upgrade function).""" old: str new: str - method: Callable[[Path], None] - files: List[Path] + method: t.Callable[[Path], None] + files: t.List[Path] +_GENERAL_DATA_PATH = Path("settings/generaldata.ini") + UPGRADE_METHODS = [ - UpgradeMethod("700", "710", upgrade_710, [Path("settings/generaldata.ini")]), + UpgradeMethod("700", "710", upgrade_710, [_GENERAL_DATA_PATH]), UpgradeMethod("710", "720", upgrade_720, []), - UpgradeMethod("720", "800", upgrade_800, [Path("settings/generaldata.ini")]), - UpgradeMethod( - "800", - "810", - upgrade_810, - [Path("settings/generaldata.ini"), Path("input")], - ), + UpgradeMethod("720", "800", upgrade_800, [_GENERAL_DATA_PATH]), + UpgradeMethod("800", "810", upgrade_810, [_GENERAL_DATA_PATH, Path("input")]), UpgradeMethod("810", "820", upgrade_820, [Path("input/links")]), - UpgradeMethod( - "820", - "830", - upgrade_830, - [Path("settings/generaldata.ini"), Path("input/areas")], - ), - UpgradeMethod("830", "840", upgrade_840, [Path("settings/generaldata.ini")]), - UpgradeMethod("840", "850", upgrade_850, [Path("settings/generaldata.ini")]), - UpgradeMethod( - "850", - "860", - upgrade_860, - [Path("input"), Path("settings/generaldata.ini")], - ), + UpgradeMethod("820", "830", upgrade_830, [_GENERAL_DATA_PATH, Path("input/areas")]), + UpgradeMethod("830", "840", upgrade_840, [_GENERAL_DATA_PATH]), + UpgradeMethod("840", "850", upgrade_850, [_GENERAL_DATA_PATH]), + UpgradeMethod("850", "860", upgrade_860, [Path("input"), _GENERAL_DATA_PATH]), ] @@ -127,7 +114,7 @@ def get_current_version(study_path: Path) -> str: ) -def can_upgrade_version(from_version: str, to_version: str) -> List[Path]: +def can_upgrade_version(from_version: str, to_version: str) -> t.List[Path]: """ Checks if upgrading from one version to another is possible. @@ -190,7 +177,7 @@ def _update_study_antares_file(target_version: str, study_path: Path) -> None: file.write_text(content, encoding="utf-8") -def _copies_only_necessary_files(files_to_upgrade: List[Path], study_path: Path, tmp_path: Path) -> List[Path]: +def _copies_only_necessary_files(files_to_upgrade: t.List[Path], study_path: Path, tmp_path: Path) -> t.List[Path]: """ Copies files concerned by the version upgrader into a temporary directory. Args: @@ -221,7 +208,7 @@ def _copies_only_necessary_files(files_to_upgrade: List[Path], study_path: Path, return files_to_retrieve -def _filters_out_children_files(files_to_upgrade: List[Path]) -> List[Path]: +def _filters_out_children_files(files_to_upgrade: t.List[Path]) -> t.List[Path]: """ Filters out children paths of "input" if "input" is already in the list. Args: @@ -237,7 +224,7 @@ def _filters_out_children_files(files_to_upgrade: List[Path]) -> List[Path]: return files_to_upgrade -def _replace_safely_original_files(files_to_replace: List[Path], study_path: Path, tmp_path: Path) -> None: +def _replace_safely_original_files(files_to_replace: t.List[Path], study_path: Path, tmp_path: Path) -> None: """ Replace files/folders of the study that should be upgraded by their copy already upgraded in the tmp directory. It uses Path.rename() and an intermediary tmp directory to swap the folders safely. diff --git a/tests/storage/business/test_xpansion_manager.py b/tests/storage/business/test_xpansion_manager.py index 1703325e8b..bb5651bcbd 100644 --- a/tests/storage/business/test_xpansion_manager.py +++ b/tests/storage/business/test_xpansion_manager.py @@ -81,6 +81,14 @@ def make_link_and_areas(empty_study: FileStudy) -> None: make_link(empty_study) +def set_up_xpansion_manager(tmp_path: Path) -> t.Tuple[FileStudy, RawStudy, XpansionManager]: + empty_study = make_empty_study(tmp_path, 810) + study = RawStudy(id="1", path=str(empty_study.config.study_path), version="810") + xpansion_manager = make_xpansion_manager(empty_study) + xpansion_manager.create_xpansion_configuration(study) + return empty_study, study, xpansion_manager + + @pytest.mark.unit_test @pytest.mark.parametrize( "version, expected_output", @@ -117,7 +125,7 @@ def test_create_configuration(tmp_path: Path, version: int, expected_output: JSO Test the creation of a configuration. """ empty_study = make_empty_study(tmp_path, version) - study = RawStudy(id="1", path=empty_study.config.study_path, version=version) + study = RawStudy(id="1", path=str(empty_study.config.study_path), version=str(version)) xpansion_manager = make_xpansion_manager(empty_study) with pytest.raises(ChildNotFoundError): @@ -135,7 +143,7 @@ def test_delete_xpansion_configuration(tmp_path: Path) -> None: Test the deletion of a configuration. """ empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) + study = RawStudy(id="1", path=str(empty_study.config.study_path), version="810") xpansion_manager = make_xpansion_manager(empty_study) with pytest.raises(ChildNotFoundError): @@ -183,7 +191,7 @@ def test_get_xpansion_settings(tmp_path: Path, version: int, expected_output: JS """ empty_study = make_empty_study(tmp_path, version) - study = RawStudy(id="1", path=empty_study.config.study_path, version=version) + study = RawStudy(id="1", path=str(empty_study.config.study_path), version=str(version)) xpansion_manager = make_xpansion_manager(empty_study) xpansion_manager.create_xpansion_configuration(study) @@ -197,12 +205,7 @@ def test_update_xpansion_settings(tmp_path: Path) -> None: """ Test the retrieval of the xpansion settings. """ - - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - - xpansion_manager.create_xpansion_configuration(study) + _, study, xpansion_manager = set_up_xpansion_manager(tmp_path) new_settings_obj = { "optimality_gap": 4.0, @@ -246,10 +249,7 @@ def test_update_xpansion_settings(tmp_path: Path) -> None: @pytest.mark.unit_test def test_add_candidate(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) actual = empty_study.tree.get(["user", "expansion", "candidates"]) assert actual == {} @@ -298,10 +298,7 @@ def test_add_candidate(tmp_path: Path) -> None: @pytest.mark.unit_test def test_get_candidate(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) assert empty_study.tree.get(["user", "expansion", "candidates"]) == {} @@ -334,10 +331,7 @@ def test_get_candidate(tmp_path: Path) -> None: @pytest.mark.unit_test def test_get_candidates(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) assert empty_study.tree.get(["user", "expansion", "candidates"]) == {} @@ -372,10 +366,7 @@ def test_get_candidates(tmp_path: Path) -> None: @pytest.mark.unit_test def test_update_candidates(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) assert empty_study.tree.get(["user", "expansion", "candidates"]) == {} @@ -406,10 +397,7 @@ def test_update_candidates(tmp_path: Path) -> None: @pytest.mark.unit_test def test_delete_candidate(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) assert empty_study.tree.get(["user", "expansion", "candidates"]) == {} @@ -442,10 +430,7 @@ def test_delete_candidate(tmp_path: Path) -> None: @pytest.mark.unit_test def test_update_constraints(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) with pytest.raises(XpansionFileNotFoundError): xpansion_manager.update_xpansion_constraints_settings(study=study, constraints_file_name="non_existent_file") @@ -464,10 +449,7 @@ def test_update_constraints(tmp_path: Path) -> None: @pytest.mark.unit_test def test_add_resources(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) filename1 = "constraints1.txt" filename2 = "constraints2.txt" @@ -520,10 +502,8 @@ def test_add_resources(tmp_path: Path) -> None: @pytest.mark.unit_test def test_list_root_resources(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) + constraints_file_content = b"0" constraints_file_name = "unknownfile.txt" @@ -533,10 +513,7 @@ def test_list_root_resources(tmp_path: Path) -> None: @pytest.mark.unit_test def test_get_single_constraints(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) constraints_file_content = b"0" constraints_file_name = "constraints.txt" @@ -549,12 +526,18 @@ def test_get_single_constraints(tmp_path: Path) -> None: ) +@pytest.mark.unit_test +def test_get_settings_without_sensitivity(tmp_path: Path) -> None: + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) + + empty_study.tree.delete(["user", "expansion", "sensitivity"]) + # should not fail even if the folder doesn't exist as it's optional + xpansion_manager.get_xpansion_settings(study) + + @pytest.mark.unit_test def test_get_all_constraints(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + _, study, xpansion_manager = set_up_xpansion_manager(tmp_path) filename1 = "constraints1.txt" filename2 = "constraints2.txt" @@ -576,10 +559,7 @@ def test_get_all_constraints(tmp_path: Path) -> None: @pytest.mark.unit_test def test_add_capa(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) filename1 = "capa1.txt" filename2 = "capa2.txt" @@ -610,10 +590,7 @@ def test_add_capa(tmp_path: Path) -> None: @pytest.mark.unit_test def test_delete_capa(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + empty_study, study, xpansion_manager = set_up_xpansion_manager(tmp_path) filename1 = "capa1.txt" filename2 = "capa2.txt" @@ -636,10 +613,7 @@ def test_delete_capa(tmp_path: Path) -> None: @pytest.mark.unit_test def test_get_single_capa(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + _, study, xpansion_manager = set_up_xpansion_manager(tmp_path) filename1 = "capa1.txt" filename2 = "capa2.txt" @@ -664,10 +638,7 @@ def test_get_single_capa(tmp_path: Path) -> None: @pytest.mark.unit_test def test_get_all_capa(tmp_path: Path) -> None: - empty_study = make_empty_study(tmp_path, 810) - study = RawStudy(id="1", path=empty_study.config.study_path, version=810) - xpansion_manager = make_xpansion_manager(empty_study) - xpansion_manager.create_xpansion_configuration(study) + _, study, xpansion_manager = set_up_xpansion_manager(tmp_path) filename1 = "capa1.txt" filename2 = "capa2.txt" diff --git a/tests/storage/repository/filesystem/test_folder_node.py b/tests/storage/repository/filesystem/test_folder_node.py index a01214eedb..d08360e223 100644 --- a/tests/storage/repository/filesystem/test_folder_node.py +++ b/tests/storage/repository/filesystem/test_folder_node.py @@ -1,9 +1,14 @@ +import json +import textwrap +import typing as t from pathlib import Path from unittest.mock import Mock import pytest from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig +from antarest.study.storage.rawstudy.model.filesystem.factory import StudyFactory +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ChildNotFoundError from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode from antarest.study.storage.rawstudy.model.filesystem.inode import INode from antarest.study.storage.rawstudy.model.filesystem.raw_file_node import RawFileNode @@ -11,7 +16,7 @@ from tests.storage.repository.filesystem.utils import TestMiddleNode, TestSubNode -def build_tree() -> INode: +def build_tree() -> INode[t.Any, t.Any, t.Any]: config = Mock() config.path.exist.return_value = True config.zip_path = None @@ -26,7 +31,7 @@ def build_tree() -> INode: @pytest.mark.unit_test -def test_get(): +def test_get() -> None: tree = build_tree() res = tree.get(["input"]) @@ -37,7 +42,97 @@ def test_get(): @pytest.mark.unit_test -def test_get_depth(): +def test_get_input_areas_sets(tmp_path: Path) -> None: + """ + Read the content of the `sets.ini` file in the `input/areas` directory. + The goal of this test is to verify the behavior of the `get` method of the `FileStudyTree` class + for the case where the subdirectories or the INI file do not exist. + """ + + study_factory = StudyFactory(Mock(), Mock(), Mock()) + study_id = "c5633166-afe1-4ce5-9305-75bc2779aad6" + file_study = study_factory.create_from_fs(tmp_path, study_id, use_cache=False) + url = ["input", "areas", "sets"] # sets.ini + + # Empty study tree structure + actual = file_study.tree.get(url) + assert actual == {} + + # Add the "settings" directory + tmp_path.joinpath("input").mkdir() + actual = file_study.tree.get(url) + assert actual == {} + + # Add the "areas" directory + tmp_path.joinpath("input/areas").mkdir() + actual = file_study.tree.get(url) + assert actual == {} + + # Add the "sets.ini" file + sets = textwrap.dedent( + """\ + [all areas] + caption = All areas + comments = Spatial aggregates on all areas + output = false + apply-filter = add-all + """ + ) + tmp_path.joinpath("input/areas/sets.ini").write_text(sets) + actual = file_study.tree.get(url) + expected = { + "all areas": { + "caption": "All areas", + "comments": "Spatial aggregates on all areas", + "output": False, + "apply-filter": "add-all", + } + } + assert actual == expected + + +@pytest.mark.unit_test +def test_get_user_expansion_sensitivity_sensitivity_in(tmp_path: Path) -> None: + """ + Read the content of the `sensitivity_in.json` file in the `user/expansion/sensitivity` directory. + The goal of this test is to verify the behavior of the `get` method of the `FileStudyTree` class + for the case where the subdirectories or the JSON file do not exist. + """ + + study_factory = StudyFactory(Mock(), Mock(), Mock()) + study_id = "616ac707-c108-47af-9e02-c37cc043511a" + file_study = study_factory.create_from_fs(tmp_path, study_id, use_cache=False) + url = ["user", "expansion", "sensitivity", "sensitivity_in"] + + # Empty study tree structure + # fixme: bad error message + with pytest.raises(ChildNotFoundError, match=r"'expansion' not a child of User"): + file_study.tree.get(url) + + # Add the "user" directory + tmp_path.joinpath("user").mkdir() + with pytest.raises(ChildNotFoundError, match=r"'expansion' not a child of User"): + file_study.tree.get(url) + + # Add the "expansion" directory + tmp_path.joinpath("user/expansion").mkdir() + with pytest.raises(ChildNotFoundError, match=r"'sensitivity' not a child of Expansion"): + file_study.tree.get(url) + + # Add the "sensitivity" directory + tmp_path.joinpath("user/expansion/sensitivity").mkdir() + actual = file_study.tree.get(url) + assert actual == {} + + # Add the "sensitivity_in.json" file + sensitivity_obj = {"epsilon": 10000.0, "projection": ["pv", "battery"], "capex": True} + tmp_path.joinpath("user/expansion/sensitivity/sensitivity_in.json").write_text(json.dumps(sensitivity_obj)) + actual_obj = file_study.tree.get(url) + assert actual_obj == sensitivity_obj + + +@pytest.mark.unit_test +def test_get_depth() -> None: config = Mock() config.path.exist.return_value = True tree = TestMiddleNode( @@ -46,7 +141,7 @@ def test_get_depth(): children={"childA": build_tree(), "childB": build_tree()}, ) - expected = { + expected: t.Dict[str, t.Dict[str, t.Any]] = { "childA": {}, "childB": {}, } @@ -54,7 +149,7 @@ def test_get_depth(): assert tree.get(depth=1) == expected -def test_validate(): +def test_validate() -> None: config = Mock() config.path.exist.return_value = True tree = TestMiddleNode( @@ -77,7 +172,7 @@ def test_validate(): @pytest.mark.unit_test -def test_save(): +def test_save() -> None: tree = build_tree() tree.save(105, ["output"]) @@ -88,7 +183,7 @@ def test_save(): @pytest.mark.unit_test -def test_filter(): +def test_filter() -> None: tree = build_tree() expected_json = { @@ -100,7 +195,7 @@ def test_filter(): assert tree.get(["*", "value"]) == expected_json -def test_delete(tmp_path: Path): +def test_delete(tmp_path: Path) -> None: folder_node = tmp_path / "folder_node" folder_node.mkdir() sub_folder = folder_node / "sub_folder" @@ -124,7 +219,7 @@ def test_delete(tmp_path: Path): assert folder_node.exists() assert sub_folder.exists() - config = FileStudyTreeConfig(study_path=tmp_path, path=folder_node, study_id=-1, version=-1) + config = FileStudyTreeConfig(study_path=tmp_path, path=folder_node, study_id="-1", version=-1) tree_node = TestMiddleNode( context=Mock(), config=config,