From e688768f254025d5f1ac772080222860011af205 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 5 Oct 2023 10:24:49 +0200 Subject: [PATCH] feat(api): improve and correct the implementation of the short-term storage manager. --- antarest/study/business/st_storage_manager.py | 142 +++------ antarest/study/business/utils.py | 16 + .../model/filesystem/config/st_storage.py | 37 ++- .../study_data_blueprint/test_st_storage.py | 279 +++++++++--------- 4 files changed, 210 insertions(+), 264 deletions(-) diff --git a/antarest/study/business/st_storage_manager.py b/antarest/study/business/st_storage_manager.py index 5b3fe49c0b..f16ff680d4 100644 --- a/antarest/study/business/st_storage_manager.py +++ b/antarest/study/business/st_storage_manager.py @@ -1,11 +1,10 @@ import functools import json import operator -import re -from typing import Any, Dict, List, Mapping, MutableMapping, Sequence, Optional +from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Sequence import numpy as np -from pydantic import BaseModel, Extra, Field, root_validator, validator +from pydantic import BaseModel, Extra, root_validator, validator from typing_extensions import Literal from antarest.core.exceptions import ( @@ -13,9 +12,13 @@ STStorageFieldsNotFoundError, STStorageMatrixNotFoundError, ) -from antarest.study.business.utils import AllOptionalMetaclass, FormFieldsBaseModel, execute_or_add_commands +from antarest.study.business.utils import AllOptionalMetaclass, camel_case_model, execute_or_add_commands from antarest.study.model import Study -from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import STStorageConfig, STStorageGroup +from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import ( + STStorageConfig, + STStorageGroup, + STStorageProperties, +) from antarest.study.storage.storage_service import StudyStorageService from antarest.study.storage.variantstudy.model.command.create_st_storage import CreateSTStorage from antarest.study.storage.variantstudy.model.command.remove_st_storage import RemoveSTStorage @@ -24,108 +27,16 @@ _HOURS_IN_YEAR = 8760 -class FormBaseModel(FormFieldsBaseModel): - """ - A foundational model for all form-based models, providing common configurations. - """ - - class Config: - validate_assignment = True - allow_population_by_field_name = True - - -class StorageForm(FormBaseModel): +@camel_case_model +class StorageInput(STStorageProperties, metaclass=AllOptionalMetaclass): """ - Model representing the form used to create/edit a new short-term storage entry. + Model representing the form used to EDIT an existing short-term storage. """ - name: str = Field( - description="Name of the storage.", - ) - group: STStorageGroup = Field( - STStorageGroup.OTHER1, - description="Energy storage system group.", - ) - injection_nominal_capacity: float = Field( - 0, - description="Injection nominal capacity (MW)", - ge=0, - ) - withdrawal_nominal_capacity: float = Field( - 0, - description="Withdrawal nominal capacity (MW)", - ge=0, - ) - reservoir_capacity: float = Field( - 0, - description="Reservoir capacity (MWh)", - ge=0, - ) - efficiency: float = Field( - 0, - description="Efficiency of the storage system", - ge=0, - le=1, - ) - initial_level: float = Field( - 0, - description="Initial level of the storage system", - ge=0, - ) - initial_level_optim: bool = Field( - False, - description="Flag indicating if the initial level is optimized", - ) - - @validator("name") - def validate_name_st_storage(cls, value: str) -> str: - """ - Check if the field (name) is valid - :param value: name of st storage - :return: value if is correct or raise an exception - """ - pattern = r"^(?![0-9]+$).*" - if len(value) < 1 or value is None or not re.match(pattern, value): - raise ValueError(f"The field name: {value} is not valid") - return value - - -class StorageUpdate(StorageForm, metaclass=AllOptionalMetaclass): - """set fields as optional""" - - -class StorageCreation(StorageUpdate): - """set value's fields as optional""" - class Config: @staticmethod def schema_extra(schema: MutableMapping[str, Any]) -> None: - schema["example"] = StorageCreation( - name="Siemens Battery", - group=STStorageGroup.BATTERY, - injection_nominal_capacity=0, - withdrawal_nominal_capacity=0, - reservoir_capacity=0, - efficiency=0, - initial_level=0, - initial_level_optim=False, - ) - - @property - def to_config(self) -> STStorageConfig: - values = self.dict(by_alias=False) - return STStorageConfig(**values) - - -class StorageInput(StorageUpdate): - """ - Model representing the form used to edit existing short-term storage details. - """ - - class Config: - @staticmethod - def schema_extra(schema: MutableMapping[str, Any]) -> None: - schema["example"] = StorageCreation( + schema["example"] = StorageInput( name="Siemens Battery", group=STStorageGroup.BATTERY, injection_nominal_capacity=150, @@ -137,15 +48,32 @@ def schema_extra(schema: MutableMapping[str, Any]) -> None: ) -class StorageOutput(StorageInput): +class StorageCreation(StorageInput): """ - Model representing the form used to display the details of a short-term storage entry. + Model representing the form used to CREATE a new short-term storage. """ - id: str = Field( - description="Short-term storage ID", - regex=r"[a-zA-Z0-9_(),& -]+", - ) + # noinspection Pydantic + @validator("name", pre=True) + def validate_name(cls, name: Optional[str]) -> str: + """ + Validator to check if the name is not empty. + """ + if not name: + raise ValueError("'name' must not be empty") + return name + + @property + def to_config(self) -> STStorageConfig: + values = self.dict(by_alias=False, exclude_none=True) + return STStorageConfig(**values) + + +@camel_case_model +class StorageOutput(STStorageConfig): + """ + Model representing the form used to display the details of a short-term storage entry. + """ class Config: @staticmethod diff --git a/antarest/study/business/utils.py b/antarest/study/business/utils.py index b602c60138..33b62d766c 100644 --- a/antarest/study/business/utils.py +++ b/antarest/study/business/utils.py @@ -117,3 +117,19 @@ def __new__( annotations[field] = Optional[annotations[field]] namespaces["__annotations__"] = annotations return super().__new__(cls, name, bases, namespaces) + + +def camel_case_model(model: Type[BaseModel]) -> Type[BaseModel]: + """ + This decorator can be used to modify a model to use camel case aliases. + + Args: + model: The pydantic model to modify. + + Returns: + The modified model. + """ + model.__config__.alias_generator = to_camel_case + for field_name, field in model.__fields__.items(): + field.alias = to_camel_case(field_name) + return model diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/st_storage.py b/antarest/study/storage/rawstudy/model/filesystem/config/st_storage.py index d2a68c4799..b82910a191 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/st_storage.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/st_storage.py @@ -30,22 +30,18 @@ class STStorageGroup(EnumIgnoreCase): # noinspection SpellCheckingInspection -class STStorageConfig(BaseModel): - """ - Manage the configuration files in the context of Short-Term Storage. - It provides a convenient way to read and write configuration data from/to an INI file format. +class STStorageProperties( + BaseModel, + extra=Extra.forbid, + validate_assignment=True, + allow_population_by_field_name=True, +): """ + Properties of a short-term storage system read from the configuration files. - class Config: - extra = Extra.forbid - allow_population_by_field_name = True + All aliases match the name of the corresponding field in the INI files. + """ - # The `id` field is a calculated from the `name` if not provided. - # This value must be stored in the config cache. - id: str = Field( - description="Short-term storage ID", - regex=r"[a-zA-Z0-9_(),& -]+", - ) name: str = Field( description="Short-term storage name", regex=r"[a-zA-Z0-9_(),& -]+", @@ -90,6 +86,21 @@ class Config: alias="initialleveloptim", ) + +# noinspection SpellCheckingInspection +class STStorageConfig(STStorageProperties): + """ + Manage the configuration files in the context of Short-Term Storage. + It provides a convenient way to read and write configuration data from/to an INI file format. + """ + + # The `id` field is a calculated from the `name` if not provided. + # This value must be stored in the config cache. + id: str = Field( + description="Short-term storage ID", + regex=r"[a-zA-Z0-9_(),& -]+", + ) + @root_validator(pre=True) def calculate_storage_id(cls, values: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index 36be740c5b..2d1035af3e 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -1,4 +1,3 @@ -import json import re import numpy as np @@ -9,6 +8,17 @@ from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from tests.integration.utils import wait_task_completion +DEFAULT_PROPERTIES = { + # `name` field is required + "group": "Other1", + "injectionNominalCapacity": 0.0, + "withdrawalNominalCapacity": 0.0, + "reservoirCapacity": 0.0, + "efficiency": 1.0, + "initialLevel": 0.0, + "initialLevelOptim": False, +} + @pytest.mark.unit_test class TestSTStorage: @@ -61,37 +71,46 @@ def test_lifecycle__nominal( task = wait_task_completion(client, user_access_token, task_id) assert task.status == TaskStatus.COMPLETED, task - # creation with default values (only mandatory properties specified) + # ============================= + # SHORT-TERM STORAGE CREATION + # ============================= + area_id = transform_name_to_id("FR") siemens_battery = "Siemens Battery" + + # Un attempt to create a short-term storage without name + # should raise a validation error (other properties are optional). + # Un attempt to create a short-term storage with an empty name + # or an invalid name should also raise a validation error. + attempts = [{}, {"name": ""}, {"name": "!??"}] + for attempt in attempts: + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/storages", + headers={"Authorization": f"Bearer {user_access_token}"}, + json=attempt, + ) + assert res.status_code == 422, res.json() + assert res.json()["exception"] in {"ValidationError", "RequestValidationError"}, res.json() + + # We can create a short-term storage with the following properties: + siemens_properties = { + **DEFAULT_PROPERTIES, + "name": siemens_battery, + "group": "Battery", + "injectionNominalCapacity": 1450, + "withdrawalNominalCapacity": 1350, + "reservoirCapacity": 1500, + } res = client.post( f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, - json={ - "name": siemens_battery, - "group": "Battery", - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "reservoirCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - "efficiency": 1.0, - }, + json=siemens_properties, ) assert res.status_code == 200, res.json() siemens_battery_id = res.json()["id"] assert siemens_battery_id == transform_name_to_id(siemens_battery) - assert res.json() == { - "efficiency": 1.0, - "group": "Battery", - "id": siemens_battery_id, - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "name": siemens_battery, - "reservoirCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - } + siemens_config = {**siemens_properties, "id": siemens_battery_id} + assert res.json() == siemens_config # reading the properties of a short-term storage res = client.get( @@ -99,17 +118,11 @@ def test_lifecycle__nominal( headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - assert res.json() == { - "efficiency": 1.0, - "group": "Battery", - "id": siemens_battery_id, - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "name": siemens_battery, - "reservoirCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - } + assert res.json() == siemens_config + + # ============================= + # SHORT-TERM STORAGE MATRICES + # ============================= # updating the matrix of a short-term storage array = np.random.rand(8760, 1) * 1000 @@ -143,25 +156,17 @@ def test_lifecycle__nominal( assert res.status_code == 200, res.json() assert res.json() is True + # ================================== + # SHORT-TERM STORAGE LIST / GROUPS + # ================================== + # Reading the list of short-term storages res = client.get( f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - assert res.json() == [ - { - "efficiency": 1.0, - "group": "Battery", - "id": siemens_battery_id, - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "name": siemens_battery, - "reservoirCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - } - ] + assert res.json() == [siemens_config] # updating properties res = client.patch( @@ -173,34 +178,23 @@ def test_lifecycle__nominal( }, ) assert res.status_code == 200, res.json() - assert json.loads(res.text) == { - "id": siemens_battery_id, + siemens_config = { + **siemens_config, "name": "New Siemens Battery", - "group": "Battery", - "efficiency": 1.0, - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, "reservoirCapacity": 2500, } + assert res.json() == siemens_config res = client.get( f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - assert res.json() == { - "id": siemens_battery_id, - "name": "New Siemens Battery", - "group": "Battery", - "efficiency": 1.0, - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - "reservoirCapacity": 2500, - } + assert res.json() == siemens_config + + # =========================== + # SHORT-TERM STORAGE UPDATE + # =========================== # updating properties res = client.patch( @@ -211,52 +205,38 @@ def test_lifecycle__nominal( "reservoirCapacity": 0, }, ) - assert res.status_code == 200, res.json() - assert json.loads(res.text) == { - "id": siemens_battery_id, - "name": "New Siemens Battery", - "group": "Battery", - "efficiency": 1.0, + siemens_config = { + **siemens_config, "initialLevel": 5900, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, "reservoirCapacity": 0, } - # Check the put with the wrong efficiency + assert res.json() == siemens_config + + # An attempt to update the `efficiency` property with an invalid value + # should raise a validation error. + # The `efficiency` property must be a float between 0 and 1. + bad_properties = {"efficiency": 2.0} res = client.patch( f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, - json={ - "efficiency": 2.0, - "initialLevel": 0.0, - "initialLevelOptim": True, - "injectionNominalCapacity": 2450, - "name": "New Siemens Battery", - "reservoirCapacity": 2500, - "withdrawalNominalCapacity": 2350, - }, + json=bad_properties, ) assert res.status_code == 422, res.json() + assert res.json()["exception"] == "ValidationError", res.json() + # The short-term storage properties should not have been updated. res = client.get( f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - assert res.json() == { - "id": siemens_battery_id, - "name": "New Siemens Battery", - "group": "Battery", - "efficiency": 1.0, - "initialLevel": 5900, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - "reservoirCapacity": 0, - } + assert res.json() == siemens_config + + # ============================= + # SHORT-TERM STORAGE DELETION + # ============================= - # deletion of short-term storages + # To delete a short-term storage, we need to provide its ID. res = client.request( "DELETE", f"/v1/studies/{study_id}/areas/{area_id}/storages", @@ -266,7 +246,7 @@ def test_lifecycle__nominal( assert res.status_code == 204, res.json() assert res.text in {"", "null"} # Old FastAPI versions return 'null'. - # deletion of short-term storages with empty list + # If the short-term storage list is empty, the deletion should be a no-op. res = client.request( "DELETE", f"/v1/studies/{study_id}/areas/{area_id}/storages", @@ -276,66 +256,79 @@ def test_lifecycle__nominal( assert res.status_code == 204, res.json() assert res.text in {"", "null"} # Old FastAPI versions return 'null'. - # deletion of short-term storages with multiple IDs + # It's possible to delete multiple short-term storages at once. + # In the following example, we will create two short-term storages: + siemens_properties = { + "name": siemens_battery, + "group": "Battery", + "injectionNominalCapacity": 1450, + "withdrawalNominalCapacity": 1350, + "reservoirCapacity": 1500, + "efficiency": 0.90, + "initialLevel": 200, + "initialLevelOptim": False, + } res = client.post( f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, - json={ - "name": siemens_battery, - "group": "Battery", - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "reservoirCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - "efficiency": 1.0, - }, + json=siemens_properties, ) assert res.status_code == 200, res.json() - siemens_battery_id1 = res.json()["id"] - - siemens_battery_del = f"{siemens_battery}del" + siemens_battery_id = res.json()["id"] + # Create another short-term storage: "Grand'Maison" + grand_maison = "Grand'Maison" + grand_maison_properties = { + "name": grand_maison, + "group": "PSP_closed", + "injectionNominalCapacity": 1500, + "withdrawalNominalCapacity": 1800, + "reservoirCapacity": 20000, + "efficiency": 0.78, + "initialLevel": 10000, + } res = client.post( f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, - json={ - "name": siemens_battery_del, - "group": "Battery", - "initialLevel": 0.0, - "initialLevelOptim": False, - "injectionNominalCapacity": 0.0, - "reservoirCapacity": 0.0, - "withdrawalNominalCapacity": 0.0, - "efficiency": 1.0, - }, + json=grand_maison_properties, + ) + assert res.status_code == 200, res.json() + grand_maison_id = res.json()["id"] + + # We can check that we have 2 short-term storages in the list. + # Reading the list of short-term storages + res = client.get( + f"/v1/studies/{study_id}/areas/{area_id}/storages", + headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - siemens_battery_id2 = res.json()["id"] + siemens_config = {**DEFAULT_PROPERTIES, **siemens_properties, "id": siemens_battery_id} + grand_maison_config = {**DEFAULT_PROPERTIES, **grand_maison_properties, "id": grand_maison_id} + assert res.json() == [siemens_config, grand_maison_config] + # We can delete the two short-term storages at once. res = client.request( "DELETE", f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, - json=[siemens_battery_id1, siemens_battery_id2], + json=[siemens_battery_id, grand_maison_id], ) assert res.status_code == 204, res.json() assert res.text in {"", "null"} # Old FastAPI versions return 'null'. - # Check the removal + # The list of short-term storages should be empty. res = client.get( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", + f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, ) - obj = res.json() - description = obj["description"] - assert siemens_battery_id in description - assert re.search(r"fields of storage", description, flags=re.IGNORECASE) - assert re.search(r"not found", description, flags=re.IGNORECASE) + assert res.status_code == 200, res.json() + assert res.json() == [] - assert res.status_code == 404, res.json() + # =========================== + # SHORT-TERM STORAGE ERRORS + # =========================== - # Check delete with the wrong value of area_id + # Check delete with the wrong value of `area_id` bad_area_id = "bad_area" res = client.request( "DELETE", @@ -353,7 +346,7 @@ def test_lifecycle__nominal( flags=re.IGNORECASE, ) - # Check delete with the wrong value of study_id + # Check delete with the wrong value of `study_id` bad_study_id = "bad_study" res = client.request( "DELETE", @@ -366,8 +359,7 @@ def test_lifecycle__nominal( assert res.status_code == 404, res.json() assert bad_study_id in description - # Check get with wrong area_id - + # Check get with wrong `area_id` res = client.get( f"/v1/studies/{study_id}/areas/{bad_area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -377,8 +369,7 @@ def test_lifecycle__nominal( assert bad_area_id in description assert res.status_code == 404, res.json() - # Check get with wrong study_id - + # Check get with wrong `study_id` res = client.get( f"/v1/studies/{bad_study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -388,7 +379,7 @@ def test_lifecycle__nominal( assert res.status_code == 404, res.json() assert bad_study_id in description - # Check post with wrong study_id + # Check POST with wrong `study_id` res = client.post( f"/v1/studies/{bad_study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -399,7 +390,7 @@ def test_lifecycle__nominal( assert res.status_code == 404, res.json() assert bad_study_id in description - # Check post with wrong area_id + # Check POST with wrong `area_id` res = client.post( f"/v1/studies/{study_id}/areas/{bad_area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -421,7 +412,7 @@ def test_lifecycle__nominal( assert re.search(r"Area ", description, flags=re.IGNORECASE) assert re.search(r"does not exist ", description, flags=re.IGNORECASE) - # Check post with wrong group + # Check POST with wrong `group` res = client.post( f"/v1/studies/{study_id}/areas/{bad_area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -432,7 +423,7 @@ def test_lifecycle__nominal( description = obj["description"] assert re.search(r"not a valid enumeration member", description, flags=re.IGNORECASE) - # Check the put with the wrong area_id + # Check PATCH with the wrong `area_id` res = client.patch( f"/v1/studies/{study_id}/areas/{bad_area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -452,7 +443,7 @@ def test_lifecycle__nominal( assert bad_area_id in description assert re.search(r"not a child of ", description, flags=re.IGNORECASE) - # Check the put with the wrong siemens_battery_id + # Check PATCH with the wrong `siemens_battery_id` res = client.patch( f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, @@ -473,7 +464,7 @@ def test_lifecycle__nominal( assert re.search(r"fields of storage", description, flags=re.IGNORECASE) assert re.search(r"not found", description, flags=re.IGNORECASE) - # Check the put with the wrong study_id + # Check PATCH with the wrong `study_id` res = client.patch( f"/v1/studies/{bad_study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"},