From d775971e0a33c1c039407c577863c1245c2e6913 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 6 Mar 2024 14:11:20 +0100 Subject: [PATCH 01/15] feat(clusters): add new endpoint for duplication --- antarest/core/exceptions.py | 9 ++++ .../business/areas/renewable_management.py | 28 +++++++++- .../business/areas/st_storage_management.py | 18 +++++++ .../business/areas/thermal_management.py | 40 +++++++++++++- antarest/study/business/utils.py | 7 +++ antarest/study/web/study_data_blueprint.py | 34 ++++++++++++ .../study_data_blueprint/test_renewable.py | 54 +++++++++++++++---- .../study_data_blueprint/test_st_storage.py | 50 +++++++++++++++-- .../study_data_blueprint/test_thermal.py | 49 ++++++++++++++--- 9 files changed, 265 insertions(+), 24 deletions(-) diff --git a/antarest/core/exceptions.py b/antarest/core/exceptions.py index cada3a5f5d..e4236e7976 100644 --- a/antarest/core/exceptions.py +++ b/antarest/core/exceptions.py @@ -312,3 +312,12 @@ def __init__(self, area_id: str) -> None: HTTPStatus.NOT_FOUND, f"Cluster configuration for area: '{area_id}' not found", ) + + +class ClusterAlreadyExists(HTTPException): + """Exception raised when trying to create a cluster with an already existing id.""" + + def __init__(self, cluster_type: str, cluster_id: str) -> None: + super().__init__( + HTTPStatus.CONFLICT, f"{cluster_type} cluster {cluster_id} already exists and could not be created" + ) diff --git a/antarest/study/business/areas/renewable_management.py b/antarest/study/business/areas/renewable_management.py index ab9a2e9802..70e5ad1089 100644 --- a/antarest/study/business/areas/renewable_management.py +++ b/antarest/study/business/areas/renewable_management.py @@ -3,10 +3,11 @@ from pydantic import validator -from antarest.core.exceptions import ClusterConfigNotFound, ClusterNotFound +from antarest.core.exceptions import ClusterAlreadyExists, ClusterConfigNotFound, ClusterNotFound from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.business.utils import AllOptionalMetaclass, camel_case_model, execute_or_add_commands from antarest.study.model import Study +from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.config.renewable import ( RenewableConfig, RenewableConfigType, @@ -17,6 +18,7 @@ from antarest.study.storage.storage_service import StudyStorageService from antarest.study.storage.variantstudy.model.command.create_renewables_cluster import CreateRenewablesCluster from antarest.study.storage.variantstudy.model.command.remove_renewables_cluster import RemoveRenewablesCluster +from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig __all__ = ( @@ -273,3 +275,27 @@ def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[st ] execute_or_add_commands(study, file_study, commands, self.storage_service) + + def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> RenewableClusterOutput: + new_id = transform_name_to_id(new_name, lower=False) + existing_ids = [cluster.id for cluster in self.get_clusters(study, area_id)] + if new_id in existing_ids: + raise ClusterAlreadyExists("Renewable", new_id) + # Cluster creation + current_cluster = self.get_cluster(study, area_id, source_id) + current_cluster.name = new_name + creation_form = RenewableClusterCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) + new_cluster = self.create_cluster(study, area_id, creation_form) + # Matrix edition + current_path = f"input/renewables/series/{area_id}/{source_id.lower()}/series" + new_path = f"input/renewables/series/{area_id}/{new_id.lower()}/series" + current_matrix = self.storage_service.raw_study_service.get(study, current_path) + command = [ + ReplaceMatrix( + target=new_path, + matrix=current_matrix["data"], + command_context=self.storage_service.variant_study_service.command_factory.command_context, + ) + ] + execute_or_add_commands(study, self._get_file_study(study), command, self.storage_service) + return new_cluster diff --git a/antarest/study/business/areas/st_storage_management.py b/antarest/study/business/areas/st_storage_management.py index d18dce9f9c..528fb26c9d 100644 --- a/antarest/study/business/areas/st_storage_management.py +++ b/antarest/study/business/areas/st_storage_management.py @@ -8,12 +8,14 @@ from typing_extensions import Literal from antarest.core.exceptions import ( + ClusterAlreadyExists, STStorageConfigNotFoundError, STStorageFieldsNotFoundError, STStorageMatrixNotFoundError, ) from antarest.study.business.utils import AllOptionalMetaclass, camel_case_model, execute_or_add_commands from antarest.study.model import Study +from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import ( STStorageConfig, STStorageGroup, @@ -418,6 +420,22 @@ def delete_storages( file_study = self._get_file_study(study) execute_or_add_commands(study, file_study, [command], self.storage_service) + def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> STStorageOutput: + new_id = transform_name_to_id(new_name) + existing_ids = [storage.id for storage in self.get_storages(study, area_id)] + if new_id in existing_ids: + raise ClusterAlreadyExists("Short term storage", new_id) + # Cluster creation + current_cluster = self.get_storage(study, area_id, source_id) + current_cluster.name = new_name + creation_form = STStorageCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) + new_storage = self.create_storage(study, area_id, creation_form) + # Matrix edition + for ts_name in STStorageTimeSeries.__args__: # type: ignore + ts = self.get_matrix(study, area_id, source_id, ts_name) + self.update_matrix(study, area_id, new_id, ts_name, ts) + return new_storage + def get_matrix( self, study: Study, diff --git a/antarest/study/business/areas/thermal_management.py b/antarest/study/business/areas/thermal_management.py index dfcc52a2a0..fc680d53f9 100644 --- a/antarest/study/business/areas/thermal_management.py +++ b/antarest/study/business/areas/thermal_management.py @@ -3,9 +3,10 @@ from pydantic import validator -from antarest.core.exceptions import ClusterConfigNotFound, ClusterNotFound +from antarest.core.exceptions import ClusterAlreadyExists, ClusterConfigNotFound, ClusterNotFound from antarest.study.business.utils import AllOptionalMetaclass, camel_case_model, execute_or_add_commands from antarest.study.model import Study +from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.config.thermal import ( Thermal860Config, Thermal860Properties, @@ -16,6 +17,7 @@ from antarest.study.storage.storage_service import StudyStorageService from antarest.study.storage.variantstudy.model.command.create_cluster import CreateCluster from antarest.study.storage.variantstudy.model.command.remove_cluster import RemoveCluster +from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig __all__ = ( @@ -286,3 +288,39 @@ def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[st ] execute_or_add_commands(study, file_study, commands, self.storage_service) + + def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> ThermalClusterOutput: + new_id = transform_name_to_id(new_name, lower=False) + existing_ids = [cluster.id for cluster in self.get_clusters(study, area_id)] + if new_id in existing_ids: + raise ClusterAlreadyExists("Thermal", new_id) + # Cluster creation + current_cluster = self.get_cluster(study, area_id, source_id) + current_cluster.name = new_name + creation_form = ThermalClusterCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) + new_cluster = self.create_cluster(study, area_id, creation_form) + # Matrix edition + lower_source_id = source_id.lower() + lower_new_id = new_id.lower() + paths = [ + f"input/thermal/series/{area_id}/{lower_source_id}/series", + f"input/thermal/prepro/{area_id}/{lower_source_id}/modulation", + f"input/thermal/prepro/{area_id}/{lower_source_id}/data", + ] + new_paths = [ + f"input/thermal/series/{area_id}/{lower_new_id}/series", + f"input/thermal/prepro/{area_id}/{lower_new_id}/modulation", + f"input/thermal/prepro/{area_id}/{lower_new_id}/data", + ] + commands = [] + storage_service = self.storage_service + for k, matrix_path in enumerate(paths): + current_matrix = storage_service.raw_study_service.get(study, matrix_path) + command = ReplaceMatrix( + target=new_paths[k], + matrix=current_matrix["data"], + command_context=storage_service.variant_study_service.command_factory.command_context, + ) + commands.append(command) + execute_or_add_commands(study, self._get_file_study(study), commands, storage_service) + return new_cluster diff --git a/antarest/study/business/utils.py b/antarest/study/business/utils.py index 53596bf797..ba95d5ddd4 100644 --- a/antarest/study/business/utils.py +++ b/antarest/study/business/utils.py @@ -8,6 +8,7 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.requests import RequestParameters from antarest.core.utils.string import to_camel_case +from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import RawStudy, Study from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.storage_service import StudyStorageService @@ -19,6 +20,12 @@ GENERAL_DATA_PATH = "settings/generaldata" +class ClusterType(EnumIgnoreCase): + ST_STORAGE = "storages" + RENEWABLE = "renewable" + THERMAL = "thermal" + + def execute_or_add_commands( study: Study, file_study: FileStudy, diff --git a/antarest/study/web/study_data_blueprint.py b/antarest/study/web/study_data_blueprint.py index c7e6ac17fd..9be2889eb5 100644 --- a/antarest/study/web/study_data_blueprint.py +++ b/antarest/study/web/study_data_blueprint.py @@ -24,10 +24,12 @@ RenewableClusterCreation, RenewableClusterInput, RenewableClusterOutput, + RenewableManager, ) from antarest.study.business.areas.st_storage_management import ( STStorageCreation, STStorageInput, + STStorageManager, STStorageMatrix, STStorageOutput, STStorageTimeSeries, @@ -36,6 +38,7 @@ ThermalClusterCreation, ThermalClusterInput, ThermalClusterOutput, + ThermalManager, ) from antarest.study.business.binding_constraint_management import ( BindingConstraintPropertiesWithName, @@ -51,6 +54,7 @@ from antarest.study.business.table_mode_management import ColumnsModelTypes, TableTemplateType from antarest.study.business.thematic_trimming_management import ThematicTrimmingFormFields from antarest.study.business.timeseries_config_management import TSFormFields +from antarest.study.business.utils import ClusterType from antarest.study.model import PatchArea, PatchCluster from antarest.study.service import StudyService from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id @@ -2019,4 +2023,34 @@ def delete_st_storages( study = study_service.check_study_access(uuid, StudyPermissionType.WRITE, params) study_service.st_storage_manager.delete_storages(study, area_id, storage_ids) + @bp.post( + path="/studies/{uuid}/areas/{area_id}/{cluster_type}/{source_cluster_id}", + tags=[APITag.study_data], + summary="Duplicates a given cluster", + ) + def duplicate_cluster( + uuid: str, + area_id: str, + cluster_type: ClusterType, + source_cluster_id: str, + new_cluster_name: str, + current_user: JWTUser = Depends(auth.get_current_user), + ) -> Union[STStorageOutput, ThermalClusterOutput, RenewableClusterOutput]: + logger.info( + f"Duplicates {cluster_type.value} {source_cluster_id} of {area_id} for study {uuid}", + extra={"user": current_user.id}, + ) + params = RequestParameters(user=current_user) + study = study_service.check_study_access(uuid, StudyPermissionType.WRITE, params) + + manager: Union[STStorageManager, RenewableManager, ThermalManager] + if cluster_type == ClusterType.ST_STORAGE: + manager = STStorageManager(study_service.storage_service) + elif cluster_type == ClusterType.RENEWABLE: + manager = RenewableManager(study_service.storage_service) + else: + manager = ThermalManager(study_service.storage_service) + + return manager.duplicate_cluster(study, area_id, source_cluster_id, new_cluster_name) + return bp diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index 14f1f4388a..498a5b44dd 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -23,6 +23,7 @@ * delete a cluster (or several clusters) * validate the consistency of the matrices (and properties) """ +import copy import json import re @@ -211,6 +212,22 @@ def test_lifecycle( assert res.status_code == 200, res.json() assert res.json() == fr_solar_pv_cfg + # ============================= + # RENEWABLE CLUSTER DUPLICATION + # ============================= + + new_name = "Duplicate of SolarPV" + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/renewable/{fr_solar_pv_id}?new_cluster_name={new_name}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201} + duplicated_config = copy.deepcopy(fr_solar_pv_cfg) + duplicated_config["name"] = new_name + duplicated_id = transform_name_to_id(new_name, lower=False) + duplicated_config["id"] = duplicated_id + assert res.json() == duplicated_config + # ============================= # RENEWABLE CLUSTER DELETION # ============================= @@ -253,28 +270,24 @@ def test_lifecycle( assert res.status_code == 200, res.json() other_cluster_id2 = res.json()["id"] - # We can delete the two renewable clusters at once. + # We can delete two renewable clusters at once. res = client.request( "DELETE", f"/v1/studies/{study_id}/areas/{area_id}/clusters/renewable", headers={"Authorization": f"Bearer {user_access_token}"}, - json=[other_cluster_id1, other_cluster_id2], + json=[other_cluster_id2, duplicated_id], ) assert res.status_code == 204, res.json() assert res.text in {"", "null"} # Old FastAPI versions return 'null'. - # The list of renewable clusters should be empty. + # There should only be one remaining cluster res = client.get( f"/v1/studies/{study_id}/areas/{area_id}/clusters/renewable", headers={"Authorization": f"Bearer {user_access_token}"}, ) - assert res.status_code == 200, res.json() - expected = [ - c - for c in EXISTING_CLUSTERS - if transform_name_to_id(c["name"], lower=False) not in [other_cluster_id1, other_cluster_id2] - ] - assert res.json() == expected + assert res.status_code == 200 + obj = res.json() + assert len(obj) == 1 # =========================== # RENEWABLE CLUSTER ERRORS @@ -422,3 +435,24 @@ def test_lifecycle( obj = res.json() description = obj["description"] assert bad_study_id in description + + # Cannot duplicate a fake cluster + fake_id = "fake_id" + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/renewable/{fake_id}?new_cluster_name=duplicata", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 404 + obj = res.json() + assert obj["description"] == f"Cluster: '{fake_id}' not found" + assert obj["exception"] == "ClusterNotFound" + + # Cannot duplicate with an existing id + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/renewable/{other_cluster_id1}?new_cluster_name={other_cluster_id1}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 409 + obj = res.json() + assert obj["description"] == f"Renewable cluster {other_cluster_id1} already exists and could not be created" + assert obj["exception"] == "ClusterAlreadyExists" diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index fdffe5efe1..ec4a1a5a02 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -1,3 +1,4 @@ +import copy import json import re from unittest.mock import ANY @@ -231,6 +232,21 @@ def test_lifecycle__nominal( assert res.status_code == 200, res.json() assert res.json() == siemens_config + # ============================= + # SHORT-TERM STORAGE DUPLICATION + # ============================= + + new_name = "Duplicate of Siemens" + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={new_name}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201} + duplicated_config = copy.deepcopy(siemens_config) + duplicated_config["name"] = new_name # type: ignore + duplicated_config["id"] = transform_name_to_id(new_name) # type: ignore + assert res.json() == duplicated_config + # ============================= # SHORT-TERM STORAGE DELETION # ============================= @@ -303,25 +319,25 @@ def test_lifecycle__nominal( assert res.status_code == 200, res.json() siemens_config = {**DEFAULT_PROPERTIES, **siemens_properties, "id": siemens_battery_id} grand_maison_config = {**DEFAULT_PROPERTIES, **grand_maison_properties, "id": grand_maison_id} - assert res.json() == [siemens_config, grand_maison_config] + assert res.json() == [duplicated_config, siemens_config, grand_maison_config] - # We can delete the two short-term storages at once. + # We can delete the three short-term storages at once. res = client.request( "DELETE", f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, - json=[siemens_battery_id, grand_maison_id], + json=[grand_maison_id, duplicated_config["id"]], ) assert res.status_code == 204, res.json() assert res.text in {"", "null"} # Old FastAPI versions return 'null'. - # The list of short-term storages should be empty. + # Only one st-storage should remain. res = client.get( f"/v1/studies/{study_id}/areas/{area_id}/storages", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - assert res.json() == [] + assert len(res.json()) == 1 # =========================== # SHORT-TERM STORAGE ERRORS @@ -450,6 +466,30 @@ def test_lifecycle__nominal( description = obj["description"] assert bad_study_id in description + # Cannot duplicate a fake st-storage + fake_id = "fake_id" + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/storages/{fake_id}?new_cluster_name=duplicata", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 404 + obj = res.json() + assert obj["description"] == f"Fields of storage '{fake_id}' not found" + assert obj["exception"] == "STStorageFieldsNotFoundError" + + # Cannot duplicate with an existing id + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 409 + obj = res.json() + assert ( + obj["description"] + == f"Short term storage cluster {siemens_battery_id} already exists and could not be created" + ) + assert obj["exception"] == "ClusterAlreadyExists" + def test__default_values( self, client: TestClient, diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index 1890d44acf..7514c6dfe6 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -27,6 +27,7 @@ * delete a cluster (or several clusters) * validate the consistency of the matrices (and properties) """ +import copy import json import re @@ -536,6 +537,22 @@ def test_lifecycle( assert res.status_code == 200, res.json() assert res.json() == fr_gas_conventional_cfg + # ============================= + # THERMAL CLUSTER DUPLICATION + # ============================= + + new_name = "Duplicate of Fr_Gas_Conventional" + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/thermal/{fr_gas_conventional_id}?new_cluster_name={new_name}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201} + duplicated_config = copy.deepcopy(fr_gas_conventional_cfg) + duplicated_config["name"] = new_name # type: ignore + duplicated_id = transform_name_to_id(new_name, lower=False) + duplicated_config["id"] = duplicated_id # type: ignore + assert res.json() == duplicated_config + # ============================= # THERMAL CLUSTER DELETION # ============================= @@ -573,18 +590,15 @@ def test_lifecycle( assert res.status_code == 204, res.json() assert res.text in {"", "null"} # Old FastAPI versions return 'null'. - # The list of thermal clusters should be empty. + # The list of thermal clusters should not contain the deleted ones. res = client.get( f"/v1/studies/{study_id}/areas/{area_id}/clusters/thermal", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 200, res.json() - expected = [ - c - for c in EXISTING_CLUSTERS - if transform_name_to_id(c["name"], lower=False) not in [other_cluster_id1, other_cluster_id2] - ] - assert res.json() == expected + deleted_clusters = [other_cluster_id1, other_cluster_id2, fr_gas_conventional_id] + for cluster in res.json(): + assert transform_name_to_id(cluster["name"], lower=False) not in deleted_clusters # =========================== # THERMAL CLUSTER ERRORS @@ -748,3 +762,24 @@ def test_lifecycle( obj = res.json() description = obj["description"] assert bad_study_id in description + + # Cannot duplicate a fake cluster + fake_id = "fake_id" + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/thermal/{fake_id}?new_cluster_name=duplicata", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 404 + obj = res.json() + assert obj["description"] == f"Cluster: '{fake_id}' not found" + assert obj["exception"] == "ClusterNotFound" + + # Cannot duplicate with an existing id + res = client.post( + f"/v1/studies/{study_id}/areas/{area_id}/thermal/{duplicated_id}?new_cluster_name={duplicated_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 409 + obj = res.json() + assert obj["description"] == f"Thermal cluster {duplicated_id} already exists and could not be created" + assert obj["exception"] == "ClusterAlreadyExists" From a370a4062097832a1e5315897fe5bf78f626aac9 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 6 Mar 2024 14:25:59 +0100 Subject: [PATCH 02/15] fix(clusters): rename endpoint for clarity's sake --- antarest/study/business/utils.py | 2 +- tests/integration/study_data_blueprint/test_st_storage.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/antarest/study/business/utils.py b/antarest/study/business/utils.py index ba95d5ddd4..aed31e7dce 100644 --- a/antarest/study/business/utils.py +++ b/antarest/study/business/utils.py @@ -21,7 +21,7 @@ class ClusterType(EnumIgnoreCase): - ST_STORAGE = "storages" + ST_STORAGE = "st-storage" RENEWABLE = "renewable" THERMAL = "thermal" diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index ec4a1a5a02..22a968128c 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -238,7 +238,7 @@ def test_lifecycle__nominal( new_name = "Duplicate of Siemens" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/st-storage/{siemens_battery_id}?new_cluster_name={new_name}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code in {200, 201} @@ -469,7 +469,7 @@ def test_lifecycle__nominal( # Cannot duplicate a fake st-storage fake_id = "fake_id" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/st-storage/{fake_id}?new_cluster_name=duplicata", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 404 @@ -479,7 +479,7 @@ def test_lifecycle__nominal( # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", + f"/v1/studies/{study_id}/areas/{area_id}/st-storage/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 409 From c3fa31b50ea2b42ce5848195797668cb727bb6bc Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 6 Mar 2024 14:58:29 +0100 Subject: [PATCH 03/15] fix(clusters): rename ClusterType to harmonize API --- antarest/study/business/utils.py | 2 +- tests/integration/study_data_blueprint/test_st_storage.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/antarest/study/business/utils.py b/antarest/study/business/utils.py index aed31e7dce..efafcb7e82 100644 --- a/antarest/study/business/utils.py +++ b/antarest/study/business/utils.py @@ -21,7 +21,7 @@ class ClusterType(EnumIgnoreCase): - ST_STORAGE = "st-storage" + ST_STORAGE = "storage" RENEWABLE = "renewable" THERMAL = "thermal" diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index 22a968128c..8b1c8d162d 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -238,7 +238,7 @@ def test_lifecycle__nominal( new_name = "Duplicate of Siemens" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/st-storage/{siemens_battery_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/storage/{siemens_battery_id}?new_cluster_name={new_name}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code in {200, 201} @@ -469,7 +469,7 @@ def test_lifecycle__nominal( # Cannot duplicate a fake st-storage fake_id = "fake_id" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/st-storage/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/storage/{fake_id}?new_cluster_name=duplicata", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 404 @@ -479,7 +479,7 @@ def test_lifecycle__nominal( # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/st-storage/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", + f"/v1/studies/{study_id}/areas/{area_id}/storage/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 409 From db05698d90dbae51c599a142d745c1d5001cf0d5 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Wed, 6 Mar 2024 16:23:26 +0100 Subject: [PATCH 04/15] test(clusters): add test for matrices --- .../study_data_blueprint/test_renewable.py | 30 ++++++++++++++++++- .../study_data_blueprint/test_st_storage.py | 17 +++++++++-- .../study_data_blueprint/test_thermal.py | 30 ++++++++++++++++++- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index 498a5b44dd..6737f53119 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -27,6 +27,7 @@ import json import re +import numpy as np import pytest from starlette.testclient import TestClient @@ -133,7 +134,23 @@ def test_lifecycle( # RENEWABLE CLUSTER MATRICES # ============================= - # TODO: add unit tests for renewable cluster matrices + matrix = np.random.randint(0, 2, size=(8760, 1)).tolist() + matrix_path = f"input/renewables/series/{area_id}/{fr_solar_pv_id.lower()}/series" + args = {"target": matrix_path, "matrix": matrix} + res = client.post( + f"/v1/studies/{study_id}/commands", + json=[{"action": "replace_matrix", "args": args}], + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201}, res.json() + + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix # ================================== # RENEWABLE CLUSTER LIST / GROUPS @@ -221,6 +238,7 @@ def test_lifecycle( f"/v1/studies/{study_id}/areas/{area_id}/renewable/{fr_solar_pv_id}?new_cluster_name={new_name}", headers={"Authorization": f"Bearer {user_access_token}"}, ) + # asserts the config is the same assert res.status_code in {200, 201} duplicated_config = copy.deepcopy(fr_solar_pv_cfg) duplicated_config["name"] = new_name @@ -228,6 +246,16 @@ def test_lifecycle( duplicated_config["id"] = duplicated_id assert res.json() == duplicated_config + # asserts the matrix has also been duplicated + new_cluster_matrix_path = f"input/renewables/series/{area_id}/{duplicated_id.lower()}/series" + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": new_cluster_matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix + # ============================= # RENEWABLE CLUSTER DELETION # ============================= diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index 8b1c8d162d..fbe8c89e33 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -124,14 +124,15 @@ def test_lifecycle__nominal( # ============================= # updating the matrix of a short-term storage - array = np.random.rand(8760, 1) * 1000 + array = np.random.randint(0, 1000, size=(8760, 1)) + array_list = array.tolist() res = client.put( f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}/series/inflows", headers={"Authorization": f"Bearer {user_access_token}"}, json={ "index": list(range(array.shape[0])), "columns": list(range(array.shape[1])), - "data": array.tolist(), + "data": array_list, }, ) assert res.status_code == 200, res.json() @@ -242,11 +243,21 @@ def test_lifecycle__nominal( headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code in {200, 201} + # asserts the config is the same duplicated_config = copy.deepcopy(siemens_config) duplicated_config["name"] = new_name # type: ignore - duplicated_config["id"] = transform_name_to_id(new_name) # type: ignore + duplicated_id = transform_name_to_id(new_name) + duplicated_config["id"] = duplicated_id # type: ignore assert res.json() == duplicated_config + # asserts the matrix has also been duplicated + res = client.get( + f"/v1/studies/{study_id}/areas/{area_id}/storages/{duplicated_id}/series/inflows", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == array_list + # ============================= # SHORT-TERM STORAGE DELETION # ============================= diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index 7514c6dfe6..43b3c1903c 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -31,6 +31,7 @@ import json import re +import numpy as np import pytest from starlette.testclient import TestClient @@ -456,7 +457,23 @@ def test_lifecycle( # THERMAL CLUSTER MATRICES # ============================= - # TODO: add unit tests for thermal cluster matrices + matrix = np.random.randint(0, 2, size=(8760, 1)).tolist() + matrix_path = f"input/thermal/prepro/{area_id}/{fr_gas_conventional_id.lower()}/data" + args = {"target": matrix_path, "matrix": matrix} + res = client.post( + f"/v1/studies/{study_id}/commands", + json=[{"action": "replace_matrix", "args": args}], + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201}, res.json() + + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix # ================================== # THERMAL CLUSTER LIST / GROUPS @@ -547,12 +564,23 @@ def test_lifecycle( headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code in {200, 201} + # asserts the config is the same duplicated_config = copy.deepcopy(fr_gas_conventional_cfg) duplicated_config["name"] = new_name # type: ignore duplicated_id = transform_name_to_id(new_name, lower=False) duplicated_config["id"] = duplicated_id # type: ignore assert res.json() == duplicated_config + # asserts the matrix has also been duplicated + new_cluster_matrix_path = f"input/thermal/prepro/{area_id}/{duplicated_id.lower()}/data" + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": new_cluster_matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix + # ============================= # THERMAL CLUSTER DELETION # ============================= From b22777c401d9d2ac3adb4cfdb33e41fc25d2da1d Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 6 Mar 2024 17:36:28 +0100 Subject: [PATCH 05/15] chore(clusters): improve error message phrasing --- antarest/core/exceptions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/antarest/core/exceptions.py b/antarest/core/exceptions.py index e4236e7976..f521ffec63 100644 --- a/antarest/core/exceptions.py +++ b/antarest/core/exceptions.py @@ -315,9 +315,10 @@ def __init__(self, area_id: str) -> None: class ClusterAlreadyExists(HTTPException): - """Exception raised when trying to create a cluster with an already existing id.""" + """Exception raised when attempting to create a cluster with an already existing ID.""" def __init__(self, cluster_type: str, cluster_id: str) -> None: super().__init__( - HTTPStatus.CONFLICT, f"{cluster_type} cluster {cluster_id} already exists and could not be created" + HTTPStatus.CONFLICT, + f"{cluster_type} cluster with ID '{cluster_id}' already exists and could not be created.", ) From 5562830cc882e9658a71f4ff996187137185ba56 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 6 Mar 2024 18:11:54 +0100 Subject: [PATCH 06/15] chore(clusters): add missing docstrings --- .../business/areas/renewable_management.py | 20 +++++++++++++++++- .../business/areas/st_storage_management.py | 21 +++++++++++++++++-- .../business/areas/thermal_management.py | 19 ++++++++++++++++- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/antarest/study/business/areas/renewable_management.py b/antarest/study/business/areas/renewable_management.py index 70e5ad1089..de16dff11b 100644 --- a/antarest/study/business/areas/renewable_management.py +++ b/antarest/study/business/areas/renewable_management.py @@ -277,15 +277,32 @@ def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[st execute_or_add_commands(study, file_study, commands, self.storage_service) def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> RenewableClusterOutput: + """ + Creates a duplicate cluster within the study area with a new name. + + Args: + study: The study in which the cluster will be duplicated. + area_id: The identifier of the area where the cluster will be duplicated. + source_id: The identifier of the cluster to be duplicated. + new_name: The new name for the duplicated cluster. + + Returns: + The duplicated cluster configuration. + + Raises: + ClusterAlreadyExists: If a cluster with the new name already exists in the area. + """ new_id = transform_name_to_id(new_name, lower=False) existing_ids = [cluster.id for cluster in self.get_clusters(study, area_id)] if new_id in existing_ids: raise ClusterAlreadyExists("Renewable", new_id) - # Cluster creation + + # Cluster duplication current_cluster = self.get_cluster(study, area_id, source_id) current_cluster.name = new_name creation_form = RenewableClusterCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) new_cluster = self.create_cluster(study, area_id, creation_form) + # Matrix edition current_path = f"input/renewables/series/{area_id}/{source_id.lower()}/series" new_path = f"input/renewables/series/{area_id}/{new_id.lower()}/series" @@ -298,4 +315,5 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name ) ] execute_or_add_commands(study, self._get_file_study(study), command, self.storage_service) + return new_cluster diff --git a/antarest/study/business/areas/st_storage_management.py b/antarest/study/business/areas/st_storage_management.py index 528fb26c9d..5a3c849cd5 100644 --- a/antarest/study/business/areas/st_storage_management.py +++ b/antarest/study/business/areas/st_storage_management.py @@ -205,7 +205,7 @@ def validate_rule_curve( upper_array = np.array(upper_rule_curve.data, dtype=np.float64) # noinspection PyUnresolvedReferences if (lower_array > upper_array).any(): - raise ValueError("Each 'lower_rule_curve' value must be lower" " or equal to each 'upper_rule_curve'") + raise ValueError("Each 'lower_rule_curve' value must be lower or equal to each 'upper_rule_curve'") return values @@ -421,15 +421,32 @@ def delete_storages( execute_or_add_commands(study, file_study, [command], self.storage_service) def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> STStorageOutput: + """ + Creates a duplicate cluster within the study area with a new name. + + Args: + study: The study in which the cluster will be duplicated. + area_id: The identifier of the area where the cluster will be duplicated. + source_id: The identifier of the cluster to be duplicated. + new_name: The new name for the duplicated cluster. + + Returns: + The duplicated cluster configuration. + + Raises: + ClusterAlreadyExists: If a cluster with the new name already exists in the area. + """ new_id = transform_name_to_id(new_name) existing_ids = [storage.id for storage in self.get_storages(study, area_id)] if new_id in existing_ids: raise ClusterAlreadyExists("Short term storage", new_id) - # Cluster creation + + # Cluster duplication current_cluster = self.get_storage(study, area_id, source_id) current_cluster.name = new_name creation_form = STStorageCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) new_storage = self.create_storage(study, area_id, creation_form) + # Matrix edition for ts_name in STStorageTimeSeries.__args__: # type: ignore ts = self.get_matrix(study, area_id, source_id, ts_name) diff --git a/antarest/study/business/areas/thermal_management.py b/antarest/study/business/areas/thermal_management.py index fc680d53f9..95c0c5d24f 100644 --- a/antarest/study/business/areas/thermal_management.py +++ b/antarest/study/business/areas/thermal_management.py @@ -290,15 +290,32 @@ def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[st execute_or_add_commands(study, file_study, commands, self.storage_service) def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> ThermalClusterOutput: + """ + Creates a duplicate cluster within the study area with a new name. + + Args: + study: The study in which the cluster will be duplicated. + area_id: The identifier of the area where the cluster will be duplicated. + source_id: The identifier of the cluster to be duplicated. + new_name: The new name for the duplicated cluster. + + Returns: + The duplicated cluster configuration. + + Raises: + ClusterAlreadyExists: If a cluster with the new name already exists in the area. + """ new_id = transform_name_to_id(new_name, lower=False) existing_ids = [cluster.id for cluster in self.get_clusters(study, area_id)] if new_id in existing_ids: raise ClusterAlreadyExists("Thermal", new_id) - # Cluster creation + + # Cluster duplication current_cluster = self.get_cluster(study, area_id, source_id) current_cluster.name = new_name creation_form = ThermalClusterCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) new_cluster = self.create_cluster(study, area_id, creation_form) + # Matrix edition lower_source_id = source_id.lower() lower_new_id = new_id.lower() From 9a83670b1b941297a76d2a1df4daa4c8ae7df27a Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 7 Mar 2024 16:17:41 +0100 Subject: [PATCH 07/15] feat(clusters): use case-sensitive enum for `ClusterType` --- antarest/study/business/utils.py | 7 ------- antarest/study/web/study_data_blueprint.py | 20 ++++++++++++++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/antarest/study/business/utils.py b/antarest/study/business/utils.py index efafcb7e82..53596bf797 100644 --- a/antarest/study/business/utils.py +++ b/antarest/study/business/utils.py @@ -8,7 +8,6 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.requests import RequestParameters from antarest.core.utils.string import to_camel_case -from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import RawStudy, Study from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.storage_service import StudyStorageService @@ -20,12 +19,6 @@ GENERAL_DATA_PATH = "settings/generaldata" -class ClusterType(EnumIgnoreCase): - ST_STORAGE = "storage" - RENEWABLE = "renewable" - THERMAL = "thermal" - - def execute_or_add_commands( study: Study, file_study: FileStudy, diff --git a/antarest/study/web/study_data_blueprint.py b/antarest/study/web/study_data_blueprint.py index 9be2889eb5..cddbee16e7 100644 --- a/antarest/study/web/study_data_blueprint.py +++ b/antarest/study/web/study_data_blueprint.py @@ -1,3 +1,4 @@ +import enum import logging from http import HTTPStatus from typing import Any, Dict, List, Optional, Sequence, Union, cast @@ -54,7 +55,6 @@ from antarest.study.business.table_mode_management import ColumnsModelTypes, TableTemplateType from antarest.study.business.thematic_trimming_management import ThematicTrimmingFormFields from antarest.study.business.timeseries_config_management import TSFormFields -from antarest.study.business.utils import ClusterType from antarest.study.model import PatchArea, PatchCluster from antarest.study.service import StudyService from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id @@ -62,6 +62,20 @@ logger = logging.getLogger(__name__) +class ClusterType(str, enum.Enum): + """ + Cluster type: + + - `STORAGE`: short-term storage + - `RENEWABLE`: renewable cluster + - `THERMAL`: thermal cluster + """ + + ST_STORAGE = "storage" + RENEWABLE = "renewable" + THERMAL = "thermal" + + def create_study_data_routes(study_service: StudyService, config: Config) -> APIRouter: """ Endpoint implementation for studies area management @@ -2048,8 +2062,10 @@ def duplicate_cluster( manager = STStorageManager(study_service.storage_service) elif cluster_type == ClusterType.RENEWABLE: manager = RenewableManager(study_service.storage_service) - else: + elif cluster_type == ClusterType.THERMAL: manager = ThermalManager(study_service.storage_service) + else: # pragma: no cover + raise NotImplementedError(f"Cluster type {cluster_type} not implemented") return manager.duplicate_cluster(study, area_id, source_cluster_id, new_cluster_name) From b098e7500fcf96d95b6be13fe167aa24c0d90040 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 7 Mar 2024 16:18:03 +0100 Subject: [PATCH 08/15] test(clusters): correct error message checking --- tests/integration/study_data_blueprint/test_renewable.py | 7 ++++--- tests/integration/study_data_blueprint/test_st_storage.py | 6 ++---- tests/integration/study_data_blueprint/test_thermal.py | 3 ++- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index 6737f53119..d697a7280d 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -229,9 +229,9 @@ def test_lifecycle( assert res.status_code == 200, res.json() assert res.json() == fr_solar_pv_cfg - # ============================= + # =============================== # RENEWABLE CLUSTER DUPLICATION - # ============================= + # =============================== new_name = "Duplicate of SolarPV" res = client.post( @@ -482,5 +482,6 @@ def test_lifecycle( ) assert res.status_code == 409 obj = res.json() - assert obj["description"] == f"Renewable cluster {other_cluster_id1} already exists and could not be created" + description = obj["description"] + assert f"'{other_cluster_id1}'" in description assert obj["exception"] == "ClusterAlreadyExists" diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index fbe8c89e33..1a76968923 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -495,10 +495,8 @@ def test_lifecycle__nominal( ) assert res.status_code == 409 obj = res.json() - assert ( - obj["description"] - == f"Short term storage cluster {siemens_battery_id} already exists and could not be created" - ) + description = obj["description"] + assert f"'{siemens_battery_id}'" in description assert obj["exception"] == "ClusterAlreadyExists" def test__default_values( diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index 43b3c1903c..380b8bbe0a 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -809,5 +809,6 @@ def test_lifecycle( ) assert res.status_code == 409 obj = res.json() - assert obj["description"] == f"Thermal cluster {duplicated_id} already exists and could not be created" + description = obj["description"] + assert f"'{duplicated_id}'" in description assert obj["exception"] == "ClusterAlreadyExists" From 88eda1306276c818722625a1c9b0e938faf34697 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 7 Mar 2024 16:23:39 +0100 Subject: [PATCH 09/15] feat(clusters): use plural form for cluster types --- antarest/study/web/study_data_blueprint.py | 18 +++++++++--------- .../study_data_blueprint/test_renewable.py | 6 +++--- .../study_data_blueprint/test_st_storage.py | 6 +++--- .../study_data_blueprint/test_thermal.py | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/antarest/study/web/study_data_blueprint.py b/antarest/study/web/study_data_blueprint.py index cddbee16e7..54109c2b00 100644 --- a/antarest/study/web/study_data_blueprint.py +++ b/antarest/study/web/study_data_blueprint.py @@ -66,14 +66,14 @@ class ClusterType(str, enum.Enum): """ Cluster type: - - `STORAGE`: short-term storage - - `RENEWABLE`: renewable cluster - - `THERMAL`: thermal cluster + - `STORAGE`: short-term storages + - `RENEWABLES`: renewable clusters + - `THERMALS`: thermal clusters """ - ST_STORAGE = "storage" - RENEWABLE = "renewable" - THERMAL = "thermal" + ST_STORAGES = "storages" + RENEWABLES = "renewables" + THERMALS = "thermals" def create_study_data_routes(study_service: StudyService, config: Config) -> APIRouter: @@ -2058,11 +2058,11 @@ def duplicate_cluster( study = study_service.check_study_access(uuid, StudyPermissionType.WRITE, params) manager: Union[STStorageManager, RenewableManager, ThermalManager] - if cluster_type == ClusterType.ST_STORAGE: + if cluster_type == ClusterType.ST_STORAGES: manager = STStorageManager(study_service.storage_service) - elif cluster_type == ClusterType.RENEWABLE: + elif cluster_type == ClusterType.RENEWABLES: manager = RenewableManager(study_service.storage_service) - elif cluster_type == ClusterType.THERMAL: + elif cluster_type == ClusterType.THERMALS: manager = ThermalManager(study_service.storage_service) else: # pragma: no cover raise NotImplementedError(f"Cluster type {cluster_type} not implemented") diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index d697a7280d..04aaa5a22a 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -235,7 +235,7 @@ def test_lifecycle( new_name = "Duplicate of SolarPV" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/renewable/{fr_solar_pv_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/renewables/{fr_solar_pv_id}?new_cluster_name={new_name}", headers={"Authorization": f"Bearer {user_access_token}"}, ) # asserts the config is the same @@ -467,7 +467,7 @@ def test_lifecycle( # Cannot duplicate a fake cluster fake_id = "fake_id" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/renewable/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/renewables/{fake_id}?new_cluster_name=duplicata", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 404 @@ -477,7 +477,7 @@ def test_lifecycle( # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/renewable/{other_cluster_id1}?new_cluster_name={other_cluster_id1}", + f"/v1/studies/{study_id}/areas/{area_id}/renewables/{other_cluster_id1}?new_cluster_name={other_cluster_id1}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 409 diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index 1a76968923..c31a2abc20 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -239,7 +239,7 @@ def test_lifecycle__nominal( new_name = "Duplicate of Siemens" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storage/{siemens_battery_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={new_name}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code in {200, 201} @@ -480,7 +480,7 @@ def test_lifecycle__nominal( # Cannot duplicate a fake st-storage fake_id = "fake_id" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storage/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/storages/{fake_id}?new_cluster_name=duplicata", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 404 @@ -490,7 +490,7 @@ def test_lifecycle__nominal( # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storage/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", + f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 409 diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index 380b8bbe0a..813d41c48a 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -560,7 +560,7 @@ def test_lifecycle( new_name = "Duplicate of Fr_Gas_Conventional" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/thermal/{fr_gas_conventional_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/thermals/{fr_gas_conventional_id}?new_cluster_name={new_name}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code in {200, 201} @@ -794,7 +794,7 @@ def test_lifecycle( # Cannot duplicate a fake cluster fake_id = "fake_id" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/thermal/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/thermals/{fake_id}?new_cluster_name=duplicata", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 404 @@ -804,7 +804,7 @@ def test_lifecycle( # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/thermal/{duplicated_id}?new_cluster_name={duplicated_id}", + f"/v1/studies/{study_id}/areas/{area_id}/thermals/{duplicated_id}?new_cluster_name={duplicated_id}", headers={"Authorization": f"Bearer {user_access_token}"}, ) assert res.status_code == 409 From 39516d646578cf12b2b133c53c0d6505288b0535 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 7 Mar 2024 16:49:53 +0100 Subject: [PATCH 10/15] fix(clusters): correct the detection of duplicates --- .../business/areas/renewable_management.py | 3 +-- .../business/areas/st_storage_management.py | 8 +++---- .../business/areas/thermal_management.py | 3 +-- antarest/study/web/study_data_blueprint.py | 2 +- .../study_data_blueprint/test_renewable.py | 22 +++++++++++-------- .../study_data_blueprint/test_st_storage.py | 21 ++++++++++-------- .../study_data_blueprint/test_thermal.py | 21 ++++++++++-------- 7 files changed, 44 insertions(+), 36 deletions(-) diff --git a/antarest/study/business/areas/renewable_management.py b/antarest/study/business/areas/renewable_management.py index de16dff11b..fac9d925a8 100644 --- a/antarest/study/business/areas/renewable_management.py +++ b/antarest/study/business/areas/renewable_management.py @@ -293,8 +293,7 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name ClusterAlreadyExists: If a cluster with the new name already exists in the area. """ new_id = transform_name_to_id(new_name, lower=False) - existing_ids = [cluster.id for cluster in self.get_clusters(study, area_id)] - if new_id in existing_ids: + if any(new_id.lower() == cluster.id.lower() for cluster in self.get_clusters(study, area_id)): raise ClusterAlreadyExists("Renewable", new_id) # Cluster duplication diff --git a/antarest/study/business/areas/st_storage_management.py b/antarest/study/business/areas/st_storage_management.py index 5a3c849cd5..0afdfe7e34 100644 --- a/antarest/study/business/areas/st_storage_management.py +++ b/antarest/study/business/areas/st_storage_management.py @@ -437,9 +437,8 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name ClusterAlreadyExists: If a cluster with the new name already exists in the area. """ new_id = transform_name_to_id(new_name) - existing_ids = [storage.id for storage in self.get_storages(study, area_id)] - if new_id in existing_ids: - raise ClusterAlreadyExists("Short term storage", new_id) + if any(new_id.lower() == storage.id.lower() for storage in self.get_storages(study, area_id)): + raise ClusterAlreadyExists("Short-term storage", new_id) # Cluster duplication current_cluster = self.get_storage(study, area_id, source_id) @@ -450,7 +449,8 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name # Matrix edition for ts_name in STStorageTimeSeries.__args__: # type: ignore ts = self.get_matrix(study, area_id, source_id, ts_name) - self.update_matrix(study, area_id, new_id, ts_name, ts) + self.update_matrix(study, area_id, new_id.lower(), ts_name, ts) + return new_storage def get_matrix( diff --git a/antarest/study/business/areas/thermal_management.py b/antarest/study/business/areas/thermal_management.py index 95c0c5d24f..00b193bd09 100644 --- a/antarest/study/business/areas/thermal_management.py +++ b/antarest/study/business/areas/thermal_management.py @@ -306,8 +306,7 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name ClusterAlreadyExists: If a cluster with the new name already exists in the area. """ new_id = transform_name_to_id(new_name, lower=False) - existing_ids = [cluster.id for cluster in self.get_clusters(study, area_id)] - if new_id in existing_ids: + if any(new_id.lower() == cluster.id.lower() for cluster in self.get_clusters(study, area_id)): raise ClusterAlreadyExists("Thermal", new_id) # Cluster duplication diff --git a/antarest/study/web/study_data_blueprint.py b/antarest/study/web/study_data_blueprint.py index 54109c2b00..893384b8ef 100644 --- a/antarest/study/web/study_data_blueprint.py +++ b/antarest/study/web/study_data_blueprint.py @@ -2047,7 +2047,7 @@ def duplicate_cluster( area_id: str, cluster_type: ClusterType, source_cluster_id: str, - new_cluster_name: str, + new_cluster_name: str = Query(..., alias="newName"), # type: ignore current_user: JWTUser = Depends(auth.get_current_user), ) -> Union[STStorageOutput, ThermalClusterOutput, RenewableClusterOutput]: logger.info( diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index 04aaa5a22a..a77e5064cb 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -235,11 +235,12 @@ def test_lifecycle( new_name = "Duplicate of SolarPV" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/renewables/{fr_solar_pv_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/renewables/{fr_solar_pv_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name}, ) # asserts the config is the same - assert res.status_code in {200, 201} + assert res.status_code in {200, 201}, res.json() duplicated_config = copy.deepcopy(fr_solar_pv_cfg) duplicated_config["name"] = new_name duplicated_id = transform_name_to_id(new_name, lower=False) @@ -282,10 +283,11 @@ def test_lifecycle( # It's possible to delete multiple renewable clusters at once. # Create two clusters + other_cluster_name = "Other Cluster 1" res = client.post( f"/v1/studies/{study_id}/areas/{area_id}/clusters/renewable", headers={"Authorization": f"Bearer {user_access_token}"}, - json={"name": "Other Cluster 1"}, + json={"name": other_cluster_name}, ) assert res.status_code == 200, res.json() other_cluster_id1 = res.json()["id"] @@ -465,23 +467,25 @@ def test_lifecycle( assert bad_study_id in description # Cannot duplicate a fake cluster - fake_id = "fake_id" + unknown_id = "unknown" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/renewables/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/renewables/{unknown_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": "duplicata"}, ) assert res.status_code == 404 obj = res.json() - assert obj["description"] == f"Cluster: '{fake_id}' not found" + assert obj["description"] == f"Cluster: '{unknown_id}' not found" assert obj["exception"] == "ClusterNotFound" # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/renewables/{other_cluster_id1}?new_cluster_name={other_cluster_id1}", + f"/v1/studies/{study_id}/areas/{area_id}/renewables/{other_cluster_id1}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": other_cluster_name.upper()}, # different case, but same ID ) - assert res.status_code == 409 + assert res.status_code == 409, res.json() obj = res.json() description = obj["description"] - assert f"'{other_cluster_id1}'" in description + assert other_cluster_name.upper() in description assert obj["exception"] == "ClusterAlreadyExists" diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index c31a2abc20..67e2bf7b24 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -239,10 +239,11 @@ def test_lifecycle__nominal( new_name = "Duplicate of Siemens" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name}, ) - assert res.status_code in {200, 201} + assert res.status_code in {200, 201}, res.json() # asserts the config is the same duplicated_config = copy.deepcopy(siemens_config) duplicated_config["name"] = new_name # type: ignore @@ -478,25 +479,27 @@ def test_lifecycle__nominal( assert bad_study_id in description # Cannot duplicate a fake st-storage - fake_id = "fake_id" + unknown_id = "unknown" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/storages/{unknown_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": "duplicata"}, ) - assert res.status_code == 404 + assert res.status_code == 404, res.json() obj = res.json() - assert obj["description"] == f"Fields of storage '{fake_id}' not found" + assert obj["description"] == f"Fields of storage '{unknown_id}' not found" assert obj["exception"] == "STStorageFieldsNotFoundError" # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}?new_cluster_name={siemens_battery_id}", + f"/v1/studies/{study_id}/areas/{area_id}/storages/{siemens_battery_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": siemens_battery.upper()}, # different case, but same ID ) - assert res.status_code == 409 + assert res.status_code == 409, res.json() obj = res.json() description = obj["description"] - assert f"'{siemens_battery_id}'" in description + assert siemens_battery.lower() in description assert obj["exception"] == "ClusterAlreadyExists" def test__default_values( diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index 813d41c48a..74f03bb804 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -560,10 +560,11 @@ def test_lifecycle( new_name = "Duplicate of Fr_Gas_Conventional" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/thermals/{fr_gas_conventional_id}?new_cluster_name={new_name}", + f"/v1/studies/{study_id}/areas/{area_id}/thermals/{fr_gas_conventional_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name}, ) - assert res.status_code in {200, 201} + assert res.status_code in {200, 201}, res.json() # asserts the config is the same duplicated_config = copy.deepcopy(fr_gas_conventional_cfg) duplicated_config["name"] = new_name # type: ignore @@ -792,23 +793,25 @@ def test_lifecycle( assert bad_study_id in description # Cannot duplicate a fake cluster - fake_id = "fake_id" + unknown_id = "unknown" res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/thermals/{fake_id}?new_cluster_name=duplicata", + f"/v1/studies/{study_id}/areas/{area_id}/thermals/{unknown_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": "duplicate"}, ) - assert res.status_code == 404 + assert res.status_code == 404, res.json() obj = res.json() - assert obj["description"] == f"Cluster: '{fake_id}' not found" + assert obj["description"] == f"Cluster: '{unknown_id}' not found" assert obj["exception"] == "ClusterNotFound" # Cannot duplicate with an existing id res = client.post( - f"/v1/studies/{study_id}/areas/{area_id}/thermals/{duplicated_id}?new_cluster_name={duplicated_id}", + f"/v1/studies/{study_id}/areas/{area_id}/thermals/{duplicated_id}", headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name.upper()}, # different case but same ID ) - assert res.status_code == 409 + assert res.status_code == 409, res.json() obj = res.json() description = obj["description"] - assert f"'{duplicated_id}'" in description + assert new_name.upper() in description assert obj["exception"] == "ClusterAlreadyExists" From 277668c0efdf3b57653d41145f42f7b8bfc5c22e Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 7 Mar 2024 16:53:50 +0100 Subject: [PATCH 11/15] test(clusters): replace usage of `copy.deepcopy` --- tests/integration/study_data_blueprint/test_renewable.py | 3 +-- tests/integration/study_data_blueprint/test_st_storage.py | 3 +-- tests/integration/study_data_blueprint/test_thermal.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index a77e5064cb..4bb71a791f 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -23,7 +23,6 @@ * delete a cluster (or several clusters) * validate the consistency of the matrices (and properties) """ -import copy import json import re @@ -241,7 +240,7 @@ def test_lifecycle( ) # asserts the config is the same assert res.status_code in {200, 201}, res.json() - duplicated_config = copy.deepcopy(fr_solar_pv_cfg) + duplicated_config = dict(fr_solar_pv_cfg) duplicated_config["name"] = new_name duplicated_id = transform_name_to_id(new_name, lower=False) duplicated_config["id"] = duplicated_id diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index 67e2bf7b24..322c4669ae 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -1,4 +1,3 @@ -import copy import json import re from unittest.mock import ANY @@ -245,7 +244,7 @@ def test_lifecycle__nominal( ) assert res.status_code in {200, 201}, res.json() # asserts the config is the same - duplicated_config = copy.deepcopy(siemens_config) + duplicated_config = dict(siemens_config) duplicated_config["name"] = new_name # type: ignore duplicated_id = transform_name_to_id(new_name) duplicated_config["id"] = duplicated_id # type: ignore diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index 74f03bb804..ab3e7745a8 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -27,7 +27,6 @@ * delete a cluster (or several clusters) * validate the consistency of the matrices (and properties) """ -import copy import json import re @@ -566,7 +565,7 @@ def test_lifecycle( ) assert res.status_code in {200, 201}, res.json() # asserts the config is the same - duplicated_config = copy.deepcopy(fr_gas_conventional_cfg) + duplicated_config = dict(fr_gas_conventional_cfg) duplicated_config["name"] = new_name # type: ignore duplicated_id = transform_name_to_id(new_name, lower=False) duplicated_config["id"] = duplicated_id # type: ignore From ac960c2a05689242a5955143489cfb09b2a7bd86 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 7 Mar 2024 17:00:15 +0100 Subject: [PATCH 12/15] feat(clusters): improve API documentation --- antarest/study/web/study_data_blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/antarest/study/web/study_data_blueprint.py b/antarest/study/web/study_data_blueprint.py index 893384b8ef..bc667f45d5 100644 --- a/antarest/study/web/study_data_blueprint.py +++ b/antarest/study/web/study_data_blueprint.py @@ -2047,7 +2047,7 @@ def duplicate_cluster( area_id: str, cluster_type: ClusterType, source_cluster_id: str, - new_cluster_name: str = Query(..., alias="newName"), # type: ignore + new_cluster_name: str = Query(..., alias="newName", title="New Cluster Name"), # type: ignore current_user: JWTUser = Depends(auth.get_current_user), ) -> Union[STStorageOutput, ThermalClusterOutput, RenewableClusterOutput]: logger.info( From 80a69cace415efa3430338c47823a59200b66c5d Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 8 Mar 2024 09:54:34 +0100 Subject: [PATCH 13/15] fix(clusters): correct thermal cluster duplication for variants --- .../business/areas/thermal_management.py | 79 +++++---- .../study_data_blueprint/test_thermal.py | 150 +++++++++++++++++- 2 files changed, 193 insertions(+), 36 deletions(-) diff --git a/antarest/study/business/areas/thermal_management.py b/antarest/study/business/areas/thermal_management.py index 00b193bd09..f44ad7ba10 100644 --- a/antarest/study/business/areas/thermal_management.py +++ b/antarest/study/business/areas/thermal_management.py @@ -42,7 +42,7 @@ class Config: def schema_extra(schema: t.MutableMapping[str, t.Any]) -> None: schema["example"] = ThermalClusterInput( group="Gas", - name="2 avail and must 1", + name="Gas Cluster XY", enabled=False, unitCount=100, nominalCapacity=1000.0, @@ -81,9 +81,9 @@ class Config: @staticmethod def schema_extra(schema: t.MutableMapping[str, t.Any]) -> None: schema["example"] = ThermalClusterOutput( - id="2 avail and must 1", + id="Gas cluster YZ", group="Gas", - name="2 avail and must 1", + name="Gas Cluster YZ", enabled=False, unitCount=100, nominalCapacity=1000.0, @@ -192,16 +192,8 @@ def create_cluster(self, study: Study, area_id: str, cluster_data: ThermalCluste """ file_study = self._get_file_study(study) - study_version = study.version - cluster = cluster_data.to_config(study_version) - # NOTE: currently, in the `CreateCluster` class, there is a confusion - # between the cluster name and the cluster ID (which is a section name). - command = CreateCluster( - area_id=area_id, - cluster_name=cluster.id, - parameters=cluster.dict(by_alias=True, exclude={"id"}), - command_context=self.storage_service.variant_study_service.command_factory.command_context, - ) + cluster = cluster_data.to_config(study.version) + command = self._make_create_cluster_cmd(area_id, cluster) execute_or_add_commands( study, file_study, @@ -211,6 +203,17 @@ def create_cluster(self, study: Study, area_id: str, cluster_data: ThermalCluste output = self.get_cluster(study, area_id, cluster.id) return output + def _make_create_cluster_cmd(self, area_id: str, cluster: ThermalConfigType) -> CreateCluster: + # NOTE: currently, in the `CreateCluster` class, there is a confusion + # between the cluster name and the cluster ID (which is a section name). + command = CreateCluster( + area_id=area_id, + cluster_name=cluster.id, + parameters=cluster.dict(by_alias=True, exclude={"id"}), + command_context=self.storage_service.variant_study_service.command_factory.command_context, + ) + return command + def update_cluster( self, study: Study, @@ -289,7 +292,13 @@ def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[st execute_or_add_commands(study, file_study, commands, self.storage_service) - def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> ThermalClusterOutput: + def duplicate_cluster( + self, + study: Study, + area_id: str, + source_id: str, + new_cluster_name: str, + ) -> ThermalClusterOutput: """ Creates a duplicate cluster within the study area with a new name. @@ -297,7 +306,7 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name study: The study in which the cluster will be duplicated. area_id: The identifier of the area where the cluster will be duplicated. source_id: The identifier of the cluster to be duplicated. - new_name: The new name for the duplicated cluster. + new_cluster_name: The new name for the duplicated cluster. Returns: The duplicated cluster configuration. @@ -305,20 +314,21 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name Raises: ClusterAlreadyExists: If a cluster with the new name already exists in the area. """ - new_id = transform_name_to_id(new_name, lower=False) - if any(new_id.lower() == cluster.id.lower() for cluster in self.get_clusters(study, area_id)): + new_id = transform_name_to_id(new_cluster_name, lower=False) + lower_new_id = new_id.lower() + if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)): raise ClusterAlreadyExists("Thermal", new_id) # Cluster duplication - current_cluster = self.get_cluster(study, area_id, source_id) - current_cluster.name = new_name - creation_form = ThermalClusterCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) - new_cluster = self.create_cluster(study, area_id, creation_form) + source_cluster = self.get_cluster(study, area_id, source_id) + source_cluster.name = new_cluster_name + creation_form = ThermalClusterCreation(**source_cluster.dict(by_alias=False, exclude={"id"})) + new_config = creation_form.to_config(study.version) + create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config) # Matrix edition lower_source_id = source_id.lower() - lower_new_id = new_id.lower() - paths = [ + source_paths = [ f"input/thermal/series/{area_id}/{lower_source_id}/series", f"input/thermal/prepro/{area_id}/{lower_source_id}/modulation", f"input/thermal/prepro/{area_id}/{lower_source_id}/data", @@ -328,15 +338,16 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name f"input/thermal/prepro/{area_id}/{lower_new_id}/modulation", f"input/thermal/prepro/{area_id}/{lower_new_id}/data", ] - commands = [] - storage_service = self.storage_service - for k, matrix_path in enumerate(paths): - current_matrix = storage_service.raw_study_service.get(study, matrix_path) - command = ReplaceMatrix( - target=new_paths[k], - matrix=current_matrix["data"], - command_context=storage_service.variant_study_service.command_factory.command_context, - ) + + # Prepare and execute commands + commands: t.List[t.Union[CreateCluster, ReplaceMatrix]] = [create_cluster_cmd] + storage_service = self.storage_service.get_storage(study) + command_context = self.storage_service.variant_study_service.command_factory.command_context + for source_path, new_path in zip(source_paths, new_paths): + current_matrix = storage_service.get(study, source_path)["data"] + command = ReplaceMatrix(target=new_path, matrix=current_matrix, command_context=command_context) commands.append(command) - execute_or_add_commands(study, self._get_file_study(study), commands, storage_service) - return new_cluster + + execute_or_add_commands(study, self._get_file_study(study), commands, self.storage_service) + + return ThermalClusterOutput(**new_config.dict(by_alias=False)) diff --git a/tests/integration/study_data_blueprint/test_thermal.py b/tests/integration/study_data_blueprint/test_thermal.py index ab3e7745a8..9fc7388642 100644 --- a/tests/integration/study_data_blueprint/test_thermal.py +++ b/tests/integration/study_data_blueprint/test_thermal.py @@ -29,6 +29,7 @@ """ import json import re +import typing as t import numpy as np import pytest @@ -566,9 +567,9 @@ def test_lifecycle( assert res.status_code in {200, 201}, res.json() # asserts the config is the same duplicated_config = dict(fr_gas_conventional_cfg) - duplicated_config["name"] = new_name # type: ignore + duplicated_config["name"] = new_name duplicated_id = transform_name_to_id(new_name, lower=False) - duplicated_config["id"] = duplicated_id # type: ignore + duplicated_config["id"] = duplicated_id assert res.json() == duplicated_config # asserts the matrix has also been duplicated @@ -814,3 +815,148 @@ def test_lifecycle( description = obj["description"] assert new_name.upper() in description assert obj["exception"] == "ClusterAlreadyExists" + + @pytest.fixture(name="base_study_id") + def base_study_id_fixture(self, request: t.Any, client: TestClient, user_access_token: str) -> str: + """Prepare a managed study for the variant study tests.""" + params = request.param + res = client.post( + "/v1/studies", + headers={"Authorization": f"Bearer {user_access_token}"}, + params=params, + ) + assert res.status_code in {200, 201}, res.json() + study_id: str = res.json() + return study_id + + @pytest.fixture(name="variant_id") + def variant_id_fixture(self, request: t.Any, client: TestClient, user_access_token: str, base_study_id: str) -> str: + """Prepare a variant study for the variant study tests.""" + name = request.param + res = client.post( + f"/v1/studies/{base_study_id}/variants", + headers={"Authorization": f"Bearer {user_access_token}"}, + params={"name": name}, + ) + assert res.status_code in {200, 201}, res.json() + study_id: str = res.json() + return study_id + + # noinspection PyTestParametrized + @pytest.mark.parametrize("base_study_id", [{"name": "Base Study", "version": 860}], indirect=True) + @pytest.mark.parametrize("variant_id", ["Variant Study"], indirect=True) + def test_variant_lifecycle(self, client: TestClient, user_access_token: str, variant_id: str) -> None: + """ + In this test, we want to check that thermal clusters can be managed + in the context of a "variant" study. + """ + # Create an area + area_name = "France" + res = client.post( + f"/v1/studies/{variant_id}/areas", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={"name": area_name, "type": "AREA"}, + ) + assert res.status_code in {200, 201}, res.json() + area_cfg = res.json() + area_id = area_cfg["id"] + + # Create a thermal cluster + cluster_name = "Th1" + res = client.post( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/thermal", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={ + "name": cluster_name, + "group": "Nuclear", + "unitCount": 13, + "nominalCapacity": 42500, + "marginalCost": 0.1, + }, + ) + assert res.status_code in {200, 201}, res.json() + cluster_id: str = res.json()["id"] + + # Update the thermal cluster + res = client.patch( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/thermal/{cluster_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={ + "marginalCost": 0.2, + }, + ) + assert res.status_code == 200, res.json() + cluster_cfg = res.json() + assert cluster_cfg["marginalCost"] == 0.2 + + # Update the prepro matrix + matrix = np.random.randint(0, 2, size=(8760, 1)).tolist() + matrix_path = f"input/thermal/prepro/{area_id}/{cluster_id.lower()}/data" + args = {"target": matrix_path, "matrix": matrix} + res = client.post( + f"/v1/studies/{variant_id}/commands", + json=[{"action": "replace_matrix", "args": args}], + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201}, res.json() + + # Duplicate the thermal cluster + new_name = "Th2" + res = client.post( + f"/v1/studies/{variant_id}/areas/{area_id}/thermals/{cluster_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name}, + ) + assert res.status_code in {200, 201}, res.json() + cluster_cfg = res.json() + assert cluster_cfg["name"] == new_name + new_id = cluster_cfg["id"] + + # Check that the duplicate has the right properties + res = client.get( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/thermal/{new_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200, res.json() + cluster_cfg = res.json() + assert cluster_cfg["group"] == "Nuclear" + assert cluster_cfg["unitCount"] == 13 + assert cluster_cfg["nominalCapacity"] == 42500 + assert cluster_cfg["marginalCost"] == 0.2 + + # Check that the duplicate has the right matrix + new_cluster_matrix_path = f"input/thermal/prepro/{area_id}/{new_id.lower()}/data" + res = client.get( + f"/v1/studies/{variant_id}/raw", + params={"path": new_cluster_matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix + + # Delete the thermal cluster + res = client.delete( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/thermal", + headers={"Authorization": f"Bearer {user_access_token}"}, + json=[cluster_id], + ) + assert res.status_code == 204, res.json() + + # Check the list of variant commands + res = client.get( + f"/v1/studies/{variant_id}/commands", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200, res.json() + commands = res.json() + assert len(commands) == 7 + actions = [command["action"] for command in commands] + assert actions == [ + "create_area", + "create_cluster", + "update_config", + "replace_matrix", + "create_cluster", + "replace_matrix", + "remove_cluster", + ] From 4e4d1579f1332b370b1745f0ea7f86f53adf15a2 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 8 Mar 2024 17:43:42 +0100 Subject: [PATCH 14/15] fix(clusters): correct renewable cluster duplication for variants --- .../business/areas/renewable_management.py | 73 +++++---- .../study_data_blueprint/test_renewable.py | 142 ++++++++++++++++++ 2 files changed, 184 insertions(+), 31 deletions(-) diff --git a/antarest/study/business/areas/renewable_management.py b/antarest/study/business/areas/renewable_management.py index fac9d925a8..c4152924bf 100644 --- a/antarest/study/business/areas/renewable_management.py +++ b/antarest/study/business/areas/renewable_management.py @@ -49,7 +49,7 @@ class Config: def schema_extra(schema: t.MutableMapping[str, t.Any]) -> None: schema["example"] = RenewableClusterInput( group="Gas", - name="2 avail and must 1", + name="Gas Cluster XY", enabled=False, unitCount=100, nominalCapacity=1000.0, @@ -87,9 +87,9 @@ class Config: @staticmethod def schema_extra(schema: t.MutableMapping[str, t.Any]) -> None: schema["example"] = RenewableClusterOutput( - id="2 avail and must 1", + id="Gas cluster YZ", group="Gas", - name="2 avail and must 1", + name="Gas Cluster YZ", enabled=False, unitCount=100, nominalCapacity=1000.0, @@ -159,23 +159,25 @@ def create_cluster( The newly created cluster. """ file_study = self._get_file_study(study) - study_version = study.version - cluster = cluster_data.to_config(study_version) - - command = CreateRenewablesCluster( - area_id=area_id, - cluster_name=cluster.id, - parameters=cluster.dict(by_alias=True, exclude={"id"}), - command_context=self.storage_service.variant_study_service.command_factory.command_context, - ) + cluster = cluster_data.to_config(study.version) + command = self._make_create_cluster_cmd(area_id, cluster) execute_or_add_commands( study, file_study, [command], self.storage_service, ) + output = self.get_cluster(study, area_id, cluster.id) + return output - return self.get_cluster(study, area_id, cluster.id) + def _make_create_cluster_cmd(self, area_id: str, cluster: RenewableConfigType) -> CreateRenewablesCluster: + command = CreateRenewablesCluster( + area_id=area_id, + cluster_name=cluster.id, + parameters=cluster.dict(by_alias=True, exclude={"id"}), + command_context=self.storage_service.variant_study_service.command_factory.command_context, + ) + return command def get_cluster(self, study: Study, area_id: str, cluster_id: str) -> RenewableClusterOutput: """ @@ -276,7 +278,13 @@ def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[st execute_or_add_commands(study, file_study, commands, self.storage_service) - def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> RenewableClusterOutput: + def duplicate_cluster( + self, + study: Study, + area_id: str, + source_id: str, + new_cluster_name: str, + ) -> RenewableClusterOutput: """ Creates a duplicate cluster within the study area with a new name. @@ -284,7 +292,7 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name study: The study in which the cluster will be duplicated. area_id: The identifier of the area where the cluster will be duplicated. source_id: The identifier of the cluster to be duplicated. - new_name: The new name for the duplicated cluster. + new_cluster_name: The new name for the duplicated cluster. Returns: The duplicated cluster configuration. @@ -292,27 +300,30 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name Raises: ClusterAlreadyExists: If a cluster with the new name already exists in the area. """ - new_id = transform_name_to_id(new_name, lower=False) - if any(new_id.lower() == cluster.id.lower() for cluster in self.get_clusters(study, area_id)): + new_id = transform_name_to_id(new_cluster_name, lower=False) + lower_new_id = new_id.lower() + if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)): raise ClusterAlreadyExists("Renewable", new_id) # Cluster duplication current_cluster = self.get_cluster(study, area_id, source_id) - current_cluster.name = new_name + current_cluster.name = new_cluster_name creation_form = RenewableClusterCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) - new_cluster = self.create_cluster(study, area_id, creation_form) + new_config = creation_form.to_config(study.version) + create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config) # Matrix edition - current_path = f"input/renewables/series/{area_id}/{source_id.lower()}/series" - new_path = f"input/renewables/series/{area_id}/{new_id.lower()}/series" - current_matrix = self.storage_service.raw_study_service.get(study, current_path) - command = [ - ReplaceMatrix( - target=new_path, - matrix=current_matrix["data"], - command_context=self.storage_service.variant_study_service.command_factory.command_context, - ) - ] - execute_or_add_commands(study, self._get_file_study(study), command, self.storage_service) + lower_source_id = source_id.lower() + source_path = f"input/renewables/series/{area_id}/{lower_source_id}/series" + new_path = f"input/renewables/series/{area_id}/{lower_new_id}/series" + + # Prepare and execute commands + storage_service = self.storage_service.get_storage(study) + command_context = self.storage_service.variant_study_service.command_factory.command_context + current_matrix = storage_service.get(study, source_path)["data"] + replace_matrix_cmd = ReplaceMatrix(target=new_path, matrix=current_matrix, command_context=command_context) + commands = [create_cluster_cmd, replace_matrix_cmd] + + execute_or_add_commands(study, self._get_file_study(study), commands, self.storage_service) - return new_cluster + return RenewableClusterOutput(**new_config.dict(by_alias=False)) diff --git a/tests/integration/study_data_blueprint/test_renewable.py b/tests/integration/study_data_blueprint/test_renewable.py index 4bb71a791f..8447c0430f 100644 --- a/tests/integration/study_data_blueprint/test_renewable.py +++ b/tests/integration/study_data_blueprint/test_renewable.py @@ -25,6 +25,7 @@ """ import json import re +import typing as t import numpy as np import pytest @@ -488,3 +489,144 @@ def test_lifecycle( description = obj["description"] assert other_cluster_name.upper() in description assert obj["exception"] == "ClusterAlreadyExists" + + @pytest.fixture(name="base_study_id") + def base_study_id_fixture(self, request: t.Any, client: TestClient, user_access_token: str) -> str: + """Prepare a managed study for the variant study tests.""" + params = request.param + res = client.post( + "/v1/studies", + headers={"Authorization": f"Bearer {user_access_token}"}, + params=params, + ) + assert res.status_code in {200, 201}, res.json() + study_id: str = res.json() + return study_id + + @pytest.fixture(name="variant_id") + def variant_id_fixture(self, request: t.Any, client: TestClient, user_access_token: str, base_study_id: str) -> str: + """Prepare a variant study for the variant study tests.""" + name = request.param + res = client.post( + f"/v1/studies/{base_study_id}/variants", + headers={"Authorization": f"Bearer {user_access_token}"}, + params={"name": name}, + ) + assert res.status_code in {200, 201}, res.json() + study_id: str = res.json() + return study_id + + # noinspection PyTestParametrized + @pytest.mark.parametrize("base_study_id", [{"name": "Base Study", "version": 860}], indirect=True) + @pytest.mark.parametrize("variant_id", ["Variant Study"], indirect=True) + def test_variant_lifecycle(self, client: TestClient, user_access_token: str, variant_id: str) -> None: + """ + In this test, we want to check that renewable clusters can be managed + in the context of a "variant" study. + """ + # Create an area + area_name = "France" + res = client.post( + f"/v1/studies/{variant_id}/areas", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={"name": area_name, "type": "AREA"}, + ) + assert res.status_code in {200, 201}, res.json() + area_cfg = res.json() + area_id = area_cfg["id"] + + # Create a renewable cluster + cluster_name = "Th1" + res = client.post( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/renewable", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={ + "name": cluster_name, + "group": "Wind Offshore", + "unitCount": 13, + "nominalCapacity": 42500, + }, + ) + assert res.status_code in {200, 201}, res.json() + cluster_id: str = res.json()["id"] + + # Update the renewable cluster + res = client.patch( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/renewable/{cluster_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={"unitCount": 15}, + ) + assert res.status_code == 200, res.json() + cluster_cfg = res.json() + assert cluster_cfg["unitCount"] == 15 + + # Update the series matrix + matrix = np.random.randint(0, 2, size=(8760, 1)).tolist() + matrix_path = f"input/renewables/series/{area_id}/{cluster_id.lower()}/series" + args = {"target": matrix_path, "matrix": matrix} + res = client.post( + f"/v1/studies/{variant_id}/commands", + json=[{"action": "replace_matrix", "args": args}], + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201}, res.json() + + # Duplicate the renewable cluster + new_name = "Th2" + res = client.post( + f"/v1/studies/{variant_id}/areas/{area_id}/renewables/{cluster_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name}, + ) + assert res.status_code in {200, 201}, res.json() + cluster_cfg = res.json() + assert cluster_cfg["name"] == new_name + new_id = cluster_cfg["id"] + + # Check that the duplicate has the right properties + res = client.get( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/renewable/{new_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200, res.json() + cluster_cfg = res.json() + assert cluster_cfg["group"] == "Wind Offshore" + assert cluster_cfg["unitCount"] == 15 + assert cluster_cfg["nominalCapacity"] == 42500 + + # Check that the duplicate has the right matrix + new_cluster_matrix_path = f"input/renewables/series/{area_id}/{new_id.lower()}/series" + res = client.get( + f"/v1/studies/{variant_id}/raw", + params={"path": new_cluster_matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix + + # Delete the renewable cluster + res = client.delete( + f"/v1/studies/{variant_id}/areas/{area_id}/clusters/renewable", + headers={"Authorization": f"Bearer {user_access_token}"}, + json=[cluster_id], + ) + assert res.status_code == 204, res.json() + + # Check the list of variant commands + res = client.get( + f"/v1/studies/{variant_id}/commands", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200, res.json() + commands = res.json() + assert len(commands) == 7 + actions = [command["action"] for command in commands] + assert actions == [ + "create_area", + "create_renewables_cluster", + "update_config", + "replace_matrix", + "create_renewables_cluster", + "replace_matrix", + "remove_renewables_cluster", + ] From 2c297032438580642cad2fe79ec80f6ca4255ba2 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 8 Mar 2024 18:51:56 +0100 Subject: [PATCH 15/15] fix(clusters): correct short-term storage duplication for variants --- .../business/areas/st_storage_management.py | 72 ++++++--- .../study_data_blueprint/test_st_storage.py | 144 ++++++++++++++++++ 2 files changed, 192 insertions(+), 24 deletions(-) diff --git a/antarest/study/business/areas/st_storage_management.py b/antarest/study/business/areas/st_storage_management.py index 0afdfe7e34..ca498c030a 100644 --- a/antarest/study/business/areas/st_storage_management.py +++ b/antarest/study/business/areas/st_storage_management.py @@ -26,6 +26,7 @@ from antarest.study.storage.storage_service import StudyStorageService from antarest.study.storage.variantstudy.model.command.create_st_storage import CreateSTStorage from antarest.study.storage.variantstudy.model.command.remove_st_storage import RemoveSTStorage +from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig __all__ = ( @@ -74,8 +75,8 @@ def validate_name(cls, name: t.Optional[str]) -> str: raise ValueError("'name' must not be empty") return name - @property - def to_config(self) -> STStorageConfig: + # noinspection PyUnusedLocal + def to_config(self, study_version: t.Union[str, int]) -> STStorageConfig: values = self.dict(by_alias=False, exclude_none=True) return STStorageConfig(**values) @@ -259,21 +260,25 @@ def create_storage( Returns: The ID of the newly created short-term storage. """ - storage = form.to_config - command = CreateSTStorage( - area_id=area_id, - parameters=storage, - command_context=self.storage_service.variant_study_service.command_factory.command_context, - ) file_study = self._get_file_study(study) + storage = form.to_config(study.version) + command = self._make_create_cluster_cmd(area_id, storage) execute_or_add_commands( study, file_study, [command], self.storage_service, ) + output = self.get_storage(study, area_id, storage_id=storage.id) + return output - return self.get_storage(study, area_id, storage_id=storage.id) + def _make_create_cluster_cmd(self, area_id: str, cluster: STStorageConfig) -> CreateSTStorage: + command = CreateSTStorage( + area_id=area_id, + parameters=cluster, + command_context=self.storage_service.variant_study_service.command_factory.command_context, + ) + return command def get_storages( self, @@ -420,7 +425,7 @@ def delete_storages( file_study = self._get_file_study(study) execute_or_add_commands(study, file_study, [command], self.storage_service) - def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name: str) -> STStorageOutput: + def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_cluster_name: str) -> STStorageOutput: """ Creates a duplicate cluster within the study area with a new name. @@ -428,7 +433,7 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name study: The study in which the cluster will be duplicated. area_id: The identifier of the area where the cluster will be duplicated. source_id: The identifier of the cluster to be duplicated. - new_name: The new name for the duplicated cluster. + new_cluster_name: The new name for the duplicated cluster. Returns: The duplicated cluster configuration. @@ -436,22 +441,42 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_name Raises: ClusterAlreadyExists: If a cluster with the new name already exists in the area. """ - new_id = transform_name_to_id(new_name) - if any(new_id.lower() == storage.id.lower() for storage in self.get_storages(study, area_id)): + new_id = transform_name_to_id(new_cluster_name) + lower_new_id = new_id.lower() + if any(lower_new_id == storage.id.lower() for storage in self.get_storages(study, area_id)): raise ClusterAlreadyExists("Short-term storage", new_id) # Cluster duplication current_cluster = self.get_storage(study, area_id, source_id) - current_cluster.name = new_name + current_cluster.name = new_cluster_name creation_form = STStorageCreation(**current_cluster.dict(by_alias=False, exclude={"id"})) - new_storage = self.create_storage(study, area_id, creation_form) + new_config = creation_form.to_config(study.version) + create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config) # Matrix edition - for ts_name in STStorageTimeSeries.__args__: # type: ignore - ts = self.get_matrix(study, area_id, source_id, ts_name) - self.update_matrix(study, area_id, new_id.lower(), ts_name, ts) + lower_source_id = source_id.lower() + ts_names = ["pmax_injection", "pmax_withdrawal", "lower_rule_curve", "upper_rule_curve", "inflows"] + source_paths = [ + STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_source_id, ts_name=ts_name) + for ts_name in ts_names + ] + new_paths = [ + STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_new_id, ts_name=ts_name) + for ts_name in ts_names + ] + + # Prepare and execute commands + commands: t.List[t.Union[CreateSTStorage, ReplaceMatrix]] = [create_cluster_cmd] + storage_service = self.storage_service.get_storage(study) + command_context = self.storage_service.variant_study_service.command_factory.command_context + for source_path, new_path in zip(source_paths, new_paths): + current_matrix = storage_service.get(study, source_path)["data"] + command = ReplaceMatrix(target=new_path, matrix=current_matrix, command_context=command_context) + commands.append(command) - return new_storage + execute_or_add_commands(study, self._get_file_study(study), commands, self.storage_service) + + return STStorageOutput(**new_config.dict(by_alias=False)) def get_matrix( self, @@ -519,12 +544,11 @@ def _save_matrix_obj( ts_name: STStorageTimeSeries, matrix_obj: t.Dict[str, t.Any], ) -> None: - file_study = self._get_file_study(study) path = STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=storage_id, ts_name=ts_name) - try: - file_study.tree.save(matrix_obj, path.split("/")) - except KeyError: - raise STStorageMatrixNotFoundError(study.id, area_id, storage_id, ts_name) from None + matrix = matrix_obj["data"] + command_context = self.storage_service.variant_study_service.command_factory.command_context + command = ReplaceMatrix(target=path, matrix=matrix, command_context=command_context) + execute_or_add_commands(study, self._get_file_study(study), [command], self.storage_service) def validate_matrices( self, diff --git a/tests/integration/study_data_blueprint/test_st_storage.py b/tests/integration/study_data_blueprint/test_st_storage.py index 322c4669ae..5f2421d911 100644 --- a/tests/integration/study_data_blueprint/test_st_storage.py +++ b/tests/integration/study_data_blueprint/test_st_storage.py @@ -1,5 +1,6 @@ import json import re +import typing as t from unittest.mock import ANY import numpy as np @@ -683,3 +684,146 @@ def test__default_values( "initiallevel": 0.0, } assert actual == expected + + @pytest.fixture(name="base_study_id") + def base_study_id_fixture(self, request: t.Any, client: TestClient, user_access_token: str) -> str: + """Prepare a managed study for the variant study tests.""" + params = request.param + res = client.post( + "/v1/studies", + headers={"Authorization": f"Bearer {user_access_token}"}, + params=params, + ) + assert res.status_code in {200, 201}, res.json() + study_id: str = res.json() + return study_id + + @pytest.fixture(name="variant_id") + def variant_id_fixture(self, request: t.Any, client: TestClient, user_access_token: str, base_study_id: str) -> str: + """Prepare a variant study for the variant study tests.""" + name = request.param + res = client.post( + f"/v1/studies/{base_study_id}/variants", + headers={"Authorization": f"Bearer {user_access_token}"}, + params={"name": name}, + ) + assert res.status_code in {200, 201}, res.json() + study_id: str = res.json() + return study_id + + # noinspection PyTestParametrized + @pytest.mark.parametrize("base_study_id", [{"name": "Base Study", "version": 860}], indirect=True) + @pytest.mark.parametrize("variant_id", ["Variant Study"], indirect=True) + def test_variant_lifecycle(self, client: TestClient, user_access_token: str, variant_id: str) -> None: + """ + In this test, we want to check that short-term storages can be managed + in the context of a "variant" study. + """ + # Create an area + area_name = "France" + res = client.post( + f"/v1/studies/{variant_id}/areas", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={"name": area_name, "type": "AREA"}, + ) + assert res.status_code in {200, 201}, res.json() + area_cfg = res.json() + area_id = area_cfg["id"] + + # Create a short-term storage + cluster_name = "Tesla1" + res = client.post( + f"/v1/studies/{variant_id}/areas/{area_id}/storages", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={ + "name": cluster_name, + "group": "Battery", + "injectionNominalCapacity": 4500, + "withdrawalNominalCapacity": 4230, + "reservoirCapacity": 5700, + }, + ) + assert res.status_code in {200, 201}, res.json() + cluster_id: str = res.json()["id"] + + # Update the short-term storage + res = client.patch( + f"/v1/studies/{variant_id}/areas/{area_id}/storages/{cluster_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + json={"reservoirCapacity": 5600}, + ) + assert res.status_code == 200, res.json() + cluster_cfg = res.json() + assert cluster_cfg["reservoirCapacity"] == 5600 + + # Update the series matrix + matrix = np.random.randint(0, 2, size=(8760, 1)).tolist() + matrix_path = f"input/st-storage/series/{area_id}/{cluster_id.lower()}/pmax_injection" + args = {"target": matrix_path, "matrix": matrix} + res = client.post( + f"/v1/studies/{variant_id}/commands", + json=[{"action": "replace_matrix", "args": args}], + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code in {200, 201}, res.json() + + # Duplicate the short-term storage + new_name = "Tesla2" + res = client.post( + f"/v1/studies/{variant_id}/areas/{area_id}/storages/{cluster_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + params={"newName": new_name}, + ) + assert res.status_code in {200, 201}, res.json() + cluster_cfg = res.json() + assert cluster_cfg["name"] == new_name + new_id = cluster_cfg["id"] + + # Check that the duplicate has the right properties + res = client.get( + f"/v1/studies/{variant_id}/areas/{area_id}/storages/{new_id}", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200, res.json() + cluster_cfg = res.json() + assert cluster_cfg["group"] == "Battery" + assert cluster_cfg["injectionNominalCapacity"] == 4500 + assert cluster_cfg["withdrawalNominalCapacity"] == 4230 + assert cluster_cfg["reservoirCapacity"] == 5600 + + # Check that the duplicate has the right matrix + new_cluster_matrix_path = f"input/st-storage/series/{area_id}/{new_id.lower()}/pmax_injection" + res = client.get( + f"/v1/studies/{variant_id}/raw", + params={"path": new_cluster_matrix_path}, + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200 + assert res.json()["data"] == matrix + + # Delete the short-term storage + res = client.delete( + f"/v1/studies/{variant_id}/areas/{area_id}/storages", + headers={"Authorization": f"Bearer {user_access_token}"}, + json=[cluster_id], + ) + assert res.status_code == 204, res.json() + + # Check the list of variant commands + res = client.get( + f"/v1/studies/{variant_id}/commands", + headers={"Authorization": f"Bearer {user_access_token}"}, + ) + assert res.status_code == 200, res.json() + commands = res.json() + assert len(commands) == 7 + actions = [command["action"] for command in commands] + assert actions == [ + "create_area", + "create_st_storage", + "update_config", + "replace_matrix", + "create_st_storage", + "replace_matrix", + "remove_st_storage", + ]