From 6187ee6abee9a21d7f153380cfa8babd4cfa3770 Mon Sep 17 00:00:00 2001 From: salemsd Date: Tue, 17 Dec 2024 10:30:33 +0100 Subject: [PATCH 01/21] feat(api): add class and method skeletons (get_outputs) --- src/antares/craft/service/service_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/antares/craft/service/service_factory.py b/src/antares/craft/service/service_factory.py index 0bd71f58..494d4960 100644 --- a/src/antares/craft/service/service_factory.py +++ b/src/antares/craft/service/service_factory.py @@ -31,7 +31,7 @@ BaseRunService, BaseShortTermStorageService, BaseStudyService, - BaseThermalService, + BaseThermalService, BaseOutputService, ) from antares.craft.service.local_services.area_local import AreaLocalService from antares.craft.service.local_services.binding_constraint_local import BindingConstraintLocalService From ebc952e76544a2a3c491a14848a777916aaf8beb Mon Sep 17 00:00:00 2001 From: salemsd Date: Tue, 17 Dec 2024 10:33:23 +0100 Subject: [PATCH 02/21] feat(api): reformat and add read_outputs in study --- src/antares/craft/service/service_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/antares/craft/service/service_factory.py b/src/antares/craft/service/service_factory.py index 494d4960..0bd71f58 100644 --- a/src/antares/craft/service/service_factory.py +++ b/src/antares/craft/service/service_factory.py @@ -31,7 +31,7 @@ BaseRunService, BaseShortTermStorageService, BaseStudyService, - BaseThermalService, BaseOutputService, + BaseThermalService, ) from antares.craft.service.local_services.area_local import AreaLocalService from antares.craft.service.local_services.binding_constraint_local import BindingConstraintLocalService From 9be49ddf1cd5ad7ad6fddce6f9a699f98f5c5f49 Mon Sep 17 00:00:00 2001 From: salemsd Date: Tue, 17 Dec 2024 17:24:38 +0100 Subject: [PATCH 03/21] feat(api): fix single dict response in read_outputs and minor changes --- src/antares/craft/service/local_services/output_local.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 4c907692..b456ff1b 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -10,12 +10,14 @@ # # This file is part of the Antares project. from typing import Any +from pandas import DataFrame from antares.craft.config.local_configuration import LocalConfiguration from antares.craft.model.output import Output from antares.craft.service.base_services import BaseOutputService + class OutputLocalService(BaseOutputService): def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -24,3 +26,6 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - def read_outputs(self) -> list[Output]: raise NotImplementedError + + def get_matrix(self, path: str) -> DataFrame: + raise NotImplementedError From 6235c9d59751c3fb9eb36e30ecec6f8f5420eb7b Mon Sep 17 00:00:00 2001 From: salemsd Date: Tue, 17 Dec 2024 17:30:03 +0100 Subject: [PATCH 04/21] feat(api): remove get_matrix (wrong branch) --- src/antares/craft/service/local_services/output_local.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index b456ff1b..8c1cbc8a 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -26,6 +26,3 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - def read_outputs(self) -> list[Output]: raise NotImplementedError - - def get_matrix(self, path: str) -> DataFrame: - raise NotImplementedError From 2609a6422f998ca40d256448d89e4ba795539eff Mon Sep 17 00:00:00 2001 From: salemsd Date: Tue, 17 Dec 2024 17:30:57 +0100 Subject: [PATCH 05/21] feat(api): reformat --- src/antares/craft/service/local_services/output_local.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 8c1cbc8a..544e593d 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -16,8 +16,6 @@ from antares.craft.model.output import Output from antares.craft.service.base_services import BaseOutputService - - class OutputLocalService(BaseOutputService): def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) -> None: super().__init__(**kwargs) From 0657565471e1d713a20ebe6ce8410c31986f69b1 Mon Sep 17 00:00:00 2001 From: salemsd Date: Wed, 18 Dec 2024 10:48:40 +0100 Subject: [PATCH 06/21] feat(api): add get_matrix skeleton --- src/antares/craft/model/study.py | 11 +++++++++++ src/antares/craft/service/api_services/output_api.py | 6 +++++- src/antares/craft/service/base_services.py | 12 ++++++++++++ .../craft/service/local_services/output_local.py | 5 ++++- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/antares/craft/model/study.py b/src/antares/craft/model/study.py index 40d5bb66..653fcad4 100644 --- a/src/antares/craft/model/study.py +++ b/src/antares/craft/model/study.py @@ -402,6 +402,17 @@ def get_output(self, output_id: str) -> Output: """ return self._outputs[output_id] + def get_matrix(self, path: str) -> pd.DataFrame: + """ + Gets an output matrix from a path + + Args: + path: the path to the matrix + + Returns: Pandas DataFrame + """ + return self._output_service.get_matrix(path) + def _verify_study_already_exists(study_directory: Path) -> None: if study_directory.exists(): diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 64b4d4dc..015d1fd1 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. - +import pandas as pd from antares.craft.api_conf.api_conf import APIconf from antares.craft.api_conf.request_wrapper import RequestWrapper @@ -34,3 +34,7 @@ def read_outputs(self) -> list[Output]: return [Output(name=output["name"], archived=output["archived"]) for output in outputs_json_list] except APIError as e: raise OutputsRetrievalError(self.study_id, e.message) + + def get_matrix(self, path: str) -> pd.DataFrame: + # do something with the path + return get_matrix(self._base_url, self.study_id, self._wrapper, path) \ No newline at end of file diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index 99458c84..70e477db 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -638,3 +638,15 @@ def read_outputs(self) -> list[Output]: Returns: Output list """ pass + + @abstractmethod + def get_matrix(self, path: str) -> pd.DataFrame: + """ + Gets an output matrix from a path + + Args: + path: the path to the matrix + + Returns: Pandas DataFrame + """ + pass \ No newline at end of file diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 544e593d..2829e134 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -10,7 +10,7 @@ # # This file is part of the Antares project. from typing import Any -from pandas import DataFrame +import pandas as pd from antares.craft.config.local_configuration import LocalConfiguration from antares.craft.model.output import Output @@ -24,3 +24,6 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - def read_outputs(self) -> list[Output]: raise NotImplementedError + + def get_matrix(self, path: str) -> pd.DataFrame: + raise NotImplementedError \ No newline at end of file From fd420d69e155a33d95fc4240c565604cfebbf285 Mon Sep 17 00:00:00 2001 From: salemsd Date: Wed, 18 Dec 2024 13:29:09 +0100 Subject: [PATCH 07/21] feat(api): reformat --- src/antares/craft/service/api_services/output_api.py | 2 +- src/antares/craft/service/base_services.py | 2 +- src/antares/craft/service/local_services/output_local.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 015d1fd1..9bd264ff 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -37,4 +37,4 @@ def read_outputs(self) -> list[Output]: def get_matrix(self, path: str) -> pd.DataFrame: # do something with the path - return get_matrix(self._base_url, self.study_id, self._wrapper, path) \ No newline at end of file + return get_matrix(self._base_url, self.study_id, self._wrapper, path) diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index 70e477db..a075b0cd 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -649,4 +649,4 @@ def get_matrix(self, path: str) -> pd.DataFrame: Returns: Pandas DataFrame """ - pass \ No newline at end of file + pass diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 2829e134..3707df80 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -10,6 +10,7 @@ # # This file is part of the Antares project. from typing import Any + import pandas as pd from antares.craft.config.local_configuration import LocalConfiguration @@ -26,4 +27,4 @@ def read_outputs(self) -> list[Output]: raise NotImplementedError def get_matrix(self, path: str) -> pd.DataFrame: - raise NotImplementedError \ No newline at end of file + raise NotImplementedError From 732e6556e27c53d657520a7bb6c1a7cfe1b3fa3d Mon Sep 17 00:00:00 2001 From: salemsd Date: Wed, 18 Dec 2024 14:47:17 +0100 Subject: [PATCH 08/21] feat(api): refactor study and output --- src/antares/craft/model/output.py | 19 ++++++++++++++++ src/antares/craft/model/study.py | 15 ++----------- .../craft/service/api_services/output_api.py | 9 -------- .../craft/service/api_services/study_api.py | 12 ++++++++++ src/antares/craft/service/base_services.py | 22 +++++++++---------- .../service/local_services/output_local.py | 3 --- .../service/local_services/study_local.py | 3 +++ 7 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index d2b57eda..dfb5dfe3 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -9,9 +9,28 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. +from typing import Any + +import pandas as pd + from pydantic import BaseModel class Output(BaseModel): name: str archived: bool + + def __init__(self, output_service, **kwargs: Any): # type: ignore + super().__init__(**kwargs) + self._output_service = output_service + + def get_matrix(self, path: str) -> pd.DataFrame: + """ + Gets the matrix of the output + + Args: + path: output path + + Returns: Pandas DataFrame + """ + return self._output_service.get_matrix(path) diff --git a/src/antares/craft/model/study.py b/src/antares/craft/model/study.py index 653fcad4..8e48f363 100644 --- a/src/antares/craft/model/study.py +++ b/src/antares/craft/model/study.py @@ -219,8 +219,8 @@ def __init__( self._area_service = service_factory.create_area_service() self._link_service = service_factory.create_link_service() self._run_service = service_factory.create_run_service() - self._output_service = service_factory.create_output_service() self._binding_constraints_service = service_factory.create_binding_constraints_service() + self._output_service = service_factory.create_output_service() self._settings = DefaultStudySettings.model_validate(settings if settings is not None else StudySettings()) self._areas: Dict[str, Area] = dict() self._links: Dict[str, Link] = dict() @@ -377,7 +377,7 @@ def read_outputs(self) -> list[Output]: Returns: Output list """ - outputs = self._output_service.read_outputs() + outputs = self._study_service.read_outputs(self._output_service) self._outputs = {output.name: output for output in outputs} return outputs @@ -402,17 +402,6 @@ def get_output(self, output_id: str) -> Output: """ return self._outputs[output_id] - def get_matrix(self, path: str) -> pd.DataFrame: - """ - Gets an output matrix from a path - - Args: - path: the path to the matrix - - Returns: Pandas DataFrame - """ - return self._output_service.get_matrix(path) - def _verify_study_already_exists(study_directory: Path) -> None: if study_directory.exists(): diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 9bd264ff..8da27fa3 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -26,15 +26,6 @@ def __init__(self, config: APIconf, study_id: str): self._base_url = f"{self.config.get_host()}/api/v1" self._wrapper = RequestWrapper(self.config.set_up_api_conf()) - def read_outputs(self) -> list[Output]: - url = f"{self._base_url}/studies/{self.study_id}/outputs" - try: - response = self._wrapper.get(url) - outputs_json_list = response.json() - return [Output(name=output["name"], archived=output["archived"]) for output in outputs_json_list] - except APIError as e: - raise OutputsRetrievalError(self.study_id, e.message) - def get_matrix(self, path: str) -> pd.DataFrame: # do something with the path return get_matrix(self._base_url, self.study_id, self._wrapper, path) diff --git a/src/antares/craft/service/api_services/study_api.py b/src/antares/craft/service/api_services/study_api.py index 5c9dc929..582b6695 100644 --- a/src/antares/craft/service/api_services/study_api.py +++ b/src/antares/craft/service/api_services/study_api.py @@ -18,6 +18,7 @@ from antares.craft.exceptions.exceptions import ( APIError, BindingConstraintDeletionError, + OutputsRetrievalError, StudyDeletionError, StudySettingsUpdateError, StudyVariantCreationError, @@ -119,3 +120,14 @@ def create_variant(self, variant_name: str) -> "Study": return study.read_study_api(self.config, variant_id) except APIError as e: raise StudyVariantCreationError(self.study_id, e.message) from e + + def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + url = f"{self._base_url}/studies/{self.study_id}/outputs" + try: + response = self._wrapper.get(url) + outputs_json_list = response.json() + return [ + Output(output_service, name=output["name"], archived=output["archived"]) for output in outputs_json_list + ] + except APIError as e: + raise OutputsRetrievalError(self.study_id, e.message) diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index a075b0cd..4f12a713 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -557,6 +557,15 @@ def create_variant(self, variant_name: str) -> "Study": """ pass + @abstractmethod + def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + """ + Gets the output list of a study + + Returns: Output list + """ + pass + class BaseRenewableService(ABC): @abstractmethod @@ -630,22 +639,13 @@ def wait_job_completion(self, job: Job, time_out: int) -> None: class BaseOutputService(ABC): - @abstractmethod - def read_outputs(self) -> list[Output]: - """ - Gets the output list of a study - - Returns: Output list - """ - pass - @abstractmethod def get_matrix(self, path: str) -> pd.DataFrame: """ - Gets an output matrix from a path + Gets the matrix of the output Args: - path: the path to the matrix + path: output path Returns: Pandas DataFrame """ diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 3707df80..620a8e37 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -23,8 +23,5 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - self.config = config self.study_name = study_name - def read_outputs(self) -> list[Output]: - raise NotImplementedError - def get_matrix(self, path: str) -> pd.DataFrame: raise NotImplementedError diff --git a/src/antares/craft/service/local_services/study_local.py b/src/antares/craft/service/local_services/study_local.py index 94641a22..bf7d99b8 100644 --- a/src/antares/craft/service/local_services/study_local.py +++ b/src/antares/craft/service/local_services/study_local.py @@ -46,3 +46,6 @@ def delete(self, children: bool) -> None: def create_variant(self, variant_name: str) -> "Study": raise NotImplementedError + + def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + raise NotImplementedError From 4edcb409d4810089019ffb6447e3e4b76504b002 Mon Sep 17 00:00:00 2001 From: salemsd Date: Wed, 18 Dec 2024 15:30:23 +0100 Subject: [PATCH 09/21] feat(api): finish base get_matrix --- src/antares/craft/model/output.py | 5 +++-- src/antares/craft/service/api_services/output_api.py | 1 - src/antares/craft/service/api_services/utils.py | 6 +++++- tests/integration/test_web_client.py | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index dfb5dfe3..493b8b58 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -29,8 +29,9 @@ def get_matrix(self, path: str) -> pd.DataFrame: Gets the matrix of the output Args: - path: output path + path: output path, eg: "mc-all/areas/south/values-hourly" Returns: Pandas DataFrame """ - return self._output_service.get_matrix(path) + full_path = f"output/{self.name}/economy/{path}" + return self._output_service.get_matrix(full_path) diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 8da27fa3..2bd06a70 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -27,5 +27,4 @@ def __init__(self, config: APIconf, study_id: str): self._wrapper = RequestWrapper(self.config.set_up_api_conf()) def get_matrix(self, path: str) -> pd.DataFrame: - # do something with the path return get_matrix(self._base_url, self.study_id, self._wrapper, path) diff --git a/src/antares/craft/service/api_services/utils.py b/src/antares/craft/service/api_services/utils.py index d7ad96df..ddc8db0e 100644 --- a/src/antares/craft/service/api_services/utils.py +++ b/src/antares/craft/service/api_services/utils.py @@ -25,5 +25,9 @@ def get_matrix(base_url: str, study_id: str, wrapper: RequestWrapper, series_pat raw_url = f"{base_url}/studies/{study_id}/raw?path={series_path}" response = wrapper.get(raw_url) json_df = response.json() - dataframe = pd.DataFrame(data=json_df["data"], index=json_df["index"], columns=json_df["columns"]) + + if "index" in json_df: + dataframe = pd.DataFrame(data=json_df["data"], index=json_df["index"], columns=json_df["columns"]) + else: + dataframe = pd.DataFrame(data=json_df["data"], columns=json_df["columns"]) return dataframe diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index c0b3954b..4c79193b 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -526,3 +526,7 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert not outputs.get(output.name).archived study_with_outputs = read_study_api(api_config, study._study_service.study_id) assert study_with_outputs.get_outputs() == outputs + + # ===== Output get_matrix ===== + + output.get_matrix("mc-all/grid/links") From 060e5f560a8808f07e509a97a59fc8d537fb2ea5 Mon Sep 17 00:00:00 2001 From: salemsd Date: Thu, 19 Dec 2024 13:37:23 +0100 Subject: [PATCH 10/21] feat(api): finish aggregate_values, add some doc, temporary test --- src/antares/craft/exceptions/exceptions.py | 11 ++++ src/antares/craft/model/output.py | 66 +++++++++++++++++++ .../craft/service/api_services/output_api.py | 12 ++++ src/antares/craft/service/base_services.py | 16 +++++ .../service/local_services/output_local.py | 5 ++ tests/integration/test_web_client.py | 4 ++ 6 files changed, 114 insertions(+) diff --git a/src/antares/craft/exceptions/exceptions.py b/src/antares/craft/exceptions/exceptions.py index 1d11b599..3c85cfd0 100644 --- a/src/antares/craft/exceptions/exceptions.py +++ b/src/antares/craft/exceptions/exceptions.py @@ -12,6 +12,8 @@ from typing import List +from antares.model.output import McType, ObjectType + class InvalidChoiceError(ValueError): def __init__(self, message: str = "Invalid choice") -> None: @@ -328,3 +330,12 @@ class OutputsRetrievalError(Exception): def __init__(self, study_id: str, message: str) -> None: self.message = f"Could not get outputs for {study_id}: " + message super().__init__(self.message) + + +class AggregateCreationError(Exception): + def __init__(self, study_id: str, output_id: str, mc_type: McType, object_type: ObjectType, message: str) -> None: + self.message = ( + f"Could not create {mc_type.value}/{object_type.value} aggregate for study {study_id}, output {output_id}: " + + message + ) + super().__init__(self.message) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 493b8b58..2c969052 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -9,6 +9,7 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. +from enum import Enum from typing import Any import pandas as pd @@ -16,6 +17,56 @@ from pydantic import BaseModel +class QueryFile(Enum): + VALUES = "values" + DETAILS = "details" + DETAILS_STSTORAGE = "details-STstorage" + DETAILS_RES = "details-res" + + +class Frequency(Enum): + HOURLY = "hourly" + DAILY = "daily" + WEEKLY = "weekly" + MONTHLY = "monthly" + ANNUAL = "annual" + + +class McType(Enum): + ALL = "mc-all" + IND = "mc-ind" + + +class ObjectType(Enum): + LINKS = "links" + AREAS = "areas" + + +class AggregationEntry(BaseModel): + """ + Represents an entry for aggregation queries + + Attributes: + query_file: The file to query. + frequency: "hourly", "daily", "weekly", "monthly", "annual" + mc_years: Monte Carlo years to include in the query. If left empty, all years are included. + type_ids: which links/areas to be selected (ex: "be - fr"). If empty, all are selected (comma separated) + columns_names: names or regexes (if query_file is of type details) to select columns (comma separated) + """ + query_file: QueryFile + frequency: Frequency + mc_years: str = "" + type_ids: str = "" + columns_names: str = "" + + def to_query(self, object_type: ObjectType) -> str: + mc_years = f"&mc_years={self.mc_years}" if self.mc_years else "" + type_ids = f"&{object_type.value}_ids={self.type_ids}" if self.type_ids else "" + columns_names = f"&columns_names={self.columns_names}" if self.columns_names else "" + + return f"query_file={self.query_file.value}&frequency={self.frequency.value}{mc_years}{type_ids}{columns_names}&format=csv" + + class Output(BaseModel): name: str archived: bool @@ -35,3 +86,18 @@ def get_matrix(self, path: str) -> pd.DataFrame: """ full_path = f"output/{self.name}/economy/{path}" return self._output_service.get_matrix(full_path) + + def aggregate_values( + self, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data + + Args: + aggregate_input: input for the /aggregate endpoint + mc_type: all or ind (enum) + object_type: links or area (enum) + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ + return self._output_service.aggregate_values(self.name, aggregation_entry, mc_type, object_type) diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 2bd06a70..823448b7 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -9,6 +9,8 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. +from io import StringIO + import pandas as pd from antares.craft.api_conf.api_conf import APIconf @@ -28,3 +30,13 @@ def __init__(self, config: APIconf, study_id: str): def get_matrix(self, path: str) -> pd.DataFrame: return get_matrix(self._base_url, self.study_id, self._wrapper, path) + + def aggregate_values( + self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + url = f"{self._base_url}/studies/{self.study_id}/{object_type.value}/aggregate/{mc_type.value}/{output_id}?{aggregation_entry.to_query(object_type)}" + try: + response = self._wrapper.get(url) + return pd.read_csv(StringIO(response.text)) + except APIError as e: + raise AggregateCreationError(self.study_id, output_id, mc_type, object_type, e.message) diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index 4f12a713..7e08a454 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -650,3 +650,19 @@ def get_matrix(self, path: str) -> pd.DataFrame: Returns: Pandas DataFrame """ pass + + @abstractmethod + def aggregate_values( + self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data + + Args: + output_id: id of the output + aggregation_entry: input for the /aggregate endpoint + mc_type: all or ind (enum) + object_type: links or areas (enum) + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 620a8e37..ef9f241b 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -25,3 +25,8 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - def get_matrix(self, path: str) -> pd.DataFrame: raise NotImplementedError + + def aggregate_values( + self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + raise NotImplementedError diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index 4c79193b..520d3f8a 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -530,3 +530,7 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): # ===== Output get_matrix ===== output.get_matrix("mc-all/grid/links") + aggregation_entry = AggregationEntry( + query_file=QueryFile.VALUES, frequency=Frequency.DAILY, mc_years="", type_ids="", columns_names="" + ) + output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) From 61aa3d2cc7afb6e1ed7fcfd904e5140d9d810e7f Mon Sep 17 00:00:00 2001 From: salemsd Date: Thu, 19 Dec 2024 14:15:43 +0100 Subject: [PATCH 11/21] feat(api): add test skeleton --- .../antares/services/api_services/test_study_api.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index 1e646651..4be1c98a 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -472,3 +472,16 @@ def test_read_outputs(self): mocker.get(run_url, json={"description": error_message}, status_code=404) with pytest.raises(OutputsRetrievalError, match=error_message): self.study.read_outputs() + + def test_get_matrix(self): + with requests_mock.Mocker() as mocker: + run_url = f"https://antares.com/api/v1/studies/{self.study_id}/outputs" + + json_output = [ + { + "name": "20241217-1115eco-sdqsd", + "type": "economy", + "settings": {}, + "archived": False, + } + ] \ No newline at end of file From d44070b04214533a19db09d6c57136469048596a Mon Sep 17 00:00:00 2001 From: salemsd Date: Thu, 19 Dec 2024 15:08:10 +0100 Subject: [PATCH 12/21] feat(api): fix imports, add all tests --- src/antares/craft/exceptions/exceptions.py | 2 +- src/antares/craft/model/output.py | 1 + .../craft/service/api_services/output_api.py | 5 +- .../craft/service/api_services/study_api.py | 5 +- src/antares/craft/service/base_services.py | 2 +- .../service/local_services/output_local.py | 3 +- .../service/local_services/study_local.py | 5 +- .../services/api_services/test_study_api.py | 48 ++++++++++++++----- tests/integration/test_web_client.py | 17 +++++-- 9 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/antares/craft/exceptions/exceptions.py b/src/antares/craft/exceptions/exceptions.py index 3c85cfd0..1e7d7568 100644 --- a/src/antares/craft/exceptions/exceptions.py +++ b/src/antares/craft/exceptions/exceptions.py @@ -12,7 +12,7 @@ from typing import List -from antares.model.output import McType, ObjectType +from antares.craft.model.output import McType, ObjectType class InvalidChoiceError(ValueError): diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 2c969052..d48c6815 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -53,6 +53,7 @@ class AggregationEntry(BaseModel): type_ids: which links/areas to be selected (ex: "be - fr"). If empty, all are selected (comma separated) columns_names: names or regexes (if query_file is of type details) to select columns (comma separated) """ + query_file: QueryFile frequency: Frequency mc_years: str = "" diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 823448b7..7b4e8264 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -15,8 +15,9 @@ from antares.craft.api_conf.api_conf import APIconf from antares.craft.api_conf.request_wrapper import RequestWrapper -from antares.craft.exceptions.exceptions import APIError, OutputsRetrievalError -from antares.craft.model.output import Output +from antares.craft.exceptions.exceptions import AggregateCreationError, APIError +from antares.craft.model.output import AggregationEntry, McType, ObjectType +from antares.craft.service.api_services.utils import get_matrix from antares.craft.service.base_services import BaseOutputService diff --git a/src/antares/craft/service/api_services/study_api.py b/src/antares/craft/service/api_services/study_api.py index 582b6695..3f186878 100644 --- a/src/antares/craft/service/api_services/study_api.py +++ b/src/antares/craft/service/api_services/study_api.py @@ -24,6 +24,7 @@ StudyVariantCreationError, ) from antares.craft.model.binding_constraint import BindingConstraint +from antares.craft.model.output import Output from antares.craft.model.settings.adequacy_patch import AdequacyPatchParameters from antares.craft.model.settings.advanced_parameters import AdvancedParameters from antares.craft.model.settings.general import GeneralParameters @@ -32,7 +33,7 @@ from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.settings.thematic_trimming import ThematicTrimmingParameters from antares.craft.model.settings.time_series import TimeSeriesParameters -from antares.craft.service.base_services import BaseStudyService +from antares.craft.service.base_services import BaseOutputService, BaseStudyService if TYPE_CHECKING: from antares.craft.model.study import Study @@ -121,7 +122,7 @@ def create_variant(self, variant_name: str) -> "Study": except APIError as e: raise StudyVariantCreationError(self.study_id, e.message) from e - def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + def read_outputs(self, output_service: BaseOutputService) -> list[Output]: url = f"{self._base_url}/studies/{self.study_id}/outputs" try: response = self._wrapper.get(url) diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index 7e08a454..25365556 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -25,7 +25,7 @@ ) from antares.craft.model.hydro import Hydro, HydroMatrixName, HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi -from antares.craft.model.output import Output +from antares.craft.model.output import AggregationEntry, McType, ObjectType, Output from antares.craft.model.renewable import RenewableCluster, RenewableClusterProperties from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index ef9f241b..d87ec437 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -14,9 +14,10 @@ import pandas as pd from antares.craft.config.local_configuration import LocalConfiguration -from antares.craft.model.output import Output +from antares.craft.model.output import AggregationEntry, McType, ObjectType from antares.craft.service.base_services import BaseOutputService + class OutputLocalService(BaseOutputService): def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) -> None: super().__init__(**kwargs) diff --git a/src/antares/craft/service/local_services/study_local.py b/src/antares/craft/service/local_services/study_local.py index bf7d99b8..a11f69fd 100644 --- a/src/antares/craft/service/local_services/study_local.py +++ b/src/antares/craft/service/local_services/study_local.py @@ -14,8 +14,9 @@ from antares.craft.config.local_configuration import LocalConfiguration from antares.craft.model.binding_constraint import BindingConstraint +from antares.craft.model.output import Output from antares.craft.model.settings.study_settings import StudySettings -from antares.craft.service.base_services import BaseStudyService +from antares.craft.service.base_services import BaseOutputService, BaseStudyService if TYPE_CHECKING: from antares.craft.model.study import Study @@ -47,5 +48,5 @@ def delete(self, children: bool) -> None: def create_variant(self, variant_name: str) -> "Study": raise NotImplementedError - def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + def read_outputs(self, output_service: BaseOutputService) -> list[Output]: raise NotImplementedError diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index 4be1c98a..30c374af 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -9,7 +9,6 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. - import pytest import requests_mock @@ -18,6 +17,8 @@ from json import dumps from unittest.mock import Mock, patch +import pandas as pd + from antares.craft.api_conf.api_conf import APIconf from antares.craft.exceptions.exceptions import ( AreaCreationError, @@ -34,6 +35,7 @@ from antares.craft.model.binding_constraint import BindingConstraint, BindingConstraintProperties from antares.craft.model.hydro import HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi +from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, QueryFile from antares.craft.model.settings.general import GeneralParameters from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job, JobStatus, Solver @@ -473,15 +475,35 @@ def test_read_outputs(self): with pytest.raises(OutputsRetrievalError, match=error_message): self.study.read_outputs() - def test_get_matrix(self): - with requests_mock.Mocker() as mocker: - run_url = f"https://antares.com/api/v1/studies/{self.study_id}/outputs" - - json_output = [ - { - "name": "20241217-1115eco-sdqsd", - "type": "economy", - "settings": {}, - "archived": False, - } - ] \ No newline at end of file + # ==== get_matrix() ==== + + matrix_url = f"https://antares.com/api/v1/studies/{self.study_id}/raw?path=output/{output1.name}/economy/mc-all/grid/links" + matrix_output = {"columns": ["upstream", "downstream"], "data": [["be", "fr"]]} + mocker.get(matrix_url, json=matrix_output) + + matrix = output1.get_matrix("mc-all/grid/links") + assert isinstance(matrix, pd.DataFrame) + assert matrix.shape[1] == len(matrix_output["columns"]) + assert list(matrix.columns) == matrix_output["columns"] + assert matrix.shape[0] == len(matrix_output["data"]) + assert matrix.iloc[0].tolist() == matrix_output["data"][0] + + # ==== aggregate_values ==== + + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output1.name}?query_file=values&frequency=annual&format=csv" + aggregate_output = """ +link,timeId,FLOW LIN. EXP,FLOW LIN. STD +be - fr,1,0.000000,0.000000 +be - fr,2,0.000000,0.000000 + """ + mocker.get(aggregate_url, text=aggregate_output) + + aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.ANNUAL) + aggregated_matrix = output1.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + + assert isinstance(aggregated_matrix, pd.DataFrame) + assert aggregated_matrix.shape[1] == 4 + assert list(aggregated_matrix.columns) == ["link", "timeId", "FLOW LIN. EXP", "FLOW LIN. STD"] + assert aggregated_matrix.shape[0] == 2 + assert aggregated_matrix.iloc[0].tolist() == ["be - fr", 1, 0.0, 0.0] + assert aggregated_matrix.iloc[1].tolist() == ["be - fr", 2, 0.0, 0.0] diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index 520d3f8a..76ca2230 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -25,6 +25,7 @@ from antares.craft.model.area import AdequacyPatchMode, AreaProperties, AreaUi, FilterOption from antares.craft.model.binding_constraint import BindingConstraintProperties, ClusterData, ConstraintTerm, LinkData from antares.craft.model.link import LinkProperties, LinkStyle, LinkUi +from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, QueryFile from antares.craft.model.renewable import RenewableClusterGroup, RenewableClusterProperties, TimeSeriesInterpretation from antares.craft.model.settings.advanced_parameters import AdvancedParameters, UnitCommitmentMode from antares.craft.model.settings.general import GeneralParameters, Mode @@ -529,8 +530,14 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): # ===== Output get_matrix ===== - output.get_matrix("mc-all/grid/links") - aggregation_entry = AggregationEntry( - query_file=QueryFile.VALUES, frequency=Frequency.DAILY, mc_years="", type_ids="", columns_names="" - ) - output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + matrix = output.get_matrix("mc-all/grid/links") + + assert isinstance(matrix, pd.DataFrame) + assert not matrix.empty + + # ===== Output aggregate_values ===== + aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.DAILY) + aggregated_matrix = output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + + assert isinstance(aggregated_matrix, pd.DataFrame) + assert not aggregated_matrix.empty From 182b0a659513534c3c6a8cfee312d22c00480c35 Mon Sep 17 00:00:00 2001 From: salemsd Date: Fri, 20 Dec 2024 15:03:53 +0100 Subject: [PATCH 13/21] feat(api): refactor output, study and study_service, change tests --- src/antares/craft/exceptions/exceptions.py | 7 +-- src/antares/craft/model/output.py | 46 +++++++++++------- src/antares/craft/model/study.py | 3 +- .../craft/service/api_services/output_api.py | 4 +- .../craft/service/api_services/study_api.py | 13 ++++- src/antares/craft/service/base_services.py | 6 ++- .../service/local_services/study_local.py | 10 +++- src/antares/craft/service/service_factory.py | 5 +- .../services/api_services/test_study_api.py | 47 ++++++++++--------- tests/integration/test_web_client.py | 3 ++ 10 files changed, 91 insertions(+), 53 deletions(-) diff --git a/src/antares/craft/exceptions/exceptions.py b/src/antares/craft/exceptions/exceptions.py index 1e7d7568..fa06367c 100644 --- a/src/antares/craft/exceptions/exceptions.py +++ b/src/antares/craft/exceptions/exceptions.py @@ -12,8 +12,6 @@ from typing import List -from antares.craft.model.output import McType, ObjectType - class InvalidChoiceError(ValueError): def __init__(self, message: str = "Invalid choice") -> None: @@ -333,9 +331,8 @@ def __init__(self, study_id: str, message: str) -> None: class AggregateCreationError(Exception): - def __init__(self, study_id: str, output_id: str, mc_type: McType, object_type: ObjectType, message: str) -> None: + def __init__(self, study_id: str, output_id: str, mc_type: str, object_type: str, message: str) -> None: self.message = ( - f"Could not create {mc_type.value}/{object_type.value} aggregate for study {study_id}, output {output_id}: " - + message + f"Could not create {mc_type}/{object_type} aggregate for study {study_id}, output {output_id}: " + message ) super().__init__(self.message) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index d48c6815..108a3395 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -14,13 +14,14 @@ import pandas as pd -from pydantic import BaseModel +from pydantic import BaseModel, Field class QueryFile(Enum): VALUES = "values" + ID = "id" DETAILS = "details" - DETAILS_STSTORAGE = "details-STstorage" + DETAILS_ST_STORAGE = "details-STstorage" DETAILS_RES = "details-res" @@ -50,32 +51,43 @@ class AggregationEntry(BaseModel): query_file: The file to query. frequency: "hourly", "daily", "weekly", "monthly", "annual" mc_years: Monte Carlo years to include in the query. If left empty, all years are included. - type_ids: which links/areas to be selected (ex: "be - fr"). If empty, all are selected (comma separated) - columns_names: names or regexes (if query_file is of type details) to select columns (comma separated) + type_ids: which links/areas to be selected (ex: "be - fr"). If empty, all are selected + columns_names: names or regexes (if query_file is of type details) to select columns """ query_file: QueryFile frequency: Frequency - mc_years: str = "" - type_ids: str = "" - columns_names: str = "" + mc_years: list[str] = Field(default_factory=list) + type_ids: list[str] = Field(default_factory=list) + columns_names: list[str] = Field(default_factory=list) - def to_query(self, object_type: ObjectType) -> str: - mc_years = f"&mc_years={self.mc_years}" if self.mc_years else "" - type_ids = f"&{object_type.value}_ids={self.type_ids}" if self.type_ids else "" - columns_names = f"&columns_names={self.columns_names}" if self.columns_names else "" + def to_api_query(self, object_type: ObjectType) -> str: + mc_years = f"&mc_years={','.join(self.mc_years)}" if self.mc_years else "" + type_ids = f"&{object_type.value}_ids={','.join(self.type_ids)}" if self.type_ids else "" + columns_names = f"&columns_names={','.join(self.columns_names)}" if self.columns_names else "" return f"query_file={self.query_file.value}&frequency={self.frequency.value}{mc_years}{type_ids}{columns_names}&format=csv" -class Output(BaseModel): - name: str - archived: bool - - def __init__(self, output_service, **kwargs: Any): # type: ignore - super().__init__(**kwargs) +class Output: + def __init__(self, name: str, archived: bool, output_service): # type: ignore + self._name = name + self._archived = archived self._output_service = output_service + def __eq__(self, other: Any) -> bool: + if isinstance(other, Output): + return self._name == other._name and self._archived == other._archived + return False + + @property + def name(self) -> str: + return self._name + + @property + def archived(self) -> bool: + return self._archived + def get_matrix(self, path: str) -> pd.DataFrame: """ Gets the matrix of the output diff --git a/src/antares/craft/model/study.py b/src/antares/craft/model/study.py index 8e48f363..cfb36d53 100644 --- a/src/antares/craft/model/study.py +++ b/src/antares/craft/model/study.py @@ -220,7 +220,6 @@ def __init__( self._link_service = service_factory.create_link_service() self._run_service = service_factory.create_run_service() self._binding_constraints_service = service_factory.create_binding_constraints_service() - self._output_service = service_factory.create_output_service() self._settings = DefaultStudySettings.model_validate(settings if settings is not None else StudySettings()) self._areas: Dict[str, Area] = dict() self._links: Dict[str, Link] = dict() @@ -377,7 +376,7 @@ def read_outputs(self) -> list[Output]: Returns: Output list """ - outputs = self._study_service.read_outputs(self._output_service) + outputs = self._study_service.read_outputs() self._outputs = {output.name: output for output in outputs} return outputs diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 7b4e8264..674cd35a 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -35,9 +35,9 @@ def get_matrix(self, path: str) -> pd.DataFrame: def aggregate_values( self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType ) -> pd.DataFrame: - url = f"{self._base_url}/studies/{self.study_id}/{object_type.value}/aggregate/{mc_type.value}/{output_id}?{aggregation_entry.to_query(object_type)}" + url = f"{self._base_url}/studies/{self.study_id}/{object_type.value}/aggregate/{mc_type.value}/{output_id}?{aggregation_entry.to_api_query(object_type)}" try: response = self._wrapper.get(url) return pd.read_csv(StringIO(response.text)) except APIError as e: - raise AggregateCreationError(self.study_id, output_id, mc_type, object_type, e.message) + raise AggregateCreationError(self.study_id, output_id, mc_type.value, object_type.value, e.message) diff --git a/src/antares/craft/service/api_services/study_api.py b/src/antares/craft/service/api_services/study_api.py index 3f186878..033524ba 100644 --- a/src/antares/craft/service/api_services/study_api.py +++ b/src/antares/craft/service/api_services/study_api.py @@ -83,6 +83,7 @@ def __init__(self, config: APIconf, study_id: str): self._study_id = study_id self._base_url = f"{self.config.get_host()}/api/v1" self._wrapper = RequestWrapper(self.config.set_up_api_conf()) + self._output_service: Optional[BaseOutputService] = None @property def study_id(self) -> str: @@ -92,6 +93,13 @@ def study_id(self) -> str: def config(self) -> APIconf: return self._config + @property + def output_service(self) -> Optional[BaseOutputService]: + return self._output_service + + def set_output_service(self, output_service: BaseOutputService) -> None: + self._output_service = output_service + def update_study_settings(self, settings: StudySettings) -> Optional[StudySettings]: try: new_settings = _returns_study_settings(self._base_url, self.study_id, self._wrapper, True, settings) @@ -122,13 +130,14 @@ def create_variant(self, variant_name: str) -> "Study": except APIError as e: raise StudyVariantCreationError(self.study_id, e.message) from e - def read_outputs(self, output_service: BaseOutputService) -> list[Output]: + def read_outputs(self) -> list[Output]: url = f"{self._base_url}/studies/{self.study_id}/outputs" try: response = self._wrapper.get(url) outputs_json_list = response.json() return [ - Output(output_service, name=output["name"], archived=output["archived"]) for output in outputs_json_list + Output(output_service=self.output_service, name=output["name"], archived=output["archived"]) + for output in outputs_json_list ] except APIError as e: raise OutputsRetrievalError(self.study_id, e.message) diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index 25365556..a09a6fd1 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -558,7 +558,7 @@ def create_variant(self, variant_name: str) -> "Study": pass @abstractmethod - def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + def read_outputs(self) -> list[Output]: """ Gets the output list of a study @@ -566,6 +566,10 @@ def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: """ pass + @abstractmethod + def set_output_service(self, output_service: "BaseOutputService") -> None: + pass + class BaseRenewableService(ABC): @abstractmethod diff --git a/src/antares/craft/service/local_services/study_local.py b/src/antares/craft/service/local_services/study_local.py index a11f69fd..fd0266d8 100644 --- a/src/antares/craft/service/local_services/study_local.py +++ b/src/antares/craft/service/local_services/study_local.py @@ -27,6 +27,7 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - super().__init__(**kwargs) self._config = config self._study_name = study_name + self._output_service: Optional[BaseOutputService] = None @property def study_id(self) -> str: @@ -36,6 +37,13 @@ def study_id(self) -> str: def config(self) -> LocalConfiguration: return self._config + @property + def output_service(self) -> Optional[BaseOutputService]: + return self._output_service + + def set_output_service(self, output_service: BaseOutputService) -> None: + self._output_service = output_service + def update_study_settings(self, settings: StudySettings) -> Optional[StudySettings]: raise NotImplementedError @@ -48,5 +56,5 @@ def delete(self, children: bool) -> None: def create_variant(self, variant_name: str) -> "Study": raise NotImplementedError - def read_outputs(self, output_service: BaseOutputService) -> list[Output]: + def read_outputs(self) -> list[Output]: raise NotImplementedError diff --git a/src/antares/craft/service/service_factory.py b/src/antares/craft/service/service_factory.py index 0bd71f58..83c52e47 100644 --- a/src/antares/craft/service/service_factory.py +++ b/src/antares/craft/service/service_factory.py @@ -103,12 +103,15 @@ def create_binding_constraints_service(self) -> BaseBindingConstraintService: return binding_constraint_service def create_study_service(self) -> BaseStudyService: + study_service: BaseStudyService if isinstance(self.config, APIconf): - study_service: BaseStudyService = StudyApiService(self.config, self.study_id) + study_service = StudyApiService(self.config, self.study_id) elif isinstance(self.config, LocalConfiguration): study_service = StudyLocalService(self.config, self.study_name) else: raise TypeError(f"{ERROR_MESSAGE}{repr(self.config)}") + + study_service.set_output_service(self.create_output_service()) return study_service def create_renewable_service(self) -> BaseRenewableService: diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index 30c374af..f1c857b9 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -14,6 +14,7 @@ import re +from io import StringIO from json import dumps from unittest.mock import Mock, patch @@ -35,11 +36,12 @@ from antares.craft.model.binding_constraint import BindingConstraint, BindingConstraintProperties from antares.craft.model.hydro import HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi -from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, QueryFile +from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, Output, QueryFile from antares.craft.model.settings.general import GeneralParameters from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job, JobStatus, Solver from antares.craft.model.study import Study, create_study_api, create_variant_api, read_study_api +from antares.craft.service.api_services.output_api import OutputApiService from antares.craft.service.service_factory import ServiceFactory @@ -475,35 +477,36 @@ def test_read_outputs(self): with pytest.raises(OutputsRetrievalError, match=error_message): self.study.read_outputs() - # ==== get_matrix() ==== - - matrix_url = f"https://antares.com/api/v1/studies/{self.study_id}/raw?path=output/{output1.name}/economy/mc-all/grid/links" + def test_output_get_matrix(self): + with requests_mock.Mocker() as mocker: + output = Output( + name="test-output", output_service=OutputApiService(self.api, self.study_id), archived=False + ) + matrix_url = f"https://antares.com/api/v1/studies/{self.study_id}/raw?path=output/{output.name}/economy/mc-all/grid/links" matrix_output = {"columns": ["upstream", "downstream"], "data": [["be", "fr"]]} mocker.get(matrix_url, json=matrix_output) - matrix = output1.get_matrix("mc-all/grid/links") + matrix = output.get_matrix("mc-all/grid/links") + expected_matrix = pd.DataFrame(data=matrix_output["data"], columns=matrix_output["columns"]) assert isinstance(matrix, pd.DataFrame) - assert matrix.shape[1] == len(matrix_output["columns"]) - assert list(matrix.columns) == matrix_output["columns"] - assert matrix.shape[0] == len(matrix_output["data"]) - assert matrix.iloc[0].tolist() == matrix_output["data"][0] + assert matrix.equals(expected_matrix) - # ==== aggregate_values ==== - - aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output1.name}?query_file=values&frequency=annual&format=csv" + def test_output_aggregate_values(self): + with requests_mock.Mocker() as mocker: + output = Output( + name="test-output", output_service=OutputApiService(self.api, self.study_id), archived=False + ) + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output.name}?query_file=values&frequency=annual&format=csv" aggregate_output = """ -link,timeId,FLOW LIN. EXP,FLOW LIN. STD -be - fr,1,0.000000,0.000000 -be - fr,2,0.000000,0.000000 - """ + link,timeId,FLOW LIN. EXP,FLOW LIN. STD + be - fr,1,0.000000,0.000000 + be - fr,2,0.000000,0.000000 + """ mocker.get(aggregate_url, text=aggregate_output) aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.ANNUAL) - aggregated_matrix = output1.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + aggregated_matrix = output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + expected_matrix = pd.read_csv(StringIO(aggregate_output)) assert isinstance(aggregated_matrix, pd.DataFrame) - assert aggregated_matrix.shape[1] == 4 - assert list(aggregated_matrix.columns) == ["link", "timeId", "FLOW LIN. EXP", "FLOW LIN. STD"] - assert aggregated_matrix.shape[0] == 2 - assert aggregated_matrix.iloc[0].tolist() == ["be - fr", 1, 0.0, 0.0] - assert aggregated_matrix.iloc[1].tolist() == ["be - fr", 2, 0.0, 0.0] + assert aggregated_matrix.equals(expected_matrix) diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index 76ca2230..3bfc0a31 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -534,6 +534,8 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert isinstance(matrix, pd.DataFrame) assert not matrix.empty + assert "upstream" in matrix + assert "downstream" in matrix # ===== Output aggregate_values ===== aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.DAILY) @@ -541,3 +543,4 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert isinstance(aggregated_matrix, pd.DataFrame) assert not aggregated_matrix.empty + assert aggregated_matrix.shape == (364, 30) From 6c606e8d20ffbcad1483de0160998d2a37fc5a43 Mon Sep 17 00:00:00 2001 From: salemsd Date: Thu, 2 Jan 2025 14:45:03 +0100 Subject: [PATCH 14/21] feat(api): finish the 4 methods and adapt tests --- src/antares/craft/model/output.py | 84 +++++++++++++------ .../craft/service/api_services/output_api.py | 8 +- src/antares/craft/service/base_services.py | 4 +- .../service/local_services/output_local.py | 4 +- .../services/api_services/test_study_api.py | 45 ++++++++-- 5 files changed, 107 insertions(+), 38 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 108a3395..0d2ddf3f 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -10,19 +10,35 @@ # # This file is part of the Antares project. from enum import Enum -from typing import Any +from typing import Any, Union import pandas as pd from pydantic import BaseModel, Field -class QueryFile(Enum): +class MCIndAreas(Enum): + VALUES = "values" + DETAILS = "details" + DETAILS_ST_STORAGE = "details-STstorage" + DETAILS_RES = "details-res" + + +class MCAllAreas(Enum): VALUES = "values" - ID = "id" DETAILS = "details" DETAILS_ST_STORAGE = "details-STstorage" DETAILS_RES = "details-res" + ID = "id" + + +class MCIndLinks(Enum): + VALUES = "values" + + +class MCAllLinks(Enum): + VALUES = "values" + ID = "id" class Frequency(Enum): @@ -33,37 +49,26 @@ class Frequency(Enum): ANNUAL = "annual" -class McType(Enum): - ALL = "mc-all" - IND = "mc-ind" - - -class ObjectType(Enum): - LINKS = "links" - AREAS = "areas" - - class AggregationEntry(BaseModel): """ Represents an entry for aggregation queries Attributes: - query_file: The file to query. frequency: "hourly", "daily", "weekly", "monthly", "annual" mc_years: Monte Carlo years to include in the query. If left empty, all years are included. type_ids: which links/areas to be selected (ex: "be - fr"). If empty, all are selected columns_names: names or regexes (if query_file is of type details) to select columns """ - query_file: QueryFile + query_file: Union[MCAllAreas, MCIndAreas, MCAllLinks, MCIndLinks] frequency: Frequency mc_years: list[str] = Field(default_factory=list) type_ids: list[str] = Field(default_factory=list) columns_names: list[str] = Field(default_factory=list) - def to_api_query(self, object_type: ObjectType) -> str: + def to_api_query(self, object_type: str) -> str: mc_years = f"&mc_years={','.join(self.mc_years)}" if self.mc_years else "" - type_ids = f"&{object_type.value}_ids={','.join(self.type_ids)}" if self.type_ids else "" + type_ids = f"&{object_type}_ids={','.join(self.type_ids)}" if self.type_ids else "" columns_names = f"&columns_names={','.join(self.columns_names)}" if self.columns_names else "" return f"query_file={self.query_file.value}&frequency={self.frequency.value}{mc_years}{type_ids}{columns_names}&format=csv" @@ -100,17 +105,46 @@ def get_matrix(self, path: str) -> pd.DataFrame: full_path = f"output/{self.name}/economy/{path}" return self._output_service.get_matrix(full_path) - def aggregate_values( - self, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType - ) -> pd.DataFrame: + def aggregate_values_areas_mc_ind(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data for areas with mc-ind + + Args: + aggregation_entry: input for the /aggregate endpoint + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ + return self._output_service.aggregate_values(self.name, aggregation_entry, "areas", "ind") + + def aggregate_values_links_mc_ind(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data for links with mc-ind + + Args: + aggregation_entry: input for the /aggregate endpoint + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ + return self._output_service.aggregate_values(self.name, aggregation_entry, "links", "ind") + + def aggregate_values_areas_mc_all(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data for areas with mc-all + + Args: + aggregation_entry: input for the /aggregate endpoint + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ + return self._output_service.aggregate_values(self.name, aggregation_entry, "areas", "all") + + def aggregate_values_links_mc_all(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: """ - Creates a matrix of aggregated raw data + Creates a matrix of aggregated raw data for links with mc-all Args: - aggregate_input: input for the /aggregate endpoint - mc_type: all or ind (enum) - object_type: links or area (enum) + aggregation_entry: input for the /aggregate endpoint Returns: Pandas DataFrame corresponding to the aggregated raw data """ - return self._output_service.aggregate_values(self.name, aggregation_entry, mc_type, object_type) + return self._output_service.aggregate_values(self.name, aggregation_entry, "links", "all") diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 674cd35a..205dffbc 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -16,7 +16,7 @@ from antares.craft.api_conf.api_conf import APIconf from antares.craft.api_conf.request_wrapper import RequestWrapper from antares.craft.exceptions.exceptions import AggregateCreationError, APIError -from antares.craft.model.output import AggregationEntry, McType, ObjectType +from antares.craft.model.output import AggregationEntry from antares.craft.service.api_services.utils import get_matrix from antares.craft.service.base_services import BaseOutputService @@ -33,11 +33,11 @@ def get_matrix(self, path: str) -> pd.DataFrame: return get_matrix(self._base_url, self.study_id, self._wrapper, path) def aggregate_values( - self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + self, output_id: str, aggregation_entry: AggregationEntry, object_type: str, mc_type: str ) -> pd.DataFrame: - url = f"{self._base_url}/studies/{self.study_id}/{object_type.value}/aggregate/{mc_type.value}/{output_id}?{aggregation_entry.to_api_query(object_type)}" + url = f"{self._base_url}/studies/{self.study_id}/{object_type}/aggregate/{mc_type}/{output_id}?{aggregation_entry.to_api_query(object_type)}" try: response = self._wrapper.get(url) return pd.read_csv(StringIO(response.text)) except APIError as e: - raise AggregateCreationError(self.study_id, output_id, mc_type.value, object_type.value, e.message) + raise AggregateCreationError(self.study_id, output_id, mc_type, object_type, e.message) diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index a09a6fd1..d1fcb5fa 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -25,7 +25,7 @@ ) from antares.craft.model.hydro import Hydro, HydroMatrixName, HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi -from antares.craft.model.output import AggregationEntry, McType, ObjectType, Output +from antares.craft.model.output import AggregationEntry, Output from antares.craft.model.renewable import RenewableCluster, RenewableClusterProperties from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job @@ -657,7 +657,7 @@ def get_matrix(self, path: str) -> pd.DataFrame: @abstractmethod def aggregate_values( - self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + self, output_id: str, aggregation_entry: AggregationEntry, object_type: str, mc_type: str ) -> pd.DataFrame: """ Creates a matrix of aggregated raw data diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index d87ec437..e89e989b 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -14,7 +14,7 @@ import pandas as pd from antares.craft.config.local_configuration import LocalConfiguration -from antares.craft.model.output import AggregationEntry, McType, ObjectType +from antares.craft.model.output import AggregationEntry from antares.craft.service.base_services import BaseOutputService @@ -28,6 +28,6 @@ def get_matrix(self, path: str) -> pd.DataFrame: raise NotImplementedError def aggregate_values( - self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + self, output_id: str, aggregation_entry: AggregationEntry, object_type: str, mc_type: str ) -> pd.DataFrame: raise NotImplementedError diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index f1c857b9..80e0e8bb 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -36,7 +36,15 @@ from antares.craft.model.binding_constraint import BindingConstraint, BindingConstraintProperties from antares.craft.model.hydro import HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi -from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, Output, QueryFile +from antares.craft.model.output import ( + AggregationEntry, + Frequency, + MCAllAreas, + MCAllLinks, + MCIndAreas, + MCIndLinks, + Output, +) from antares.craft.model.settings.general import GeneralParameters from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job, JobStatus, Solver @@ -496,17 +504,44 @@ def test_output_aggregate_values(self): output = Output( name="test-output", output_service=OutputApiService(self.api, self.study_id), archived=False ) - aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output.name}?query_file=values&frequency=annual&format=csv" + + # aggregate_values_areas_mc_ind + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/ind/{output.name}?query_file=values&frequency=annual&format=csv" aggregate_output = """ link,timeId,FLOW LIN. EXP,FLOW LIN. STD be - fr,1,0.000000,0.000000 be - fr,2,0.000000,0.000000 - """ + """ mocker.get(aggregate_url, text=aggregate_output) + aggregation_entry = AggregationEntry(query_file=MCIndAreas.VALUES, frequency=Frequency.ANNUAL) + aggregated_matrix = output.aggregate_values_areas_mc_ind(aggregation_entry) + expected_matrix = pd.read_csv(StringIO(aggregate_output)) + assert isinstance(aggregated_matrix, pd.DataFrame) + assert aggregated_matrix.equals(expected_matrix) - aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.ANNUAL) - aggregated_matrix = output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + # aggregate_values_links_mc_ind + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/ind/{output.name}?query_file=values&frequency=annual&format=csv" + mocker.get(aggregate_url, text=aggregate_output) + aggregation_entry = AggregationEntry(query_file=MCIndLinks.VALUES, frequency=Frequency.ANNUAL) + aggregated_matrix = output.aggregate_values_links_mc_ind(aggregation_entry) expected_matrix = pd.read_csv(StringIO(aggregate_output)) + assert isinstance(aggregated_matrix, pd.DataFrame) + assert aggregated_matrix.equals(expected_matrix) + # aggregate_values_areas_mc_all + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/all/{output.name}?query_file=values&frequency=annual&format=csv" + mocker.get(aggregate_url, text=aggregate_output) + aggregation_entry = AggregationEntry(query_file=MCAllAreas.VALUES, frequency=Frequency.ANNUAL) + aggregated_matrix = output.aggregate_values_areas_mc_all(aggregation_entry) + expected_matrix = pd.read_csv(StringIO(aggregate_output)) + assert isinstance(aggregated_matrix, pd.DataFrame) + assert aggregated_matrix.equals(expected_matrix) + + # aggregate_values_links_mc_all + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/all/{output.name}?query_file=values&frequency=annual&format=csv" + mocker.get(aggregate_url, text=aggregate_output) + aggregation_entry = AggregationEntry(query_file=MCAllLinks.VALUES, frequency=Frequency.ANNUAL) + aggregated_matrix = output.aggregate_values_links_mc_all(aggregation_entry) + expected_matrix = pd.read_csv(StringIO(aggregate_output)) assert isinstance(aggregated_matrix, pd.DataFrame) assert aggregated_matrix.equals(expected_matrix) From 388d2eaccff72fb9bcd9fc6a409d637f1be46dc2 Mon Sep 17 00:00:00 2001 From: salemsd Date: Thu, 2 Jan 2025 15:12:38 +0100 Subject: [PATCH 15/21] feat(api): fix integration test --- src/antares/craft/service/api_services/output_api.py | 2 +- tests/integration/test_web_client.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 205dffbc..70671bf8 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -35,7 +35,7 @@ def get_matrix(self, path: str) -> pd.DataFrame: def aggregate_values( self, output_id: str, aggregation_entry: AggregationEntry, object_type: str, mc_type: str ) -> pd.DataFrame: - url = f"{self._base_url}/studies/{self.study_id}/{object_type}/aggregate/{mc_type}/{output_id}?{aggregation_entry.to_api_query(object_type)}" + url = f"{self._base_url}/studies/{self.study_id}/{object_type}/aggregate/mc-{mc_type}/{output_id}?{aggregation_entry.to_api_query(object_type)}" try: response = self._wrapper.get(url) return pd.read_csv(StringIO(response.text)) diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index 3bfc0a31..c7e9ec9b 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -25,7 +25,7 @@ from antares.craft.model.area import AdequacyPatchMode, AreaProperties, AreaUi, FilterOption from antares.craft.model.binding_constraint import BindingConstraintProperties, ClusterData, ConstraintTerm, LinkData from antares.craft.model.link import LinkProperties, LinkStyle, LinkUi -from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, QueryFile +from antares.craft.model.output import AggregationEntry, Frequency, MCAllLinks from antares.craft.model.renewable import RenewableClusterGroup, RenewableClusterProperties, TimeSeriesInterpretation from antares.craft.model.settings.advanced_parameters import AdvancedParameters, UnitCommitmentMode from antares.craft.model.settings.general import GeneralParameters, Mode @@ -538,9 +538,9 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert "downstream" in matrix # ===== Output aggregate_values ===== - aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.DAILY) - aggregated_matrix = output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + aggregation_entry = AggregationEntry(query_file=MCAllLinks.VALUES, frequency=Frequency.DAILY) + aggregated_matrix = output.aggregate_values_links_mc_all(aggregation_entry) assert isinstance(aggregated_matrix, pd.DataFrame) assert not aggregated_matrix.empty assert aggregated_matrix.shape == (364, 30) From e66c7fc7e612d300f00fd5819d65e06b33b71205 Mon Sep 17 00:00:00 2001 From: salemsd Date: Thu, 2 Jan 2025 15:15:56 +0100 Subject: [PATCH 16/21] feat(api): fix unit test --- tests/antares/services/api_services/test_study_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index 80e0e8bb..8b74fa1f 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -506,7 +506,7 @@ def test_output_aggregate_values(self): ) # aggregate_values_areas_mc_ind - aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/ind/{output.name}?query_file=values&frequency=annual&format=csv" + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/mc-ind/{output.name}?query_file=values&frequency=annual&format=csv" aggregate_output = """ link,timeId,FLOW LIN. EXP,FLOW LIN. STD be - fr,1,0.000000,0.000000 @@ -520,7 +520,7 @@ def test_output_aggregate_values(self): assert aggregated_matrix.equals(expected_matrix) # aggregate_values_links_mc_ind - aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/ind/{output.name}?query_file=values&frequency=annual&format=csv" + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-ind/{output.name}?query_file=values&frequency=annual&format=csv" mocker.get(aggregate_url, text=aggregate_output) aggregation_entry = AggregationEntry(query_file=MCIndLinks.VALUES, frequency=Frequency.ANNUAL) aggregated_matrix = output.aggregate_values_links_mc_ind(aggregation_entry) @@ -529,7 +529,7 @@ def test_output_aggregate_values(self): assert aggregated_matrix.equals(expected_matrix) # aggregate_values_areas_mc_all - aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/all/{output.name}?query_file=values&frequency=annual&format=csv" + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/mc-all/{output.name}?query_file=values&frequency=annual&format=csv" mocker.get(aggregate_url, text=aggregate_output) aggregation_entry = AggregationEntry(query_file=MCAllAreas.VALUES, frequency=Frequency.ANNUAL) aggregated_matrix = output.aggregate_values_areas_mc_all(aggregation_entry) @@ -538,7 +538,7 @@ def test_output_aggregate_values(self): assert aggregated_matrix.equals(expected_matrix) # aggregate_values_links_mc_all - aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/all/{output.name}?query_file=values&frequency=annual&format=csv" + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output.name}?query_file=values&frequency=annual&format=csv" mocker.get(aggregate_url, text=aggregate_output) aggregation_entry = AggregationEntry(query_file=MCAllLinks.VALUES, frequency=Frequency.ANNUAL) aggregated_matrix = output.aggregate_values_links_mc_all(aggregation_entry) From 73fa7176efa492505469745c587dd82513e38737 Mon Sep 17 00:00:00 2001 From: salemsd Date: Fri, 3 Jan 2025 17:02:49 +0100 Subject: [PATCH 17/21] feat(api): fix doc and remove eq --- src/antares/craft/model/output.py | 7 +------ src/antares/craft/service/base_services.py | 3 ++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 0d2ddf3f..424fbe27 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -10,7 +10,7 @@ # # This file is part of the Antares project. from enum import Enum -from typing import Any, Union +from typing import Union import pandas as pd @@ -80,11 +80,6 @@ def __init__(self, name: str, archived: bool, output_service): # type: ignore self._archived = archived self._output_service = output_service - def __eq__(self, other: Any) -> bool: - if isinstance(other, Output): - return self._name == other._name and self._archived == other._archived - return False - @property def name(self) -> str: return self._name diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index d1fcb5fa..e5faef36 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -664,9 +664,10 @@ def aggregate_values( Args: output_id: id of the output - aggregation_entry: input for the /aggregate endpoint + aggregation_entry: input (query_file, frequency, mc_years, ..) mc_type: all or ind (enum) object_type: links or areas (enum) Returns: Pandas DataFrame corresponding to the aggregated raw data """ + pass From 6374a3f533e4c542c50632b7c8fb070f95a6ced0 Mon Sep 17 00:00:00 2001 From: salemsd Date: Mon, 6 Jan 2025 13:14:50 +0100 Subject: [PATCH 18/21] feat(api): refactor aggregate methods, add some tests --- src/antares/craft/model/output.py | 100 +++++++++++++++--- .../craft/service/api_services/output_api.py | 5 +- src/antares/craft/service/base_services.py | 5 +- .../service/local_services/output_local.py | 2 +- .../services/api_services/test_study_api.py | 18 +--- tests/integration/test_web_client.py | 11 +- 6 files changed, 103 insertions(+), 38 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 424fbe27..4cba6087 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -10,11 +10,11 @@ # # This file is part of the Antares project. from enum import Enum -from typing import Union +from typing import Any, Optional, Union import pandas as pd -from pydantic import BaseModel, Field +from pydantic import BaseModel class MCIndAreas(Enum): @@ -62,9 +62,9 @@ class AggregationEntry(BaseModel): query_file: Union[MCAllAreas, MCIndAreas, MCAllLinks, MCIndLinks] frequency: Frequency - mc_years: list[str] = Field(default_factory=list) - type_ids: list[str] = Field(default_factory=list) - columns_names: list[str] = Field(default_factory=list) + mc_years: Optional[list[str]] = None + type_ids: Optional[list[str]] = None + columns_names: Optional[list[str]] = None def to_api_query(self, object_type: str) -> str: mc_years = f"&mc_years={','.join(self.mc_years)}" if self.mc_years else "" @@ -80,6 +80,11 @@ def __init__(self, name: str, archived: bool, output_service): # type: ignore self._archived = archived self._output_service = output_service + def __eq__(self, other: Any) -> bool: + if isinstance(other, Output): + return self._name == other._name and self._archived == other._archived + return False + @property def name(self) -> str: return self._name @@ -97,49 +102,112 @@ def get_matrix(self, path: str) -> pd.DataFrame: Returns: Pandas DataFrame """ - full_path = f"output/{self.name}/economy/{path}" - return self._output_service.get_matrix(full_path) - - def aggregate_values_areas_mc_ind(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + return self._output_service.get_matrix(self.name, path) + + def aggregate_areas_mc_ind( + self, + query_file: str, + frequency: str, + mc_years: Optional[list[str]] = None, + areas_ids: Optional[list[str]] = None, + columns_names: Optional[list[str]] = None, + ) -> pd.DataFrame: """ Creates a matrix of aggregated raw data for areas with mc-ind Args: - aggregation_entry: input for the /aggregate endpoint + query_file: values from McIndAreas + frequency: values from Frequency Returns: Pandas DataFrame corresponding to the aggregated raw data """ + aggregation_entry = AggregationEntry( + query_file=MCIndAreas(query_file), + frequency=Frequency(frequency), + mc_years=mc_years, + type_ids=areas_ids, + columns_names=columns_names, + ) + return self._output_service.aggregate_values(self.name, aggregation_entry, "areas", "ind") - def aggregate_values_links_mc_ind(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + def aggregate_links_mc_ind( + self, + query_file: str, + frequency: str, + mc_years: Optional[list[str]] = None, + areas_ids: Optional[list[str]] = None, + columns_names: Optional[list[str]] = None, + ) -> pd.DataFrame: """ Creates a matrix of aggregated raw data for links with mc-ind Args: - aggregation_entry: input for the /aggregate endpoint + query_file: values from McIndLinks + frequency: values from Frequency Returns: Pandas DataFrame corresponding to the aggregated raw data """ + aggregation_entry = AggregationEntry( + query_file=MCIndLinks(query_file), + frequency=Frequency(frequency), + mc_years=mc_years, + type_ids=areas_ids, + columns_names=columns_names, + ) + return self._output_service.aggregate_values(self.name, aggregation_entry, "links", "ind") - def aggregate_values_areas_mc_all(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + def aggregate_areas_mc_all( + self, + query_file: str, + frequency: str, + mc_years: Optional[list[str]] = None, + areas_ids: Optional[list[str]] = None, + columns_names: Optional[list[str]] = None, + ) -> pd.DataFrame: """ Creates a matrix of aggregated raw data for areas with mc-all Args: - aggregation_entry: input for the /aggregate endpoint + query_file: values from McAllAreas + frequency: values from Frequency Returns: Pandas DataFrame corresponding to the aggregated raw data """ + aggregation_entry = AggregationEntry( + query_file=MCAllAreas(query_file), + frequency=Frequency(frequency), + mc_years=mc_years, + type_ids=areas_ids, + columns_names=columns_names, + ) + return self._output_service.aggregate_values(self.name, aggregation_entry, "areas", "all") - def aggregate_values_links_mc_all(self, aggregation_entry: AggregationEntry) -> pd.DataFrame: + def aggregate_links_mc_all( + self, + query_file: str, + frequency: str, + mc_years: Optional[list[str]] = None, + areas_ids: Optional[list[str]] = None, + columns_names: Optional[list[str]] = None, + ) -> pd.DataFrame: """ Creates a matrix of aggregated raw data for links with mc-all Args: - aggregation_entry: input for the /aggregate endpoint + query_file: values from McAllLinks + frequency: values from Frequency Returns: Pandas DataFrame corresponding to the aggregated raw data """ + aggregation_entry = AggregationEntry( + query_file=MCAllLinks(query_file), + frequency=Frequency(frequency), + mc_years=mc_years, + type_ids=areas_ids, + columns_names=columns_names, + ) + return self._output_service.aggregate_values(self.name, aggregation_entry, "links", "all") diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 70671bf8..18a7e88d 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -29,8 +29,9 @@ def __init__(self, config: APIconf, study_id: str): self._base_url = f"{self.config.get_host()}/api/v1" self._wrapper = RequestWrapper(self.config.set_up_api_conf()) - def get_matrix(self, path: str) -> pd.DataFrame: - return get_matrix(self._base_url, self.study_id, self._wrapper, path) + def get_matrix(self, output_id: str, file_path: str) -> pd.DataFrame: + full_path = f"output/{output_id}/economy/{file_path}" + return get_matrix(self._base_url, self.study_id, self._wrapper, full_path) def aggregate_values( self, output_id: str, aggregation_entry: AggregationEntry, object_type: str, mc_type: str diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index e5faef36..6c78b651 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -644,12 +644,13 @@ def wait_job_completion(self, job: Job, time_out: int) -> None: class BaseOutputService(ABC): @abstractmethod - def get_matrix(self, path: str) -> pd.DataFrame: + def get_matrix(self, output_id: str, file_path: str) -> pd.DataFrame: """ Gets the matrix of the output Args: - path: output path + output_id: id of the output + file_path: output path Returns: Pandas DataFrame """ diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index e89e989b..70d541ea 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -24,7 +24,7 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - self.config = config self.study_name = study_name - def get_matrix(self, path: str) -> pd.DataFrame: + def get_matrix(self, output_id: str, file_path: str) -> pd.DataFrame: raise NotImplementedError def aggregate_values( diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index 8b74fa1f..f1f4241d 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -37,12 +37,6 @@ from antares.craft.model.hydro import HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi from antares.craft.model.output import ( - AggregationEntry, - Frequency, - MCAllAreas, - MCAllLinks, - MCIndAreas, - MCIndLinks, Output, ) from antares.craft.model.settings.general import GeneralParameters @@ -513,8 +507,7 @@ def test_output_aggregate_values(self): be - fr,2,0.000000,0.000000 """ mocker.get(aggregate_url, text=aggregate_output) - aggregation_entry = AggregationEntry(query_file=MCIndAreas.VALUES, frequency=Frequency.ANNUAL) - aggregated_matrix = output.aggregate_values_areas_mc_ind(aggregation_entry) + aggregated_matrix = output.aggregate_areas_mc_ind("values", "annual") expected_matrix = pd.read_csv(StringIO(aggregate_output)) assert isinstance(aggregated_matrix, pd.DataFrame) assert aggregated_matrix.equals(expected_matrix) @@ -522,8 +515,7 @@ def test_output_aggregate_values(self): # aggregate_values_links_mc_ind aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-ind/{output.name}?query_file=values&frequency=annual&format=csv" mocker.get(aggregate_url, text=aggregate_output) - aggregation_entry = AggregationEntry(query_file=MCIndLinks.VALUES, frequency=Frequency.ANNUAL) - aggregated_matrix = output.aggregate_values_links_mc_ind(aggregation_entry) + aggregated_matrix = output.aggregate_links_mc_ind("values", "annual", columns_names=["fr"]) expected_matrix = pd.read_csv(StringIO(aggregate_output)) assert isinstance(aggregated_matrix, pd.DataFrame) assert aggregated_matrix.equals(expected_matrix) @@ -531,8 +523,7 @@ def test_output_aggregate_values(self): # aggregate_values_areas_mc_all aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/areas/aggregate/mc-all/{output.name}?query_file=values&frequency=annual&format=csv" mocker.get(aggregate_url, text=aggregate_output) - aggregation_entry = AggregationEntry(query_file=MCAllAreas.VALUES, frequency=Frequency.ANNUAL) - aggregated_matrix = output.aggregate_values_areas_mc_all(aggregation_entry) + aggregated_matrix = output.aggregate_areas_mc_all("values", "annual") expected_matrix = pd.read_csv(StringIO(aggregate_output)) assert isinstance(aggregated_matrix, pd.DataFrame) assert aggregated_matrix.equals(expected_matrix) @@ -540,8 +531,7 @@ def test_output_aggregate_values(self): # aggregate_values_links_mc_all aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output.name}?query_file=values&frequency=annual&format=csv" mocker.get(aggregate_url, text=aggregate_output) - aggregation_entry = AggregationEntry(query_file=MCAllLinks.VALUES, frequency=Frequency.ANNUAL) - aggregated_matrix = output.aggregate_values_links_mc_all(aggregation_entry) + aggregated_matrix = output.aggregate_links_mc_all("values", "annual") expected_matrix = pd.read_csv(StringIO(aggregate_output)) assert isinstance(aggregated_matrix, pd.DataFrame) assert aggregated_matrix.equals(expected_matrix) diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index c7e9ec9b..f49e21a3 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -25,7 +25,6 @@ from antares.craft.model.area import AdequacyPatchMode, AreaProperties, AreaUi, FilterOption from antares.craft.model.binding_constraint import BindingConstraintProperties, ClusterData, ConstraintTerm, LinkData from antares.craft.model.link import LinkProperties, LinkStyle, LinkUi -from antares.craft.model.output import AggregationEntry, Frequency, MCAllLinks from antares.craft.model.renewable import RenewableClusterGroup, RenewableClusterProperties, TimeSeriesInterpretation from antares.craft.model.settings.advanced_parameters import AdvancedParameters, UnitCommitmentMode from antares.craft.model.settings.general import GeneralParameters, Mode @@ -534,13 +533,19 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert isinstance(matrix, pd.DataFrame) assert not matrix.empty + assert matrix.shape == (1, 2) assert "upstream" in matrix assert "downstream" in matrix + assert matrix.loc[0, "upstream"] == "be" + assert matrix.loc[0, "downstream"] == "fr" # ===== Output aggregate_values ===== - aggregation_entry = AggregationEntry(query_file=MCAllLinks.VALUES, frequency=Frequency.DAILY) - aggregated_matrix = output.aggregate_values_links_mc_all(aggregation_entry) + aggregated_matrix = output.aggregate_links_mc_all("values", "daily") assert isinstance(aggregated_matrix, pd.DataFrame) assert not aggregated_matrix.empty assert aggregated_matrix.shape == (364, 30) + assert aggregated_matrix["link"].apply(lambda x: x == "be - fr").all() + expected_values = list(range(1, 101)) + matrix_values = aggregated_matrix.loc[0:99, 'timeId'].tolist() + assert expected_values == matrix_values \ No newline at end of file From 6374e1875dd25a24bf22f465802039a6856036d5 Mon Sep 17 00:00:00 2001 From: salemsd Date: Mon, 6 Jan 2025 13:15:46 +0100 Subject: [PATCH 19/21] feat(api): reformat --- tests/integration/test_web_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index f49e21a3..10e4c6b2 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -547,5 +547,5 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert aggregated_matrix.shape == (364, 30) assert aggregated_matrix["link"].apply(lambda x: x == "be - fr").all() expected_values = list(range(1, 101)) - matrix_values = aggregated_matrix.loc[0:99, 'timeId'].tolist() - assert expected_values == matrix_values \ No newline at end of file + matrix_values = aggregated_matrix.loc[0:99, "timeId"].tolist() + assert expected_values == matrix_values From 62dac0bb618fc386f1edc4212b1ef9b1bfd47f5e Mon Sep 17 00:00:00 2001 From: salemsd Date: Mon, 6 Jan 2025 15:07:39 +0100 Subject: [PATCH 20/21] feat(api): remove __eq__ and fix tests --- src/antares/craft/model/output.py | 5 ----- tests/integration/test_web_client.py | 17 ++++++++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 4cba6087..9a2c93e5 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -80,11 +80,6 @@ def __init__(self, name: str, archived: bool, output_service): # type: ignore self._archived = archived self._output_service = output_service - def __eq__(self, other: Any) -> bool: - if isinstance(other, Output): - return self._name == other._name and self._archived == other._archived - return False - @property def name(self) -> str: return self._name diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index 10e4c6b2..816c79ed 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -13,6 +13,7 @@ import numpy as np import pandas as pd +from pandas import DataFrame from antares.craft.api_conf.api_conf import APIconf from antares.craft.exceptions.exceptions import ( @@ -525,19 +526,21 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert len(outputs) == 1 assert not outputs.get(output.name).archived study_with_outputs = read_study_api(api_config, study._study_service.study_id) - assert study_with_outputs.get_outputs() == outputs + outputs_from_api = study_with_outputs.get_outputs() + assert all( + outputs_from_api[output].name == outputs[output].name and outputs_from_api[output].archived == outputs[ + output].archived + for output in outputs_from_api + ) # ===== Output get_matrix ===== matrix = output.get_matrix("mc-all/grid/links") assert isinstance(matrix, pd.DataFrame) - assert not matrix.empty - assert matrix.shape == (1, 2) - assert "upstream" in matrix - assert "downstream" in matrix - assert matrix.loc[0, "upstream"] == "be" - assert matrix.loc[0, "downstream"] == "fr" + data = {'upstream': ['be'], 'downstream': ['fr']} + expected_matrix = pd.DataFrame(data) + assert matrix.equals(expected_matrix) # ===== Output aggregate_values ===== From fa2c2dce21627ce0420b0836e364b70cf8b4910a Mon Sep 17 00:00:00 2001 From: salemsd Date: Mon, 6 Jan 2025 15:08:11 +0100 Subject: [PATCH 21/21] feat(api): reformat --- src/antares/craft/model/output.py | 2 +- tests/integration/test_web_client.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index 9a2c93e5..2046daf8 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -10,7 +10,7 @@ # # This file is part of the Antares project. from enum import Enum -from typing import Any, Optional, Union +from typing import Optional, Union import pandas as pd diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index 816c79ed..93bd70da 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -13,7 +13,6 @@ import numpy as np import pandas as pd -from pandas import DataFrame from antares.craft.api_conf.api_conf import APIconf from antares.craft.exceptions.exceptions import ( @@ -528,8 +527,8 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): study_with_outputs = read_study_api(api_config, study._study_service.study_id) outputs_from_api = study_with_outputs.get_outputs() assert all( - outputs_from_api[output].name == outputs[output].name and outputs_from_api[output].archived == outputs[ - output].archived + outputs_from_api[output].name == outputs[output].name + and outputs_from_api[output].archived == outputs[output].archived for output in outputs_from_api ) @@ -538,7 +537,7 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): matrix = output.get_matrix("mc-all/grid/links") assert isinstance(matrix, pd.DataFrame) - data = {'upstream': ['be'], 'downstream': ['fr']} + data = {"upstream": ["be"], "downstream": ["fr"]} expected_matrix = pd.DataFrame(data) assert matrix.equals(expected_matrix)