diff --git a/src/antares/craft/exceptions/exceptions.py b/src/antares/craft/exceptions/exceptions.py index 1d11b59..1e7d756 100644 --- a/src/antares/craft/exceptions/exceptions.py +++ b/src/antares/craft/exceptions/exceptions.py @@ -12,6 +12,8 @@ from typing import List +from antares.craft.model.output import McType, ObjectType + class InvalidChoiceError(ValueError): def __init__(self, message: str = "Invalid choice") -> None: @@ -328,3 +330,12 @@ class OutputsRetrievalError(Exception): def __init__(self, study_id: str, message: str) -> None: self.message = f"Could not get outputs for {study_id}: " + message super().__init__(self.message) + + +class AggregateCreationError(Exception): + def __init__(self, study_id: str, output_id: str, mc_type: McType, object_type: ObjectType, message: str) -> None: + self.message = ( + f"Could not create {mc_type.value}/{object_type.value} aggregate for study {study_id}, output {output_id}: " + + message + ) + super().__init__(self.message) diff --git a/src/antares/craft/model/output.py b/src/antares/craft/model/output.py index d2b57ed..d48c681 100644 --- a/src/antares/craft/model/output.py +++ b/src/antares/craft/model/output.py @@ -9,9 +9,96 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. +from enum import Enum +from typing import Any + +import pandas as pd + from pydantic import BaseModel +class QueryFile(Enum): + VALUES = "values" + DETAILS = "details" + DETAILS_STSTORAGE = "details-STstorage" + DETAILS_RES = "details-res" + + +class Frequency(Enum): + HOURLY = "hourly" + DAILY = "daily" + WEEKLY = "weekly" + MONTHLY = "monthly" + ANNUAL = "annual" + + +class McType(Enum): + ALL = "mc-all" + IND = "mc-ind" + + +class ObjectType(Enum): + LINKS = "links" + AREAS = "areas" + + +class AggregationEntry(BaseModel): + """ + Represents an entry for aggregation queries + + Attributes: + query_file: The file to query. + frequency: "hourly", "daily", "weekly", "monthly", "annual" + mc_years: Monte Carlo years to include in the query. If left empty, all years are included. + type_ids: which links/areas to be selected (ex: "be - fr"). If empty, all are selected (comma separated) + columns_names: names or regexes (if query_file is of type details) to select columns (comma separated) + """ + + query_file: QueryFile + frequency: Frequency + mc_years: str = "" + type_ids: str = "" + columns_names: str = "" + + def to_query(self, object_type: ObjectType) -> str: + mc_years = f"&mc_years={self.mc_years}" if self.mc_years else "" + type_ids = f"&{object_type.value}_ids={self.type_ids}" if self.type_ids else "" + columns_names = f"&columns_names={self.columns_names}" if self.columns_names else "" + + return f"query_file={self.query_file.value}&frequency={self.frequency.value}{mc_years}{type_ids}{columns_names}&format=csv" + + class Output(BaseModel): name: str archived: bool + + def __init__(self, output_service, **kwargs: Any): # type: ignore + super().__init__(**kwargs) + self._output_service = output_service + + def get_matrix(self, path: str) -> pd.DataFrame: + """ + Gets the matrix of the output + + Args: + path: output path, eg: "mc-all/areas/south/values-hourly" + + Returns: Pandas DataFrame + """ + full_path = f"output/{self.name}/economy/{path}" + return self._output_service.get_matrix(full_path) + + def aggregate_values( + self, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data + + Args: + aggregate_input: input for the /aggregate endpoint + mc_type: all or ind (enum) + object_type: links or area (enum) + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ + return self._output_service.aggregate_values(self.name, aggregation_entry, mc_type, object_type) diff --git a/src/antares/craft/model/study.py b/src/antares/craft/model/study.py index 40d5bb6..8e48f36 100644 --- a/src/antares/craft/model/study.py +++ b/src/antares/craft/model/study.py @@ -219,8 +219,8 @@ def __init__( self._area_service = service_factory.create_area_service() self._link_service = service_factory.create_link_service() self._run_service = service_factory.create_run_service() - self._output_service = service_factory.create_output_service() self._binding_constraints_service = service_factory.create_binding_constraints_service() + self._output_service = service_factory.create_output_service() self._settings = DefaultStudySettings.model_validate(settings if settings is not None else StudySettings()) self._areas: Dict[str, Area] = dict() self._links: Dict[str, Link] = dict() @@ -377,7 +377,7 @@ def read_outputs(self) -> list[Output]: Returns: Output list """ - outputs = self._output_service.read_outputs() + outputs = self._study_service.read_outputs(self._output_service) self._outputs = {output.name: output for output in outputs} return outputs diff --git a/src/antares/craft/service/api_services/output_api.py b/src/antares/craft/service/api_services/output_api.py index 64b4d4d..7b4e826 100644 --- a/src/antares/craft/service/api_services/output_api.py +++ b/src/antares/craft/service/api_services/output_api.py @@ -9,12 +9,15 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. +from io import StringIO +import pandas as pd from antares.craft.api_conf.api_conf import APIconf from antares.craft.api_conf.request_wrapper import RequestWrapper -from antares.craft.exceptions.exceptions import APIError, OutputsRetrievalError -from antares.craft.model.output import Output +from antares.craft.exceptions.exceptions import AggregateCreationError, APIError +from antares.craft.model.output import AggregationEntry, McType, ObjectType +from antares.craft.service.api_services.utils import get_matrix from antares.craft.service.base_services import BaseOutputService @@ -26,11 +29,15 @@ def __init__(self, config: APIconf, study_id: str): self._base_url = f"{self.config.get_host()}/api/v1" self._wrapper = RequestWrapper(self.config.set_up_api_conf()) - def read_outputs(self) -> list[Output]: - url = f"{self._base_url}/studies/{self.study_id}/outputs" + def get_matrix(self, path: str) -> pd.DataFrame: + return get_matrix(self._base_url, self.study_id, self._wrapper, path) + + def aggregate_values( + self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + url = f"{self._base_url}/studies/{self.study_id}/{object_type.value}/aggregate/{mc_type.value}/{output_id}?{aggregation_entry.to_query(object_type)}" try: response = self._wrapper.get(url) - outputs_json_list = response.json() - return [Output(name=output["name"], archived=output["archived"]) for output in outputs_json_list] + return pd.read_csv(StringIO(response.text)) except APIError as e: - raise OutputsRetrievalError(self.study_id, e.message) + raise AggregateCreationError(self.study_id, output_id, mc_type, object_type, e.message) diff --git a/src/antares/craft/service/api_services/study_api.py b/src/antares/craft/service/api_services/study_api.py index 5c9dc92..3f18687 100644 --- a/src/antares/craft/service/api_services/study_api.py +++ b/src/antares/craft/service/api_services/study_api.py @@ -18,11 +18,13 @@ from antares.craft.exceptions.exceptions import ( APIError, BindingConstraintDeletionError, + OutputsRetrievalError, StudyDeletionError, StudySettingsUpdateError, StudyVariantCreationError, ) from antares.craft.model.binding_constraint import BindingConstraint +from antares.craft.model.output import Output from antares.craft.model.settings.adequacy_patch import AdequacyPatchParameters from antares.craft.model.settings.advanced_parameters import AdvancedParameters from antares.craft.model.settings.general import GeneralParameters @@ -31,7 +33,7 @@ from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.settings.thematic_trimming import ThematicTrimmingParameters from antares.craft.model.settings.time_series import TimeSeriesParameters -from antares.craft.service.base_services import BaseStudyService +from antares.craft.service.base_services import BaseOutputService, BaseStudyService if TYPE_CHECKING: from antares.craft.model.study import Study @@ -119,3 +121,14 @@ def create_variant(self, variant_name: str) -> "Study": return study.read_study_api(self.config, variant_id) except APIError as e: raise StudyVariantCreationError(self.study_id, e.message) from e + + def read_outputs(self, output_service: BaseOutputService) -> list[Output]: + url = f"{self._base_url}/studies/{self.study_id}/outputs" + try: + response = self._wrapper.get(url) + outputs_json_list = response.json() + return [ + Output(output_service, name=output["name"], archived=output["archived"]) for output in outputs_json_list + ] + except APIError as e: + raise OutputsRetrievalError(self.study_id, e.message) diff --git a/src/antares/craft/service/api_services/utils.py b/src/antares/craft/service/api_services/utils.py index d7ad96d..ddc8db0 100644 --- a/src/antares/craft/service/api_services/utils.py +++ b/src/antares/craft/service/api_services/utils.py @@ -25,5 +25,9 @@ def get_matrix(base_url: str, study_id: str, wrapper: RequestWrapper, series_pat raw_url = f"{base_url}/studies/{study_id}/raw?path={series_path}" response = wrapper.get(raw_url) json_df = response.json() - dataframe = pd.DataFrame(data=json_df["data"], index=json_df["index"], columns=json_df["columns"]) + + if "index" in json_df: + dataframe = pd.DataFrame(data=json_df["data"], index=json_df["index"], columns=json_df["columns"]) + else: + dataframe = pd.DataFrame(data=json_df["data"], columns=json_df["columns"]) return dataframe diff --git a/src/antares/craft/service/base_services.py b/src/antares/craft/service/base_services.py index 99458c8..2536555 100644 --- a/src/antares/craft/service/base_services.py +++ b/src/antares/craft/service/base_services.py @@ -25,7 +25,7 @@ ) from antares.craft.model.hydro import Hydro, HydroMatrixName, HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi -from antares.craft.model.output import Output +from antares.craft.model.output import AggregationEntry, McType, ObjectType, Output from antares.craft.model.renewable import RenewableCluster, RenewableClusterProperties from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job @@ -557,6 +557,15 @@ def create_variant(self, variant_name: str) -> "Study": """ pass + @abstractmethod + def read_outputs(self, output_service: "BaseOutputService") -> list[Output]: + """ + Gets the output list of a study + + Returns: Output list + """ + pass + class BaseRenewableService(ABC): @abstractmethod @@ -631,10 +640,29 @@ def wait_job_completion(self, job: Job, time_out: int) -> None: class BaseOutputService(ABC): @abstractmethod - def read_outputs(self) -> list[Output]: + def get_matrix(self, path: str) -> pd.DataFrame: """ - Gets the output list of a study + Gets the matrix of the output - Returns: Output list + Args: + path: output path + + Returns: Pandas DataFrame """ pass + + @abstractmethod + def aggregate_values( + self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: + """ + Creates a matrix of aggregated raw data + + Args: + output_id: id of the output + aggregation_entry: input for the /aggregate endpoint + mc_type: all or ind (enum) + object_type: links or areas (enum) + + Returns: Pandas DataFrame corresponding to the aggregated raw data + """ diff --git a/src/antares/craft/service/local_services/output_local.py b/src/antares/craft/service/local_services/output_local.py index 4c90769..d87ec43 100644 --- a/src/antares/craft/service/local_services/output_local.py +++ b/src/antares/craft/service/local_services/output_local.py @@ -11,8 +11,10 @@ # This file is part of the Antares project. from typing import Any +import pandas as pd + from antares.craft.config.local_configuration import LocalConfiguration -from antares.craft.model.output import Output +from antares.craft.model.output import AggregationEntry, McType, ObjectType from antares.craft.service.base_services import BaseOutputService @@ -22,5 +24,10 @@ def __init__(self, config: LocalConfiguration, study_name: str, **kwargs: Any) - self.config = config self.study_name = study_name - def read_outputs(self) -> list[Output]: + def get_matrix(self, path: str) -> pd.DataFrame: + raise NotImplementedError + + def aggregate_values( + self, output_id: str, aggregation_entry: AggregationEntry, mc_type: McType, object_type: ObjectType + ) -> pd.DataFrame: raise NotImplementedError diff --git a/src/antares/craft/service/local_services/study_local.py b/src/antares/craft/service/local_services/study_local.py index 94641a2..a11f69f 100644 --- a/src/antares/craft/service/local_services/study_local.py +++ b/src/antares/craft/service/local_services/study_local.py @@ -14,8 +14,9 @@ from antares.craft.config.local_configuration import LocalConfiguration from antares.craft.model.binding_constraint import BindingConstraint +from antares.craft.model.output import Output from antares.craft.model.settings.study_settings import StudySettings -from antares.craft.service.base_services import BaseStudyService +from antares.craft.service.base_services import BaseOutputService, BaseStudyService if TYPE_CHECKING: from antares.craft.model.study import Study @@ -46,3 +47,6 @@ def delete(self, children: bool) -> None: def create_variant(self, variant_name: str) -> "Study": raise NotImplementedError + + def read_outputs(self, output_service: BaseOutputService) -> list[Output]: + raise NotImplementedError diff --git a/tests/antares/services/api_services/test_study_api.py b/tests/antares/services/api_services/test_study_api.py index 1e64665..30c374a 100644 --- a/tests/antares/services/api_services/test_study_api.py +++ b/tests/antares/services/api_services/test_study_api.py @@ -9,7 +9,6 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. - import pytest import requests_mock @@ -18,6 +17,8 @@ from json import dumps from unittest.mock import Mock, patch +import pandas as pd + from antares.craft.api_conf.api_conf import APIconf from antares.craft.exceptions.exceptions import ( AreaCreationError, @@ -34,6 +35,7 @@ from antares.craft.model.binding_constraint import BindingConstraint, BindingConstraintProperties from antares.craft.model.hydro import HydroProperties from antares.craft.model.link import Link, LinkProperties, LinkUi +from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, QueryFile from antares.craft.model.settings.general import GeneralParameters from antares.craft.model.settings.study_settings import StudySettings from antares.craft.model.simulation import AntaresSimulationParameters, Job, JobStatus, Solver @@ -472,3 +474,36 @@ def test_read_outputs(self): mocker.get(run_url, json={"description": error_message}, status_code=404) with pytest.raises(OutputsRetrievalError, match=error_message): self.study.read_outputs() + + # ==== get_matrix() ==== + + matrix_url = f"https://antares.com/api/v1/studies/{self.study_id}/raw?path=output/{output1.name}/economy/mc-all/grid/links" + matrix_output = {"columns": ["upstream", "downstream"], "data": [["be", "fr"]]} + mocker.get(matrix_url, json=matrix_output) + + matrix = output1.get_matrix("mc-all/grid/links") + assert isinstance(matrix, pd.DataFrame) + assert matrix.shape[1] == len(matrix_output["columns"]) + assert list(matrix.columns) == matrix_output["columns"] + assert matrix.shape[0] == len(matrix_output["data"]) + assert matrix.iloc[0].tolist() == matrix_output["data"][0] + + # ==== aggregate_values ==== + + aggregate_url = f"https://antares.com/api/v1/studies/{self.study_id}/links/aggregate/mc-all/{output1.name}?query_file=values&frequency=annual&format=csv" + aggregate_output = """ +link,timeId,FLOW LIN. EXP,FLOW LIN. STD +be - fr,1,0.000000,0.000000 +be - fr,2,0.000000,0.000000 + """ + mocker.get(aggregate_url, text=aggregate_output) + + aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.ANNUAL) + aggregated_matrix = output1.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + + assert isinstance(aggregated_matrix, pd.DataFrame) + assert aggregated_matrix.shape[1] == 4 + assert list(aggregated_matrix.columns) == ["link", "timeId", "FLOW LIN. EXP", "FLOW LIN. STD"] + assert aggregated_matrix.shape[0] == 2 + assert aggregated_matrix.iloc[0].tolist() == ["be - fr", 1, 0.0, 0.0] + assert aggregated_matrix.iloc[1].tolist() == ["be - fr", 2, 0.0, 0.0] diff --git a/tests/integration/test_web_client.py b/tests/integration/test_web_client.py index c0b3954..76ca223 100644 --- a/tests/integration/test_web_client.py +++ b/tests/integration/test_web_client.py @@ -25,6 +25,7 @@ from antares.craft.model.area import AdequacyPatchMode, AreaProperties, AreaUi, FilterOption from antares.craft.model.binding_constraint import BindingConstraintProperties, ClusterData, ConstraintTerm, LinkData from antares.craft.model.link import LinkProperties, LinkStyle, LinkUi +from antares.craft.model.output import AggregationEntry, Frequency, McType, ObjectType, QueryFile from antares.craft.model.renewable import RenewableClusterGroup, RenewableClusterProperties, TimeSeriesInterpretation from antares.craft.model.settings.advanced_parameters import AdvancedParameters, UnitCommitmentMode from antares.craft.model.settings.general import GeneralParameters, Mode @@ -526,3 +527,17 @@ def test_creation_lifecycle(self, antares_web: AntaresWebDesktop): assert not outputs.get(output.name).archived study_with_outputs = read_study_api(api_config, study._study_service.study_id) assert study_with_outputs.get_outputs() == outputs + + # ===== Output get_matrix ===== + + matrix = output.get_matrix("mc-all/grid/links") + + assert isinstance(matrix, pd.DataFrame) + assert not matrix.empty + + # ===== Output aggregate_values ===== + aggregation_entry = AggregationEntry(query_file=QueryFile.VALUES, frequency=Frequency.DAILY) + aggregated_matrix = output.aggregate_values(aggregation_entry, McType.ALL, ObjectType.LINKS) + + assert isinstance(aggregated_matrix, pd.DataFrame) + assert not aggregated_matrix.empty