From 145d0fcfc01f5ab00c177c9f16862e698d771e86 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:38:59 +0100 Subject: [PATCH 1/7] chore: bump version to 2.18.1 --- antarest/__init__.py | 4 ++-- docs/CHANGELOG.md | 6 +++++- pyproject.toml | 2 +- sonar-project.properties | 2 +- webapp/package-lock.json | 4 ++-- webapp/package.json | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index f8b3bd0418..ed494cf4c5 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -19,9 +19,9 @@ # Standard project metadata -__version__ = "2.18.0" +__version__ = "2.18.1" __author__ = "RTE, Antares Web Team" -__date__ = "2024-11-29" +__date__ = "2024-12-02" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 00c33a8f5a..7316cc8bc3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,8 +1,12 @@ Antares Web Changelog ===================== +v2.18.1 (2024-12-02) +-------------------- + + v2.18.0 (2024-11-29) -------------------- +-------------------- ## What's Changed diff --git a/pyproject.toml b/pyproject.toml index 5e77bb439e..00613c0cc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] [project] name = "AntaREST" -version = "2.18.0" +version = "2.18.1" authors = [{name="RTE, Antares Web Team", email="andrea.sgattoni@rte-france.com" }] description="Antares Server" readme = {file = "README.md", content-type = "text/markdown"} diff --git a/sonar-project.properties b/sonar-project.properties index c0897c1d14..92b73ee890 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.11 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info -sonar.projectVersion=2.18.0 +sonar.projectVersion=2.18.1 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/*,,antarest/fastapi_jwt_auth/** \ No newline at end of file diff --git a/webapp/package-lock.json b/webapp/package-lock.json index a8fd1acc16..3ed8e546ee 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "antares-web", - "version": "2.18.0", + "version": "2.18.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "antares-web", - "version": "2.18.0", + "version": "2.18.1", "dependencies": { "@emotion/react": "11.13.3", "@emotion/styled": "11.13.0", diff --git a/webapp/package.json b/webapp/package.json index a5b9d9dca6..1abf973a65 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.18.0", + "version": "2.18.1", "private": true, "type": "module", "scripts": { From 02ec345a9ba0914aeab8618f5fb8596f06dfa183 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:49:37 +0100 Subject: [PATCH 2/7] fix(ui-tablemode): style missing (#2257) --- webapp/src/components/common/Handsontable.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/src/components/common/Handsontable.tsx b/webapp/src/components/common/Handsontable.tsx index 0e6d6c3f8a..89f3ef7c0f 100644 --- a/webapp/src/components/common/Handsontable.tsx +++ b/webapp/src/components/common/Handsontable.tsx @@ -18,6 +18,7 @@ import { styled } from "@mui/material"; import { forwardRef } from "react"; import * as RA from "ramda-adjunct"; import { SECONDARY_MAIN_COLOR } from "../../theme"; +import "handsontable/dist/handsontable.min.css"; // Register Handsontable's modules registerAllModules(); From cd196126fb3db198cbf7ddc20bd2335e38464c06 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:55:54 +0100 Subject: [PATCH 3/7] fix(ui-studies): multiple API calls on study list view (#2258) --- .../App/Studies/StudyCard/index.tsx | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/webapp/src/components/App/Studies/StudyCard/index.tsx b/webapp/src/components/App/Studies/StudyCard/index.tsx index cb74b58819..89813e6271 100644 --- a/webapp/src/components/App/Studies/StudyCard/index.tsx +++ b/webapp/src/components/App/Studies/StudyCard/index.tsx @@ -405,30 +405,27 @@ const StudyCard = memo((props: Props) => { setOpenDialog={setOpenDialog} /> - - - {t("studies.question.delete")} - - - + {/* Keep conditional rendering for dialogs and not use only `open` property, because API calls are made on mount */} + {openDialog === "properties" && ( + + )} + {openDialog === "delete" && ( + + {t("studies.question.delete")} + + )} + {openDialog === "export" && ( + + )} + {openDialog === "move" && ( + + )} ); }, areEqual); From 316c6eb43dcf4cd338b3167a326c9e413b6bfec8 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Mon, 2 Dec 2024 16:18:08 +0100 Subject: [PATCH 4/7] fix(events): avoid slow processing of events (#2259) Signed-off-by: Sylvain Leclerc --- antarest/eventbus/business/redis_eventbus.py | 20 +++++++++++++++----- antarest/eventbus/service.py | 18 ++++++++++++++---- resources/application.yaml | 6 ++++++ tests/eventbus/test_redis_event_bus.py | 5 +++-- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/antarest/eventbus/business/redis_eventbus.py b/antarest/eventbus/business/redis_eventbus.py index f3642bf994..8bbf6cbc38 100644 --- a/antarest/eventbus/business/redis_eventbus.py +++ b/antarest/eventbus/business/redis_eventbus.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) REDIS_STORE_KEY = "events" +MAX_EVENTS_LIST_SIZE = 1000 class RedisEventBus(IEventBusBackend): @@ -41,14 +42,23 @@ def pull_queue(self, queue: str) -> Optional[Event]: return None def get_events(self) -> List[Event]: + messages = [] try: - event = self.pubsub.get_message(ignore_subscribe_messages=True) - if event is not None: - return [Event.parse_raw(event["data"])] + while msg := self.pubsub.get_message(ignore_subscribe_messages=True): + messages.append(msg) + if len(messages) >= MAX_EVENTS_LIST_SIZE: + break except Exception: - logger.error("Failed to retrieve or parse event !", exc_info=True) + logger.error("Failed to retrieve events !", exc_info=True) - return [] + events = [] + for msg in messages: + try: + events.append(Event.model_validate_json(msg["data"])) + except Exception: + logger.error(f"Failed to parse event ! {msg}", exc_info=True) + + return events def clear_events(self) -> None: # Nothing to do diff --git a/antarest/eventbus/service.py b/antarest/eventbus/service.py index 201efb3be6..eadf75e8c8 100644 --- a/antarest/eventbus/service.py +++ b/antarest/eventbus/service.py @@ -24,6 +24,9 @@ logger = logging.getLogger(__name__) +EVENT_LOOP_REST_TIME = 0.2 + + class EventBusService(IEventBus): def __init__(self, backend: IEventBusBackend, autostart: bool = True) -> None: self.backend = backend @@ -76,18 +79,22 @@ def remove_listener(self, listener_id: str) -> None: async def _run_loop(self) -> None: while True: - time.sleep(0.2) try: - await self._on_events() + processed_events_count = await self._on_events() + # Give the loop some rest if it has nothing to do + if processed_events_count == 0: + await asyncio.sleep(EVENT_LOOP_REST_TIME) except Exception as e: logger.error("Unexpected error when processing events", exc_info=e) - async def _on_events(self) -> None: + async def _on_events(self) -> int: + processed_events_count = 0 with self.lock: for queue in self.consumers: if len(self.consumers[queue]) > 0: event = self.backend.pull_queue(queue) while event is not None: + processed_events_count += 1 try: await list(self.consumers[queue].values())[ random.randint(0, len(self.consumers[queue]) - 1) @@ -99,7 +106,9 @@ async def _on_events(self) -> None: ) event = self.backend.pull_queue(queue) - for e in self.backend.get_events(): + events = self.backend.get_events() + processed_events_count += len(events) + for e in events: if e.type in self.listeners: responses = await asyncio.gather( *[ @@ -115,6 +124,7 @@ async def _on_events(self) -> None: exc_info=res, ) self.backend.clear_events() + return processed_events_count def _async_loop(self, new_loop: bool = True) -> None: loop = asyncio.new_event_loop() if new_loop else asyncio.get_event_loop() diff --git a/resources/application.yaml b/resources/application.yaml index c962d09015..21ec482177 100644 --- a/resources/application.yaml +++ b/resources/application.yaml @@ -52,3 +52,9 @@ logging: # True to get sqlalchemy logs debug: False + +# Uncomment these lines to use redis as a backend for the eventbus +# It is required to use redis when using this application on multiple workers in a preforked model like gunicorn for instance +#redis: +# host: localhost +# port: 6379 diff --git a/tests/eventbus/test_redis_event_bus.py b/tests/eventbus/test_redis_event_bus.py index 37f69e1fad..fa8722f742 100644 --- a/tests/eventbus/test_redis_event_bus.py +++ b/tests/eventbus/test_redis_event_bus.py @@ -29,8 +29,9 @@ def test_lifecycle(): payload="foo", permissions=PermissionInfo(public_mode=PublicMode.READ), ) + serialized = event.model_dump_json() - pubsub_mock.get_message.return_value = {"data": serialized} + pubsub_mock.get_message = Mock(side_effect=[{"data": serialized}, {"data": serialized}, None]) eventbus.push_event(event) redis_client.publish.assert_called_once_with("events", serialized) - assert eventbus.get_events() == [event] + assert eventbus.get_events() == [event, event] From b66a81623c2f7abef5d10d07196cdb9a80dfe3cf Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:27:48 +0100 Subject: [PATCH 5/7] chore: update changelog for 2.18.1 release --- docs/CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7316cc8bc3..2a35203f73 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,16 @@ Antares Web Changelog v2.18.1 (2024-12-02) -------------------- +## What's Changed + +### Bug Fixes + +* **ui-tablemode**: style missing [`2257`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/2257) +* **ui-studies**: multiple API calls on study list view [`2258`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/2258) +* **events**: avoid slow processing of events [`2259`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/2259) + +**Full Changelog**: https://github.com/AntaresSimulatorTeam/AntaREST/compare/v2.18.0...v2.18.1 + v2.18.0 (2024-11-29) -------------------- From 1d4f40eb08ac033f2bcbc76a4769751031629274 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Tue, 3 Dec 2024 10:02:40 +0100 Subject: [PATCH 6/7] fix(ci): use fixed versions for gh actions for build stability (#2255) --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0e41bf08b1..c0f87cc216 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - - uses: isort/isort-action@master + - uses: isort/isort-action@v1.1.1 with: sort-paths: antarest, tests requirementsFiles: "requirements-dev.txt" @@ -156,7 +156,7 @@ jobs: with: name: python-code-coverage-report - name: SonarCloud Scan - uses: sonarsource/sonarcloud-github-action@master + uses: sonarsource/sonarcloud-github-action@v3.1.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From fe4bf0a6a23ec6f7adb30ba3e94f9ec27317f35e Mon Sep 17 00:00:00 2001 From: Theo Pascoli <48944759+TheoPascoli@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:06:25 +0100 Subject: [PATCH 7/7] fix(links): fix a bug that occurred when updating links via table-mode (#2256) --- antarest/study/business/link_management.py | 127 +++------- antarest/study/business/model/link_model.py | 119 ++++++++- .../study/business/table_mode_management.py | 13 +- .../rawstudy/model/filesystem/config/links.py | 231 ------------------ .../study_data_blueprint/test_link.py | 58 ++++- .../study_data_blueprint/test_table_mode.py | 159 ++++++------ .../storage/business/test_arealink_manager.py | 2 +- 7 files changed, 297 insertions(+), 412 deletions(-) delete mode 100644 antarest/study/storage/rawstudy/model/filesystem/config/links.py diff --git a/antarest/study/business/link_management.py b/antarest/study/business/link_management.py index 777ce3cd9e..7c6a971fde 100644 --- a/antarest/study/business/link_management.py +++ b/antarest/study/business/link_management.py @@ -11,34 +11,21 @@ # This file is part of the Antares project. import typing as t -from typing import Any, Dict +from typing import Any from antares.study.version import StudyVersion -from antarest.core.exceptions import ConfigFileNotFound, LinkNotFound, LinkValidationError +from antarest.core.exceptions import LinkNotFound from antarest.core.model import JSON -from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model from antarest.study.business.model.link_model import LinkBaseDTO, LinkDTO, LinkInternal from antarest.study.business.utils import execute_or_add_commands from antarest.study.model import RawStudy, Study -from antarest.study.storage.rawstudy.model.filesystem.config.links import LinkProperties from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.storage_service import StudyStorageService from antarest.study.storage.variantstudy.model.command.create_link import CreateLink from antarest.study.storage.variantstudy.model.command.remove_link import RemoveLink -from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig from antarest.study.storage.variantstudy.model.command.update_link import UpdateLink -_ALL_LINKS_PATH = "input/links" - - -@all_optional_model -@camel_case_model -class LinkOutput(LinkProperties): - """ - DTO object use to get the link information. - """ - class LinkManager: def __init__(self, storage_service: StudyStorageService) -> None: @@ -61,6 +48,17 @@ def get_all_links(self, study: Study) -> t.List[LinkDTO]: return result + def get_link(self, study: RawStudy, link: LinkInternal) -> LinkInternal: + file_study = self.storage_service.get_storage(study).get_raw(study) + + link_properties = self._get_link_if_exists(file_study, link) + + link_properties.update({"area1": link.area1, "area2": link.area2}) + + updated_link = LinkInternal.model_validate(link_properties) + + return updated_link + def create_link(self, study: Study, link_creation_dto: LinkDTO) -> LinkDTO: link = link_creation_dto.to_internal(StudyVersion.parse(study.version)) @@ -80,16 +78,16 @@ def create_link(self, study: Study, link_creation_dto: LinkDTO) -> LinkDTO: return link_creation_dto def update_link(self, study: RawStudy, area_from: str, area_to: str, link_update_dto: LinkBaseDTO) -> LinkDTO: - link_dto = LinkDTO(area1=area_from, area2=area_to, **link_update_dto.model_dump()) + link_dto = LinkDTO(area1=area_from, area2=area_to, **link_update_dto.model_dump(exclude_unset=True)) link = link_dto.to_internal(StudyVersion.parse(study.version)) file_study = self.storage_service.get_storage(study).get_raw(study) - self.get_link_if_exists(file_study, link) + self._get_link_if_exists(file_study, link) command = UpdateLink( - area1=area_from, - area2=area_to, + area1=link.area1, + area2=link.area2, parameters=link.model_dump( include=link_update_dto.model_fields_set, exclude={"area1", "area2"}, exclude_none=True ), @@ -99,25 +97,21 @@ def update_link(self, study: RawStudy, area_from: str, area_to: str, link_update execute_or_add_commands(study, file_study, [command], self.storage_service) - updated_link = self.get_internal_link(study, link) + updated_link = self.get_link(study, link) return updated_link.to_dto() - def get_link_if_exists(self, file_study: FileStudy, link: LinkInternal) -> dict[str, Any]: - try: - return file_study.tree.get(["input", "links", link.area1, "properties", link.area2]) - except KeyError: - raise LinkNotFound(f"The link {link.area1} -> {link.area2} is not present in the study") - - def get_internal_link(self, study: RawStudy, link: LinkInternal) -> LinkInternal: - file_study = self.storage_service.get_storage(study).get_raw(study) - - link_properties = self.get_link_if_exists(file_study, link) - - link_properties.update({"area1": link.area1, "area2": link.area2}) - updated_link = LinkInternal.model_validate(link_properties) + def update_links( + self, + study: RawStudy, + update_links_by_ids: t.Mapping[t.Tuple[str, str], LinkBaseDTO], + ) -> t.Mapping[t.Tuple[str, str], LinkBaseDTO]: + new_links_by_ids = {} + for (area1, area2), update_link_dto in update_links_by_ids.items(): + updated_link = self.update_link(study, area1, area2, update_link_dto) + new_links_by_ids[(area1, area2)] = updated_link - return updated_link + return new_links_by_ids def delete_link(self, study: RawStudy, area1_id: str, area2_id: str) -> None: file_study = self.storage_service.get_storage(study).get_raw(study) @@ -129,69 +123,12 @@ def delete_link(self, study: RawStudy, area1_id: str, area2_id: str) -> None: ) execute_or_add_commands(study, file_study, [command], self.storage_service) - def get_all_links_props(self, study: RawStudy) -> t.Mapping[t.Tuple[str, str], LinkOutput]: - """ - Retrieves all links properties from the study. - - Args: - study: The raw study object. - Returns: - A mapping of link IDS `(area1_id, area2_id)` to link properties. - Raises: - ConfigFileNotFound: if a configuration file is not found. - """ - file_study = self.storage_service.get_storage(study).get_raw(study) - - # Get the link information from the `input/links/{area1}/properties.ini` file. - path = _ALL_LINKS_PATH + def _get_link_if_exists(self, file_study: FileStudy, link: LinkInternal) -> dict[str, Any]: try: - links_cfg = file_study.tree.get(path.split("/"), depth=5) + return file_study.tree.get(["input", "links", link.area1, "properties", link.area2]) except KeyError: - raise ConfigFileNotFound(path) from None - - # areas_cfg contains a dictionary where the keys are the area IDs, - # and the values are objects that can be converted to `LinkFolder`. - links_by_ids = {} - for area1_id, entries in links_cfg.items(): - property_map = entries.get("properties") or {} - for area2_id, properties_cfg in property_map.items(): - area1_id, area2_id = sorted([area1_id, area2_id]) - properties = LinkProperties(**properties_cfg) - links_by_ids[(area1_id, area2_id)] = LinkOutput(**properties.model_dump(mode="json", by_alias=False)) - - return links_by_ids - - def update_links_props( - self, - study: RawStudy, - update_links_by_ids: t.Mapping[t.Tuple[str, str], LinkOutput], - ) -> t.Mapping[t.Tuple[str, str], LinkOutput]: - old_links_by_ids = self.get_all_links_props(study) - new_links_by_ids = {} - file_study = self.storage_service.get_storage(study).get_raw(study) - commands = [] - for (area1, area2), update_link_dto in update_links_by_ids.items(): - # Update the link properties. - old_link_dto = old_links_by_ids[(area1, area2)] - new_link_dto = old_link_dto.copy( - update=update_link_dto.model_dump(mode="json", by_alias=False, exclude_none=True) - ) - new_links_by_ids[(area1, area2)] = new_link_dto - - # Convert the DTO to a configuration object and update the configuration file. - properties = LinkProperties(**new_link_dto.model_dump(by_alias=False)) - path = f"{_ALL_LINKS_PATH}/{area1}/properties/{area2}" - cmd = UpdateConfig( - target=path, - data=properties.to_config(), - command_context=self.storage_service.variant_study_service.command_factory.command_context, - study_version=file_study.config.version, - ) - commands.append(cmd) - - execute_or_add_commands(study, file_study, commands, self.storage_service) - return new_links_by_ids + raise LinkNotFound(f"The link {link.area1} -> {link.area2} is not present in the study") @staticmethod def get_table_schema() -> JSON: - return LinkOutput.schema() + return LinkBaseDTO.model_json_schema() diff --git a/antarest/study/business/model/link_model.py b/antarest/study/business/model/link_model.py index 977ce69ee3..07398dc88d 100644 --- a/antarest/study/business/model/link_model.py +++ b/antarest/study/business/model/link_model.py @@ -12,19 +12,122 @@ import typing as t from antares.study.version import StudyVersion -from pydantic import ConfigDict, Field, model_validator +from pydantic import BeforeValidator, ConfigDict, Field, PlainSerializer, model_validator from antarest.core.exceptions import LinkValidationError from antarest.core.serialization import AntaresBaseModel from antarest.core.utils.string import to_camel_case, to_kebab_case +from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import STUDY_VERSION_8_2 -from antarest.study.storage.rawstudy.model.filesystem.config.links import ( - AssetType, - FilterOption, - LinkStyle, - TransmissionCapacity, - comma_separated_enum_list, -) + + +class AssetType(EnumIgnoreCase): + """ + Enum representing the type of asset for a link between two areas. + + Attributes: + AC: Represents an Alternating Current link. This is the most common type of electricity transmission. + DC: Represents a Direct Current link. This is typically used for long-distance transmission. + GAZ: Represents a gas link. This is used when the link is related to gas transmission. + VIRT: Represents a virtual link. This is used when the link doesn't physically exist + but is used for modeling purposes. + OTHER: Represents any other type of link that doesn't fall into the above categories. + """ + + AC = "ac" + DC = "dc" + GAZ = "gaz" + VIRT = "virt" + OTHER = "other" + + +class TransmissionCapacity(EnumIgnoreCase): + """ + Enum representing the transmission capacity of a link. + + Attributes: + INFINITE: Represents a link with infinite transmission capacity. + This means there are no limits on the amount of electricity that can be transmitted. + IGNORE: Represents a link where the transmission capacity is ignored. + This means the capacity is not considered during simulations. + ENABLED: Represents a link with a specific transmission capacity. + This means the capacity is considered in the model and has a certain limit. + """ + + INFINITE = "infinite" + IGNORE = "ignore" + ENABLED = "enabled" + + +class LinkStyle(EnumIgnoreCase): + """ + Enum representing the style of a link in a network visualization. + + Attributes: + DOT: Represents a dotted line style. + PLAIN: Represents a solid line style. + DASH: Represents a dashed line style. + DOT_DASH: Represents a line style with alternating dots and dashes. + """ + + DOT = "dot" + PLAIN = "plain" + DASH = "dash" + DOT_DASH = "dotdash" + OTHER = "other" + + +class FilterOption(EnumIgnoreCase): + """ + Enum representing the time filter options for data visualization or analysis in Antares Web. + + Attributes: + HOURLY: Represents filtering data by the hour. + DAILY: Represents filtering data by the day. + WEEKLY: Represents filtering data by the week. + MONTHLY: Represents filtering data by the month. + ANNUAL: Represents filtering data by the year. + """ + + HOURLY = "hourly" + DAILY = "daily" + WEEKLY = "weekly" + MONTHLY = "monthly" + ANNUAL = "annual" + + +def validate_filters( + filter_value: t.Union[t.List[FilterOption], str], enum_cls: t.Type[FilterOption] +) -> t.List[FilterOption]: + if isinstance(filter_value, str): + if not filter_value.strip(): + return [] + + filter_accepted_values = [e for e in enum_cls] + + options = filter_value.replace(" ", "").split(",") + + invalid_options = [opt for opt in options if opt not in filter_accepted_values] + if invalid_options: + raise LinkValidationError( + f"Invalid value(s) in filters: {', '.join(invalid_options)}. " + f"Allowed values are: {', '.join(filter_accepted_values)}." + ) + options_enum: t.List[FilterOption] = list(dict.fromkeys(enum_cls(opt) for opt in options)) + return options_enum + + return filter_value + + +def join_with_comma(values: t.List[FilterOption]) -> str: + return ", ".join(value.name.lower() for value in values) + + +comma_separated_enum_list = t.Annotated[ + t.List[FilterOption], + BeforeValidator(lambda x: validate_filters(x, FilterOption)), + PlainSerializer(lambda x: join_with_comma(x)), +] DEFAULT_COLOR = 112 FILTER_VALUES: t.List[FilterOption] = [ diff --git a/antarest/study/business/table_mode_management.py b/antarest/study/business/table_mode_management.py index 5905c78faf..aba99c9b57 100644 --- a/antarest/study/business/table_mode_management.py +++ b/antarest/study/business/table_mode_management.py @@ -25,7 +25,8 @@ from antarest.study.business.areas.thermal_management import ThermalClusterInput, ThermalManager from antarest.study.business.binding_constraint_management import BindingConstraintManager, ConstraintInput from antarest.study.business.enum_ignore_case import EnumIgnoreCase -from antarest.study.business.link_management import LinkManager, LinkOutput +from antarest.study.business.link_management import LinkManager +from antarest.study.business.model.link_model import LinkBaseDTO from antarest.study.model import STUDY_VERSION_8_2, RawStudy _TableIndex = str # row name @@ -98,15 +99,15 @@ def _get_table_data_unsafe(self, study: RawStudy, table_type: TableModeType) -> areas_map = self._area_manager.get_all_area_props(study) data = {area_id: area.model_dump(mode="json", by_alias=True) for area_id, area in areas_map.items()} elif table_type == TableModeType.LINK: - links_map = self._link_manager.get_all_links_props(study) + links_map = self._link_manager.get_all_links(study) excludes = ( set() if StudyVersion.parse(study.version) >= STUDY_VERSION_8_2 else {"filter_synthesis", "filter_year_by_year"} ) data = { - f"{area1_id} / {area2_id}": link.model_dump(mode="json", by_alias=True, exclude=excludes) - for (area1_id, area2_id), link in links_map.items() + f"{link.area1} / {link.area2}": link.model_dump(mode="json", by_alias=True, exclude=excludes) + for link in links_map } elif table_type == TableModeType.THERMAL: thermals_by_areas = self._thermal_manager.get_all_thermals_props(study) @@ -199,8 +200,8 @@ def update_table_data( data = {area_id: area.model_dump(by_alias=True, exclude_none=True) for area_id, area in areas_map.items()} return data elif table_type == TableModeType.LINK: - links_map = {tuple(key.split(" / ")): LinkOutput(**values) for key, values in data.items()} - updated_map = self._link_manager.update_links_props(study, links_map) # type: ignore + links_map = {tuple(key.split(" / ")): LinkBaseDTO(**values) for key, values in data.items()} + updated_map = self._link_manager.update_links(study, links_map) # type: ignore excludes = ( set() if StudyVersion.parse(study.version) >= STUDY_VERSION_8_2 diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/links.py b/antarest/study/storage/rawstudy/model/filesystem/config/links.py deleted file mode 100644 index 9bfe12cd11..0000000000 --- a/antarest/study/storage/rawstudy/model/filesystem/config/links.py +++ /dev/null @@ -1,231 +0,0 @@ -# Copyright (c) 2024, RTE (https://www.rte-france.com) -# -# See AUTHORS.txt -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# SPDX-License-Identifier: MPL-2.0 -# -# This file is part of the Antares project. - -""" -Object model used to read and update link configuration. -""" - -import typing as t - -from pydantic import BeforeValidator, Field, PlainSerializer, field_validator, model_validator - -from antarest.core.exceptions import LinkValidationError -from antarest.study.business.enum_ignore_case import EnumIgnoreCase -from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import ( - validate_color_rgb, - validate_colors, - validate_filtering, -) -from antarest.study.storage.rawstudy.model.filesystem.config.ini_properties import IniProperties - - -# noinspection SpellCheckingInspection -class AssetType(EnumIgnoreCase): - """ - Enum representing the type of asset for a link between two areas. - - Attributes: - AC: Represents an Alternating Current link. This is the most common type of electricity transmission. - DC: Represents a Direct Current link. This is typically used for long-distance transmission. - GAZ: Represents a gas link. This is used when the link is related to gas transmission. - VIRT: Represents a virtual link. This is used when the link doesn't physically exist - but is used for modeling purposes. - OTHER: Represents any other type of link that doesn't fall into the above categories. - """ - - AC = "ac" - DC = "dc" - GAZ = "gaz" - VIRT = "virt" - OTHER = "other" - - -class TransmissionCapacity(EnumIgnoreCase): - """ - Enum representing the transmission capacity of a link. - - Attributes: - INFINITE: Represents a link with infinite transmission capacity. - This means there are no limits on the amount of electricity that can be transmitted. - IGNORE: Represents a link where the transmission capacity is ignored. - This means the capacity is not considered during simulations. - ENABLED: Represents a link with a specific transmission capacity. - This means the capacity is considered in the model and has a certain limit. - """ - - INFINITE = "infinite" - IGNORE = "ignore" - ENABLED = "enabled" - - -class LinkStyle(EnumIgnoreCase): - """ - Enum representing the style of a link in a network visualization. - - Attributes: - DOT: Represents a dotted line style. - PLAIN: Represents a solid line style. - DASH: Represents a dashed line style. - DOT_DASH: Represents a line style with alternating dots and dashes. - """ - - DOT = "dot" - PLAIN = "plain" - DASH = "dash" - DOT_DASH = "dotdash" - OTHER = "other" - - -class FilterOption(EnumIgnoreCase): - """ - Enum representing the time filter options for data visualization or analysis in Antares Web. - - Attributes: - HOURLY: Represents filtering data by the hour. - DAILY: Represents filtering data by the day. - WEEKLY: Represents filtering data by the week. - MONTHLY: Represents filtering data by the month. - ANNUAL: Represents filtering data by the year. - """ - - HOURLY = "hourly" - DAILY = "daily" - WEEKLY = "weekly" - MONTHLY = "monthly" - ANNUAL = "annual" - - -def validate_filters( - filter_value: t.Union[t.List[FilterOption], str], enum_cls: t.Type[FilterOption] -) -> t.List[FilterOption]: - if isinstance(filter_value, str): - if not filter_value.strip(): - return [] - - filter_accepted_values = [e for e in enum_cls] - - options = filter_value.replace(" ", "").split(",") - - invalid_options = [opt for opt in options if opt not in filter_accepted_values] - if invalid_options: - raise LinkValidationError( - f"Invalid value(s) in filters: {', '.join(invalid_options)}. " - f"Allowed values are: {', '.join(filter_accepted_values)}." - ) - - return [enum_cls(opt) for opt in options] - - return filter_value - - -def join_with_comma(values: t.List[FilterOption]) -> str: - return ", ".join(value.name.lower() for value in values) - - -comma_separated_enum_list = t.Annotated[ - t.List[FilterOption], - BeforeValidator(lambda x: validate_filters(x, FilterOption)), - PlainSerializer(lambda x: join_with_comma(x)), -] - - -class LinkProperties(IniProperties): - """ - Configuration read from a section in the `input/links//properties.ini` file. - - Usage: - - >>> from antarest.study.storage.rawstudy.model.filesystem.config.links import LinkProperties - >>> from pprint import pprint - - Create and validate a new `LinkProperties` object from a dictionary read from a configuration file. - - >>> obj = { - ... "hurdles-cost": "false", - ... "loop-flow": "false", - ... "use-phase-shifter": "false", - ... "transmission-capacities": "infinite", - ... "asset-type": "ac", - ... "link-style": "plain", - ... "link-width": "1", - ... "colorr": "80", - ... "colorg": "192", - ... "colorb": "255", - ... "comments": "This is a link", - ... "display-comments": "true", - ... "filter-synthesis": "hourly, daily, weekly, monthly, annual", - ... "filter-year-by-year": "hourly, daily, weekly, monthly, annual", - ... } - - >>> opt = LinkProperties(**obj) - - >>> pprint(opt.model_dump(by_alias=True), width=80) - {'asset-type': , - 'colorRgb': '#50C0FF', - 'comments': 'This is a link', - 'display-comments': True, - 'filter-synthesis': 'hourly, daily, weekly, monthly, annual', - 'filter-year-by-year': 'hourly, daily, weekly, monthly, annual', - 'hurdles-cost': False, - 'link-style': 'plain', - 'link-width': 1, - 'loop-flow': False, - 'transmission-capacities': , - 'use-phase-shifter': False} - - >>> pprint(opt.to_config(), width=80) - {'asset-type': 'ac', - 'colorb': 255, - 'colorg': 192, - 'colorr': 80, - 'comments': 'This is a link', - 'display-comments': True, - 'filter-synthesis': 'hourly, daily, weekly, monthly, annual', - 'filter-year-by-year': 'hourly, daily, weekly, monthly, annual', - 'hurdles-cost': False, - 'link-style': 'plain', - 'link-width': 1, - 'loop-flow': False, - 'transmission-capacities': 'infinite', - 'use-phase-shifter': False} - """ - - hurdles_cost: bool = Field(default=False, alias="hurdles-cost") - loop_flow: bool = Field(default=False, alias="loop-flow") - use_phase_shifter: bool = Field(default=False, alias="use-phase-shifter") - transmission_capacities: TransmissionCapacity = Field( - default=TransmissionCapacity.ENABLED, alias="transmission-capacities" - ) - asset_type: AssetType = Field(default=AssetType.AC, alias="asset-type") - link_style: str = Field(default="plain", alias="link-style") - link_width: int = Field(default=1, alias="link-width") - comments: str = Field(default="", alias="comments") # unknown field?! - display_comments: bool = Field(default=True, alias="display-comments") - filter_synthesis: str = Field(default="", alias="filter-synthesis") - filter_year_by_year: str = Field(default="", alias="filter-year-by-year") - color_rgb: str = Field( - "#707070", - alias="colorRgb", - description="color of the area in the map", - ) - - @field_validator("filter_synthesis", "filter_year_by_year", mode="before") - def _validate_filtering(cls, v: t.Any) -> str: - return validate_filtering(v) - - @field_validator("color_rgb", mode="before") - def _validate_color_rgb(cls, v: t.Any) -> str: - return validate_color_rgb(v) - - @model_validator(mode="before") - def _validate_colors(cls, values: t.MutableMapping[str, t.Any]) -> t.Mapping[str, t.Any]: - return validate_colors(values) diff --git a/tests/integration/study_data_blueprint/test_link.py b/tests/integration/study_data_blueprint/test_link.py index 635aedb830..8cf04d58f8 100644 --- a/tests/integration/study_data_blueprint/test_link.py +++ b/tests/integration/study_data_blueprint/test_link.py @@ -9,12 +9,10 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. -from sys import stderr - import pytest from starlette.testclient import TestClient -from antarest.study.storage.rawstudy.model.filesystem.config.links import TransmissionCapacity +from antarest.study.business.model.link_model import TransmissionCapacity from tests.integration.prepare_proxy import PreparerProxy @@ -70,6 +68,32 @@ def test_link_update(self, client: TestClient, user_access_token: str, study_typ } assert expected == res.json() + # Test update link area not ordered + + res = client.put( + f"/v1/studies/{study_id}/links/{area2_id}/{area1_id}", + json={"hurdlesCost": False}, + ) + assert res.status_code == 200 + expected = { + "area1": "area 1", + "area2": "area 2", + "assetType": "ac", + "colorb": 112, + "colorg": 112, + "colorr": 150, + "displayComments": True, + "filterSynthesis": "hourly, daily, weekly, monthly, annual", + "filterYearByYear": "hourly, daily, weekly, monthly, annual", + "hurdlesCost": False, + "linkStyle": "plain", + "linkWidth": 1.0, + "loopFlow": False, + "transmissionCapacities": "enabled", + "usePhaseShifter": False, + } + assert expected == res.json() + # Test update link with non existing area res = client.put( @@ -286,6 +310,34 @@ def test_link_820(self, client: TestClient, user_access_token: str, study_type: } assert expected == res.json() + # Test create link with double value in filter + + client.delete(f"/v1/studies/{study_id}/links/{area1_id}/{area2_id}") + res = client.post( + f"/v1/studies/{study_id}/links", + json={"area1": area1_id, "area2": area2_id, "filterSynthesis": "hourly, hourly"}, + ) + + assert res.status_code == 200, res.json() + expected = { + "area1": "area 1", + "area2": "area 2", + "assetType": "ac", + "colorb": 112, + "colorg": 112, + "colorr": 112, + "displayComments": True, + "filterSynthesis": "hourly", + "filterYearByYear": "hourly, daily, weekly, monthly, annual", + "hurdlesCost": False, + "linkStyle": "plain", + "linkWidth": 1.0, + "loopFlow": False, + "transmissionCapacities": "enabled", + "usePhaseShifter": False, + } + assert expected == res.json() + def test_create_link_810(self, client: TestClient, user_access_token: str) -> None: client.headers = {"Authorization": f"Bearer {user_access_token}"} # type: ignore diff --git a/tests/integration/study_data_blueprint/test_table_mode.py b/tests/integration/study_data_blueprint/test_table_mode.py index a7b1878020..17133f63be 100644 --- a/tests/integration/study_data_blueprint/test_table_mode.py +++ b/tests/integration/study_data_blueprint/test_table_mode.py @@ -14,9 +14,11 @@ import typing as t import pytest +from antares.study.version import StudyVersion from starlette.testclient import TestClient from antarest.core.tasks.model import TaskStatus +from antarest.study.model import STUDY_VERSION_8_2 from tests.integration.utils import wait_task_completion # noinspection SpellCheckingInspection @@ -188,61 +190,63 @@ def test_lifecycle__nominal( assert res.status_code == 200, res.json() actual = res.json() assert set(actual["properties"]) == { - "colorRgb", - "comments", - "hurdlesCost", - "loopFlow", - "usePhaseShifter", - "transmissionCapacities", "assetType", - "linkStyle", - "linkWidth", + "colorb", + "colorg", + "colorr", "displayComments", "filterSynthesis", "filterYearByYear", + "hurdlesCost", + "linkStyle", + "linkWidth", + "loopFlow", + "transmissionCapacities", + "usePhaseShifter", } - res = client.put( - f"/v1/studies/{internal_study_id}/table-mode/links", - json={ - "de / fr": { - "colorRgb": "#FFA500", - "displayComments": False, - "filterSynthesis": "hourly, daily, weekly, annual", - "filterYearByYear": "hourly, daily, monthly, annual", - "hurdlesCost": True, - "linkStyle": "plain", - "linkWidth": 2, - "loopFlow": False, - "transmissionCapacities": "ignore", - }, - "es / fr": { - "colorRgb": "#FF6347", - "displayComments": True, - "filterSynthesis": "hourly, daily, weekly, monthly, annual, annual", # duplicate is ignored - "filterYearByYear": "hourly, daily, weekly, annual", - "hurdlesCost": True, - "linkStyle": "plain", - "linkWidth": 1, - "loopFlow": False, - "transmissionCapacities": "enabled", - "usePhaseShifter": True, - }, - "fr / it": { - "comments": "Link from France to Italie", - "assetType": "DC", # case-insensitive - }, + # Test links + + json_input = { + "de / fr": { + "assetType": "ac", + "colorb": 100, + "colorg": 150, + "colorr": 200, + "displayComments": False, + "hurdlesCost": True, + "linkStyle": "plain", + "linkWidth": 2, + "loopFlow": False, + "transmissionCapacities": "ignore", }, - ) - assert res.status_code == 200, res.json() + "es / fr": { + "assetType": "ac", + "colorb": 100, + "colorg": 150, + "colorr": 200, + "displayComments": True, + "hurdlesCost": True, + "linkStyle": "plain", + "linkWidth": 1, + "loopFlow": False, + "transmissionCapacities": "enabled", + "usePhaseShifter": True, + }, + "fr / it": { + "assetType": "dc", # case-insensitive + }, + } + expected_links = { "de / fr": { + "area1": "de", + "area2": "fr", "assetType": "ac", - "colorRgb": "#FFA500", - "comments": "", + "colorb": 100, + "colorg": 150, + "colorr": 200, "displayComments": False, - "filterSynthesis": "hourly, daily, weekly, annual", - "filterYearByYear": "hourly, daily, monthly, annual", "hurdlesCost": True, "linkStyle": "plain", "linkWidth": 2, @@ -251,12 +255,13 @@ def test_lifecycle__nominal( "usePhaseShifter": False, }, "de / it": { + "area1": "de", + "area2": "it", "assetType": "ac", - "colorRgb": "#707070", - "comments": "", + "colorr": 112, + "colorg": 112, + "colorb": 112, "displayComments": True, - "filterSynthesis": "hourly, daily, weekly, monthly, annual", - "filterYearByYear": "hourly, daily, weekly, monthly, annual", "hurdlesCost": False, "linkStyle": "plain", "linkWidth": 1, @@ -265,12 +270,13 @@ def test_lifecycle__nominal( "usePhaseShifter": False, }, "es / fr": { + "area1": "es", + "area2": "fr", "assetType": "ac", - "colorRgb": "#FF6347", - "comments": "", + "colorb": 100, + "colorg": 150, + "colorr": 200, "displayComments": True, - "filterSynthesis": "hourly, daily, weekly, monthly, annual", - "filterYearByYear": "hourly, daily, weekly, annual", "hurdlesCost": True, "linkStyle": "plain", "linkWidth": 1, @@ -279,12 +285,13 @@ def test_lifecycle__nominal( "usePhaseShifter": True, }, "fr / it": { + "area1": "fr", + "area2": "it", "assetType": "dc", - "colorRgb": "#707070", - "comments": "Link from France to Italie", + "colorb": 112, + "colorg": 112, + "colorr": 112, "displayComments": True, - "filterSynthesis": "", - "filterYearByYear": "hourly", "hurdlesCost": True, "linkStyle": "plain", "linkWidth": 1, @@ -293,23 +300,39 @@ def test_lifecycle__nominal( "usePhaseShifter": False, }, } - # removes filter fields for study version prior to v8.2 - if study_version < 820: - for key in expected_links: - del expected_links[key]["filterSynthesis"] - del expected_links[key]["filterYearByYear"] - # asserts actual equals expected without the non-updated link. + # Add filters for versions > 8.2 + if StudyVersion.parse(study_version) > STUDY_VERSION_8_2: + for link in json_input.values(): + link.update( + { + "filterSynthesis": "hourly, daily, weekly, monthly, annual", + "filterYearByYear": "hourly, daily, weekly, monthly, annual", + } + ) + for link in expected_links.values(): + link.update( + { + "filterSynthesis": "hourly, daily, weekly, monthly, annual", + "filterYearByYear": "hourly, daily, weekly, monthly, annual", + } + ) + + res = client.put(f"/v1/studies/{internal_study_id}/table-mode/links", json=json_input) + assert res.status_code == 200, res.json() + actual = res.json() - expected_result = copy.deepcopy(expected_links) - del expected_result["de / it"] - assert actual == expected_result + expected_partial = copy.deepcopy(expected_links) + del expected_partial["de / it"] + assert actual == expected_partial res = client.get(f"/v1/studies/{internal_study_id}/table-mode/links") assert res.status_code == 200, res.json() - actual = res.json() - # asserts the `de / it` link is not removed. - assert actual == expected_links + assert res.json() == expected_links + + # GET request to make sure that the GET /links works + res = client.get(f"/v1/studies/{internal_study_id}/links") + assert res.status_code == 200, res.json() # Table Mode - Thermal Clusters # ============================= diff --git a/tests/storage/business/test_arealink_manager.py b/tests/storage/business/test_arealink_manager.py index adae212c4f..b61074b8b7 100644 --- a/tests/storage/business/test_arealink_manager.py +++ b/tests/storage/business/test_arealink_manager.py @@ -25,11 +25,11 @@ from antarest.matrixstore.service import SimpleMatrixService from antarest.study.business.area_management import AreaCreationDTO, AreaManager, AreaType, UpdateAreaUi from antarest.study.business.link_management import LinkDTO, LinkManager +from antarest.study.business.model.link_model import AssetType, LinkStyle, TransmissionCapacity from antarest.study.model import Patch, PatchArea, PatchCluster, RawStudy, StudyAdditionalData from antarest.study.repository import StudyMetadataRepository from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.files import build -from antarest.study.storage.rawstudy.model.filesystem.config.links import AssetType, LinkStyle, TransmissionCapacity from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet, FileStudyTreeConfig, Link from antarest.study.storage.rawstudy.model.filesystem.config.thermal import ThermalConfig from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy