From 67dd82e4fee683db9d66d95242b15789f872669c Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 11 Oct 2023 18:21:18 +0200 Subject: [PATCH 1/5] build: prepare hotfix v2.15.3 (unreleased) --- antarest/__init__.py | 4 ++-- docs/CHANGELOG.md | 4 ++++ setup.py | 2 +- sonar-project.properties | 2 +- webapp/package-lock.json | 2 +- webapp/package.json | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index 447ab7edd4..210b69a193 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -7,9 +7,9 @@ # Standard project metadata -__version__ = "2.15.2" +__version__ = "2.15.3" __author__ = "RTE, Antares Web Team" -__date__ = "2023-10-11" +__date__ = "unreleased" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4fc768bc04..3ad29ebaf1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,10 @@ Antares Web Changelog ===================== +v2.15.3 (unreleased) +-------------------- + + v2.15.2 (2023-10-11) -------------------- diff --git a/setup.py b/setup.py index d0b8c5deb3..1fc961fedf 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="AntaREST", - version="2.15.2", + version="2.15.3", description="Antares Server", long_description=Path("README.md").read_text(encoding="utf-8"), long_description_content_type="text/markdown", diff --git a/sonar-project.properties b/sonar-project.properties index 1d3a132a04..8742aa5a83 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.8 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info -sonar.projectVersion=2.15.2 +sonar.projectVersion=2.15.3 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/* \ No newline at end of file diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 3ab3cca0b7..e179a3f1ca 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.15.2", + "version": "2.15.3", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/webapp/package.json b/webapp/package.json index 506c324bf2..77204beb44 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.15.2", + "version": "2.15.3", "private": true, "engines": { "node": "18.16.1" From 8175d274b178ffacb6d438c20b1867297f239c3f Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:16:59 +0200 Subject: [PATCH 2/5] fix(api-raw): correct the `/studies/{uuid}/raw` endpoint to return "text/plain" content for matrices (#1766) --- antarest/study/web/raw_studies_blueprint.py | 11 ++- .../test_fetch_raw_data.py | 67 ++++++++++++++++--- tests/storage/web/test_studies_bp.py | 8 ++- 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index 1105a5f74a..21d19620e2 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -96,11 +96,16 @@ def get_study( extra={"user": current_user.id}, ) parameters = RequestParameters(user=current_user) - output = study_service.get(uuid, path, depth, formatted, parameters) + + resource_path = pathlib.PurePosixPath(path) + output = study_service.get(uuid, str(resource_path), depth=depth, formatted=formatted, params=parameters) if isinstance(output, bytes): - resource_path = pathlib.Path(path) - suffix = resource_path.suffix.lower() + # Guess the suffix form the target data + parent_cfg = study_service.get(uuid, str(resource_path.parent), depth=2, formatted=True, params=parameters) + child = parent_cfg[resource_path.name] + suffix = pathlib.PurePosixPath(child).suffix + content_type, encoding = CONTENT_TYPES.get(suffix, (None, None)) if content_type == "application/json": # Use `JSONResponse` to ensure to return a valid JSON response diff --git a/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py b/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py index 7b5bc4e38e..04a6b3fbfc 100644 --- a/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py +++ b/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py @@ -2,7 +2,7 @@ import json import pathlib import shutil -from urllib.parse import urlencode +from unittest.mock import ANY import numpy as np import pytest @@ -54,9 +54,9 @@ def test_get_study( user_folder_dir = study_dir.joinpath("user/folder") for file_path in user_folder_dir.glob("*.*"): rel_path = file_path.relative_to(study_dir).as_posix() - query_string = urlencode({"path": f"/{rel_path}", "depth": 1}) res = client.get( - f"/v1/studies/{study_id}/raw?{query_string}", + f"/v1/studies/{study_id}/raw", + params={"path": f"/{rel_path}", "depth": 1}, headers=headers, ) res.raise_for_status() @@ -80,9 +80,9 @@ def test_get_study( user_folder_dir = study_dir.joinpath("user/unknown") for file_path in user_folder_dir.glob("*.*"): rel_path = file_path.relative_to(study_dir) - query_string = urlencode({"path": f"/{rel_path.as_posix()}", "depth": 1}) res = client.get( - f"/v1/studies/{study_id}/raw?{query_string}", + f"/v1/studies/{study_id}/raw", + params={"path": f"/{rel_path.as_posix()}", "depth": 1}, headers=headers, ) res.raise_for_status() @@ -90,22 +90,71 @@ def test_get_study( expected = file_path.read_bytes() assert actual == expected + # If we ask for properties, we should have a JSON content + rel_path = "/input/links/de/properties/fr" + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": f"/{rel_path}", "depth": 2}, + headers=headers, + ) + res.raise_for_status() + actual = res.json() + assert actual == { + "asset-type": "ac", + "colorb": 112, + "colorg": 112, + "colorr": 112, + "display-comments": True, + "filter-synthesis": "", + "filter-year-by-year": "hourly", + "hurdles-cost": True, + "link-style": "plain", + "link-width": 1, + "loop-flow": False, + "transmission-capacities": "enabled", + "use-phase-shifter": False, + } + + # If we ask for a matrix, we should have a JSON content if formatted is True + rel_path = "/input/links/de/fr" + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": f"/{rel_path}", "formatted": True}, + headers=headers, + ) + res.raise_for_status() + actual = res.json() + assert actual == {"index": ANY, "columns": ANY, "data": ANY} + + # If we ask for a matrix, we should have a CSV content if formatted is False + rel_path = "/input/links/de/fr" + res = client.get( + f"/v1/studies/{study_id}/raw", + params={"path": f"/{rel_path}", "formatted": False}, + headers=headers, + ) + res.raise_for_status() + actual = res.text + actual_lines = actual.splitlines() + first_row = [float(x) for x in actual_lines[0].split("\t")] + assert first_row == [100000, 100000, 0.010000, 0.010000, 0, 0, 0, 0] + # Some files can be corrupted user_folder_dir = study_dir.joinpath("user/bad") for file_path in user_folder_dir.glob("*.*"): rel_path = file_path.relative_to(study_dir) - query_string = urlencode({"path": f"/{rel_path.as_posix()}", "depth": 1}) res = client.get( - f"/v1/studies/{study_id}/raw?{query_string}", + f"/v1/studies/{study_id}/raw", + params={"path": f"/{rel_path.as_posix()}", "depth": 1}, headers=headers, ) assert res.status_code == http.HTTPStatus.UNPROCESSABLE_ENTITY # We can access to the configuration the classic way, # for instance, we can get the list of areas: - query_string = urlencode({"path": "/input/areas/list", "depth": 1}) res = client.get( - f"/v1/studies/{study_id}/raw?{query_string}", + f"/v1/studies/{study_id}/raw", + params={"path": "/input/areas/list", "depth": 1}, headers=headers, ) res.raise_for_status() diff --git a/tests/storage/web/test_studies_bp.py b/tests/storage/web/test_studies_bp.py index 91fde089c1..05366331ca 100644 --- a/tests/storage/web/test_studies_bp.py +++ b/tests/storage/web/test_studies_bp.py @@ -72,7 +72,9 @@ def test_server() -> None: client = TestClient(app) client.get("/v1/studies/study1/raw?path=settings/general/params") - mock_service.get.assert_called_once_with("study1", "settings/general/params", 3, True, PARAMS) + mock_service.get.assert_called_once_with( + "study1", "settings/general/params", depth=3, formatted=True, params=PARAMS + ) @pytest.mark.unit_test @@ -121,7 +123,7 @@ def test_server_with_parameters() -> None: parameters = RequestParameters(user=ADMIN) assert result.status_code == HTTPStatus.OK - mock_storage_service.get.assert_called_once_with("study1", "/", 4, True, parameters) + mock_storage_service.get.assert_called_once_with("study1", "/", depth=4, formatted=True, params=parameters) result = client.get("/v1/studies/study2/raw?depth=WRONG_TYPE") assert result.status_code == HTTPStatus.UNPROCESSABLE_ENTITY @@ -130,7 +132,7 @@ def test_server_with_parameters() -> None: assert result.status_code == HTTPStatus.OK excepted_parameters = RequestParameters(user=ADMIN) - mock_storage_service.get.assert_called_with("study2", "/", 3, True, excepted_parameters) + mock_storage_service.get.assert_called_with("study2", "/", depth=3, formatted=True, params=excepted_parameters) @pytest.mark.unit_test From 46829b853ea6b9c71cdbafbb0d5554160d7b33e1 Mon Sep 17 00:00:00 2001 From: MartinBelthle <102529366+MartinBelthle@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:43:20 +0200 Subject: [PATCH 3/5] feat(storage): make study_upgrader much faster (#1533) --- .../study/storage/study_upgrader/__init__.py | 120 ++++++++++++++---- 1 file changed, 98 insertions(+), 22 deletions(-) diff --git a/antarest/study/storage/study_upgrader/__init__.py b/antarest/study/storage/study_upgrader/__init__.py index 6fbfcfa29b..0886f3e04e 100644 --- a/antarest/study/storage/study_upgrader/__init__.py +++ b/antarest/study/storage/study_upgrader/__init__.py @@ -6,7 +6,7 @@ from http import HTTPStatus from http.client import HTTPException from pathlib import Path -from typing import Callable, NamedTuple +from typing import Callable, List, NamedTuple from antarest.core.exceptions import StudyValidationError @@ -29,18 +29,34 @@ class UpgradeMethod(NamedTuple): old: str new: str method: Callable[[Path], None] + files: List[Path] UPGRADE_METHODS = [ - UpgradeMethod("700", "710", upgrade_710), - UpgradeMethod("710", "720", upgrade_720), - UpgradeMethod("720", "800", upgrade_800), - UpgradeMethod("800", "810", upgrade_810), - UpgradeMethod("810", "820", upgrade_820), - UpgradeMethod("820", "830", upgrade_830), - UpgradeMethod("830", "840", upgrade_840), - UpgradeMethod("840", "850", upgrade_850), - UpgradeMethod("850", "860", upgrade_860), + UpgradeMethod("700", "710", upgrade_710, [Path("settings/generaldata.ini")]), + UpgradeMethod("710", "720", upgrade_720, []), + UpgradeMethod("720", "800", upgrade_800, [Path("settings/generaldata.ini")]), + UpgradeMethod( + "800", + "810", + upgrade_810, + [Path("settings/generaldata.ini"), Path("input")], + ), + UpgradeMethod("810", "820", upgrade_820, [Path("input/links")]), + UpgradeMethod( + "820", + "830", + upgrade_830, + [Path("settings/generaldata.ini"), Path("input/areas")], + ), + UpgradeMethod("830", "840", upgrade_840, [Path("settings/generaldata.ini")]), + UpgradeMethod("840", "850", upgrade_850, [Path("settings/generaldata.ini")]), + UpgradeMethod( + "850", + "860", + upgrade_860, + [Path("input"), Path("settings/generaldata.ini")], + ), ] @@ -68,10 +84,10 @@ def find_next_version(from_version: str) -> str: def upgrade_study(study_path: Path, target_version: str) -> None: tmp_dir = Path(tempfile.mkdtemp(suffix=".upgrade.tmp", prefix="~", dir=study_path.parent)) - shutil.copytree(study_path, tmp_dir, dirs_exist_ok=True) try: - src_version = get_current_version(tmp_dir) - can_upgrade_version(src_version, target_version) + src_version = get_current_version(study_path) + files_to_upgrade = can_upgrade_version(src_version, target_version) + files_to_retrieve = _copies_only_necessary_files(files_to_upgrade, study_path, tmp_dir) _do_upgrade(tmp_dir, src_version, target_version) except (StudyValidationError, InvalidUpgrade) as e: shutil.rmtree(tmp_dir) @@ -82,11 +98,7 @@ def upgrade_study(study_path: Path, target_version: str) -> None: logger.error(f"Unhandled exception : {e}", exc_info=True) raise else: - backup_dir = Path(tempfile.mkdtemp(suffix=".backup.tmp", prefix="~", dir=study_path.parent)) - backup_dir.rmdir() - study_path.rename(backup_dir) - tmp_dir.rename(study_path) - shutil.rmtree(backup_dir, ignore_errors=True) + _replace_safely_original_files(files_to_retrieve, study_path, tmp_dir) def get_current_version(study_path: Path) -> str: @@ -116,7 +128,7 @@ def get_current_version(study_path: Path) -> str: ) -def can_upgrade_version(from_version: str, to_version: str) -> None: +def can_upgrade_version(from_version: str, to_version: str) -> List[Path]: """ Checks if upgrading from one version to another is possible. @@ -124,9 +136,13 @@ def can_upgrade_version(from_version: str, to_version: str) -> None: from_version: The current version of the study. to_version: The target version of the study. + Returns: + If the upgrade is possible, the list of concerned folders and files + Raises: InvalidUpgrade: If the upgrade is not possible. """ + list_versions = [] if from_version == to_version: raise InvalidUpgrade(f"Your study is already in version '{to_version}'") @@ -138,12 +154,16 @@ def can_upgrade_version(from_version: str, to_version: str) -> None: if to_version not in targets: raise InvalidUpgrade(f"Version '{to_version}' unknown: possible versions are {', '.join(targets)}") + files = [u.files for u in UPGRADE_METHODS] curr_version = from_version - for src, dst in zip(sources, targets): + for src, dst, file in zip(sources, targets, files): if curr_version == src: + for path in file: + if path not in list_versions: + list_versions.append(path) curr_version = dst if curr_version == to_version: - return + return list_versions # This code must be unreachable! raise InvalidUpgrade( @@ -171,10 +191,66 @@ def _update_study_antares_file(target_version: str, study_path: Path) -> None: file.write_text(content, encoding="utf-8") +def _copies_only_necessary_files(files_to_upgrade: List[Path], study_path: Path, tmp_path: Path) -> List[Path]: + """ + Copies files concerned by the version upgrader into a temporary directory. + Args: + study_path: Path to the study. + tmp_path: Path to the temporary directory where the file modification will be performed. + files_to_upgrade: List[Path]: List of the files and folders concerned by the upgrade. + Returns: + The list of files and folders that were really copied. It's the same as files_to_upgrade but + without any children that has parents already in the list. + """ + files_to_upgrade.append(Path("study.antares")) + files_to_retrieve = [] + for path in files_to_upgrade: + entire_path = study_path / path + if entire_path.is_dir(): + if not (tmp_path / path).exists(): + shutil.copytree(entire_path, tmp_path / path, dirs_exist_ok=True) + files_to_retrieve.append(path) + elif len(path.parts) == 1: + shutil.copy(entire_path, tmp_path / path) + files_to_retrieve.append(path) + else: + parent_path = path.parent + (tmp_path / parent_path).mkdir(parents=True) + shutil.copy(entire_path, tmp_path / parent_path) + files_to_retrieve.append(path) + return files_to_retrieve + + +def _replace_safely_original_files(files_to_replace: List[Path], study_path: Path, tmp_path: Path) -> None: + """ + Replace files/folders of the study that should be upgraded by their copy already upgraded in the tmp directory. + It uses Path.rename() and an intermediary tmp directory to swap the folders safely. + In the end, all tmp directories are removed. + Args: + study_path: Path to the study. + tmp_path: Path to the temporary directory where the file modification will be performed. + files_to_replace: List[Path]: List of files and folders that were really copied + (cf. _copies_only_necessary_files's doc just above) + """ + for k, path in enumerate(files_to_replace): + backup_dir = Path( + tempfile.mkdtemp( + suffix=f".backup_{k}.tmp", + prefix="~", + dir=study_path.parent, + ) + ) + backup_dir.rmdir() + original_path = study_path / path + original_path.rename(backup_dir) + (tmp_path / path).rename(original_path) + shutil.rmtree(backup_dir, ignore_errors=True) + + def _do_upgrade(study_path: Path, src_version: str, target_version: str) -> None: _update_study_antares_file(target_version, study_path) curr_version = src_version - for old, new, method in UPGRADE_METHODS: + for old, new, method, _ in UPGRADE_METHODS: if curr_version == old and curr_version != target_version: method(study_path) curr_version = new From 0923c5e990521e1a4543ec7e07ea4cab51d46162 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:57:41 +0200 Subject: [PATCH 4/5] fix(model): error 500 in cluster list when area change --- .../Modelization/Areas/common/ClusterRoot/index.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx index 160020b096..c8111a2f25 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx @@ -19,6 +19,7 @@ import { AxiosError } from "axios"; import { useSnackbar } from "notistack"; import { FieldValues } from "react-hook-form"; import { Add } from "@mui/icons-material"; +import { usePrevious } from "react-use"; import { Header, ListContainer, @@ -142,6 +143,10 @@ function ClusterRoot(props: ClusterRootProps) { setCurrentCluster(undefined); }, [currentArea]); + // Little fix, but the component must be rewritten... + const prevCurrentArea = usePrevious(currentArea); + const isAreaChanged = prevCurrentArea !== currentArea; + //////////////////////////////////////////////////////////////// // Event Handlers //////////////////////////////////////////////////////////////// @@ -184,7 +189,7 @@ function ClusterRoot(props: ClusterRootProps) { // JSX //////////////////////////////////////////////////////////////// - return currentCluster === undefined ? ( + return currentCluster === undefined || isAreaChanged ? (