diff --git a/antarest/__init__.py b/antarest/__init__.py
index 447ab7edd4..8a0831bff2 100644
--- a/antarest/__init__.py
+++ b/antarest/__init__.py
@@ -7,9 +7,9 @@
# Standard project metadata
-__version__ = "2.15.2"
+__version__ = "2.15.3"
__author__ = "RTE, Antares Web Team"
-__date__ = "2023-10-11"
+__date__ = "2023-10-12"
# noinspection SpellCheckingInspection
__credits__ = "(c) Réseau de Transport de l’Électricité (RTE)"
diff --git a/antarest/study/storage/study_upgrader/__init__.py b/antarest/study/storage/study_upgrader/__init__.py
index 6fbfcfa29b..0886f3e04e 100644
--- a/antarest/study/storage/study_upgrader/__init__.py
+++ b/antarest/study/storage/study_upgrader/__init__.py
@@ -6,7 +6,7 @@
from http import HTTPStatus
from http.client import HTTPException
from pathlib import Path
-from typing import Callable, NamedTuple
+from typing import Callable, List, NamedTuple
from antarest.core.exceptions import StudyValidationError
@@ -29,18 +29,34 @@ class UpgradeMethod(NamedTuple):
old: str
new: str
method: Callable[[Path], None]
+ files: List[Path]
UPGRADE_METHODS = [
- UpgradeMethod("700", "710", upgrade_710),
- UpgradeMethod("710", "720", upgrade_720),
- UpgradeMethod("720", "800", upgrade_800),
- UpgradeMethod("800", "810", upgrade_810),
- UpgradeMethod("810", "820", upgrade_820),
- UpgradeMethod("820", "830", upgrade_830),
- UpgradeMethod("830", "840", upgrade_840),
- UpgradeMethod("840", "850", upgrade_850),
- UpgradeMethod("850", "860", upgrade_860),
+ UpgradeMethod("700", "710", upgrade_710, [Path("settings/generaldata.ini")]),
+ UpgradeMethod("710", "720", upgrade_720, []),
+ UpgradeMethod("720", "800", upgrade_800, [Path("settings/generaldata.ini")]),
+ UpgradeMethod(
+ "800",
+ "810",
+ upgrade_810,
+ [Path("settings/generaldata.ini"), Path("input")],
+ ),
+ UpgradeMethod("810", "820", upgrade_820, [Path("input/links")]),
+ UpgradeMethod(
+ "820",
+ "830",
+ upgrade_830,
+ [Path("settings/generaldata.ini"), Path("input/areas")],
+ ),
+ UpgradeMethod("830", "840", upgrade_840, [Path("settings/generaldata.ini")]),
+ UpgradeMethod("840", "850", upgrade_850, [Path("settings/generaldata.ini")]),
+ UpgradeMethod(
+ "850",
+ "860",
+ upgrade_860,
+ [Path("input"), Path("settings/generaldata.ini")],
+ ),
]
@@ -68,10 +84,10 @@ def find_next_version(from_version: str) -> str:
def upgrade_study(study_path: Path, target_version: str) -> None:
tmp_dir = Path(tempfile.mkdtemp(suffix=".upgrade.tmp", prefix="~", dir=study_path.parent))
- shutil.copytree(study_path, tmp_dir, dirs_exist_ok=True)
try:
- src_version = get_current_version(tmp_dir)
- can_upgrade_version(src_version, target_version)
+ src_version = get_current_version(study_path)
+ files_to_upgrade = can_upgrade_version(src_version, target_version)
+ files_to_retrieve = _copies_only_necessary_files(files_to_upgrade, study_path, tmp_dir)
_do_upgrade(tmp_dir, src_version, target_version)
except (StudyValidationError, InvalidUpgrade) as e:
shutil.rmtree(tmp_dir)
@@ -82,11 +98,7 @@ def upgrade_study(study_path: Path, target_version: str) -> None:
logger.error(f"Unhandled exception : {e}", exc_info=True)
raise
else:
- backup_dir = Path(tempfile.mkdtemp(suffix=".backup.tmp", prefix="~", dir=study_path.parent))
- backup_dir.rmdir()
- study_path.rename(backup_dir)
- tmp_dir.rename(study_path)
- shutil.rmtree(backup_dir, ignore_errors=True)
+ _replace_safely_original_files(files_to_retrieve, study_path, tmp_dir)
def get_current_version(study_path: Path) -> str:
@@ -116,7 +128,7 @@ def get_current_version(study_path: Path) -> str:
)
-def can_upgrade_version(from_version: str, to_version: str) -> None:
+def can_upgrade_version(from_version: str, to_version: str) -> List[Path]:
"""
Checks if upgrading from one version to another is possible.
@@ -124,9 +136,13 @@ def can_upgrade_version(from_version: str, to_version: str) -> None:
from_version: The current version of the study.
to_version: The target version of the study.
+ Returns:
+ If the upgrade is possible, the list of concerned folders and files
+
Raises:
InvalidUpgrade: If the upgrade is not possible.
"""
+ list_versions = []
if from_version == to_version:
raise InvalidUpgrade(f"Your study is already in version '{to_version}'")
@@ -138,12 +154,16 @@ def can_upgrade_version(from_version: str, to_version: str) -> None:
if to_version not in targets:
raise InvalidUpgrade(f"Version '{to_version}' unknown: possible versions are {', '.join(targets)}")
+ files = [u.files for u in UPGRADE_METHODS]
curr_version = from_version
- for src, dst in zip(sources, targets):
+ for src, dst, file in zip(sources, targets, files):
if curr_version == src:
+ for path in file:
+ if path not in list_versions:
+ list_versions.append(path)
curr_version = dst
if curr_version == to_version:
- return
+ return list_versions
# This code must be unreachable!
raise InvalidUpgrade(
@@ -171,10 +191,66 @@ def _update_study_antares_file(target_version: str, study_path: Path) -> None:
file.write_text(content, encoding="utf-8")
+def _copies_only_necessary_files(files_to_upgrade: List[Path], study_path: Path, tmp_path: Path) -> List[Path]:
+ """
+ Copies files concerned by the version upgrader into a temporary directory.
+ Args:
+ study_path: Path to the study.
+ tmp_path: Path to the temporary directory where the file modification will be performed.
+ files_to_upgrade: List[Path]: List of the files and folders concerned by the upgrade.
+ Returns:
+ The list of files and folders that were really copied. It's the same as files_to_upgrade but
+ without any children that has parents already in the list.
+ """
+ files_to_upgrade.append(Path("study.antares"))
+ files_to_retrieve = []
+ for path in files_to_upgrade:
+ entire_path = study_path / path
+ if entire_path.is_dir():
+ if not (tmp_path / path).exists():
+ shutil.copytree(entire_path, tmp_path / path, dirs_exist_ok=True)
+ files_to_retrieve.append(path)
+ elif len(path.parts) == 1:
+ shutil.copy(entire_path, tmp_path / path)
+ files_to_retrieve.append(path)
+ else:
+ parent_path = path.parent
+ (tmp_path / parent_path).mkdir(parents=True)
+ shutil.copy(entire_path, tmp_path / parent_path)
+ files_to_retrieve.append(path)
+ return files_to_retrieve
+
+
+def _replace_safely_original_files(files_to_replace: List[Path], study_path: Path, tmp_path: Path) -> None:
+ """
+ Replace files/folders of the study that should be upgraded by their copy already upgraded in the tmp directory.
+ It uses Path.rename() and an intermediary tmp directory to swap the folders safely.
+ In the end, all tmp directories are removed.
+ Args:
+ study_path: Path to the study.
+ tmp_path: Path to the temporary directory where the file modification will be performed.
+ files_to_replace: List[Path]: List of files and folders that were really copied
+ (cf. _copies_only_necessary_files's doc just above)
+ """
+ for k, path in enumerate(files_to_replace):
+ backup_dir = Path(
+ tempfile.mkdtemp(
+ suffix=f".backup_{k}.tmp",
+ prefix="~",
+ dir=study_path.parent,
+ )
+ )
+ backup_dir.rmdir()
+ original_path = study_path / path
+ original_path.rename(backup_dir)
+ (tmp_path / path).rename(original_path)
+ shutil.rmtree(backup_dir, ignore_errors=True)
+
+
def _do_upgrade(study_path: Path, src_version: str, target_version: str) -> None:
_update_study_antares_file(target_version, study_path)
curr_version = src_version
- for old, new, method in UPGRADE_METHODS:
+ for old, new, method, _ in UPGRADE_METHODS:
if curr_version == old and curr_version != target_version:
method(study_path)
curr_version = new
diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py
index 1105a5f74a..21d19620e2 100644
--- a/antarest/study/web/raw_studies_blueprint.py
+++ b/antarest/study/web/raw_studies_blueprint.py
@@ -96,11 +96,16 @@ def get_study(
extra={"user": current_user.id},
)
parameters = RequestParameters(user=current_user)
- output = study_service.get(uuid, path, depth, formatted, parameters)
+
+ resource_path = pathlib.PurePosixPath(path)
+ output = study_service.get(uuid, str(resource_path), depth=depth, formatted=formatted, params=parameters)
if isinstance(output, bytes):
- resource_path = pathlib.Path(path)
- suffix = resource_path.suffix.lower()
+ # Guess the suffix form the target data
+ parent_cfg = study_service.get(uuid, str(resource_path.parent), depth=2, formatted=True, params=parameters)
+ child = parent_cfg[resource_path.name]
+ suffix = pathlib.PurePosixPath(child).suffix
+
content_type, encoding = CONTENT_TYPES.get(suffix, (None, None))
if content_type == "application/json":
# Use `JSONResponse` to ensure to return a valid JSON response
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 4fc768bc04..1df1b0b313 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,21 @@
Antares Web Changelog
=====================
+v2.15.3 (2023-10-12)
+--------------------
+
+### Hotfix
+
+* **api-raw:** correct the `/studies/{uuid}/raw` endpoint to return "text/plain" content for matrices [`#1766`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1766)
+* **model:** error 500 in cluster list when area change [`0923c5e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/0923c5e990521e1a4543ec7e07ea4cab51d46162)
+
+
+### Performance
+
+* **storage:** make study_upgrader much faster [`#1533`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1533)
+
+
+
v2.15.2 (2023-10-11)
--------------------
@@ -33,6 +48,7 @@ v2.15.2 (2023-10-11)
MartinBelthle
+
v2.15.1 (2023-10-05)
--------------------
diff --git a/setup.py b/setup.py
index d0b8c5deb3..1fc961fedf 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
setup(
name="AntaREST",
- version="2.15.2",
+ version="2.15.3",
description="Antares Server",
long_description=Path("README.md").read_text(encoding="utf-8"),
long_description_content_type="text/markdown",
diff --git a/sonar-project.properties b/sonar-project.properties
index 1d3a132a04..8742aa5a83 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py
sonar.python.coverage.reportPaths=coverage.xml
sonar.python.version=3.8
sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info
-sonar.projectVersion=2.15.2
+sonar.projectVersion=2.15.3
sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/*
\ No newline at end of file
diff --git a/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py b/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py
index 7b5bc4e38e..04a6b3fbfc 100644
--- a/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py
+++ b/tests/integration/raw_studies_blueprint/test_fetch_raw_data.py
@@ -2,7 +2,7 @@
import json
import pathlib
import shutil
-from urllib.parse import urlencode
+from unittest.mock import ANY
import numpy as np
import pytest
@@ -54,9 +54,9 @@ def test_get_study(
user_folder_dir = study_dir.joinpath("user/folder")
for file_path in user_folder_dir.glob("*.*"):
rel_path = file_path.relative_to(study_dir).as_posix()
- query_string = urlencode({"path": f"/{rel_path}", "depth": 1})
res = client.get(
- f"/v1/studies/{study_id}/raw?{query_string}",
+ f"/v1/studies/{study_id}/raw",
+ params={"path": f"/{rel_path}", "depth": 1},
headers=headers,
)
res.raise_for_status()
@@ -80,9 +80,9 @@ def test_get_study(
user_folder_dir = study_dir.joinpath("user/unknown")
for file_path in user_folder_dir.glob("*.*"):
rel_path = file_path.relative_to(study_dir)
- query_string = urlencode({"path": f"/{rel_path.as_posix()}", "depth": 1})
res = client.get(
- f"/v1/studies/{study_id}/raw?{query_string}",
+ f"/v1/studies/{study_id}/raw",
+ params={"path": f"/{rel_path.as_posix()}", "depth": 1},
headers=headers,
)
res.raise_for_status()
@@ -90,22 +90,71 @@ def test_get_study(
expected = file_path.read_bytes()
assert actual == expected
+ # If we ask for properties, we should have a JSON content
+ rel_path = "/input/links/de/properties/fr"
+ res = client.get(
+ f"/v1/studies/{study_id}/raw",
+ params={"path": f"/{rel_path}", "depth": 2},
+ headers=headers,
+ )
+ res.raise_for_status()
+ actual = res.json()
+ assert actual == {
+ "asset-type": "ac",
+ "colorb": 112,
+ "colorg": 112,
+ "colorr": 112,
+ "display-comments": True,
+ "filter-synthesis": "",
+ "filter-year-by-year": "hourly",
+ "hurdles-cost": True,
+ "link-style": "plain",
+ "link-width": 1,
+ "loop-flow": False,
+ "transmission-capacities": "enabled",
+ "use-phase-shifter": False,
+ }
+
+ # If we ask for a matrix, we should have a JSON content if formatted is True
+ rel_path = "/input/links/de/fr"
+ res = client.get(
+ f"/v1/studies/{study_id}/raw",
+ params={"path": f"/{rel_path}", "formatted": True},
+ headers=headers,
+ )
+ res.raise_for_status()
+ actual = res.json()
+ assert actual == {"index": ANY, "columns": ANY, "data": ANY}
+
+ # If we ask for a matrix, we should have a CSV content if formatted is False
+ rel_path = "/input/links/de/fr"
+ res = client.get(
+ f"/v1/studies/{study_id}/raw",
+ params={"path": f"/{rel_path}", "formatted": False},
+ headers=headers,
+ )
+ res.raise_for_status()
+ actual = res.text
+ actual_lines = actual.splitlines()
+ first_row = [float(x) for x in actual_lines[0].split("\t")]
+ assert first_row == [100000, 100000, 0.010000, 0.010000, 0, 0, 0, 0]
+
# Some files can be corrupted
user_folder_dir = study_dir.joinpath("user/bad")
for file_path in user_folder_dir.glob("*.*"):
rel_path = file_path.relative_to(study_dir)
- query_string = urlencode({"path": f"/{rel_path.as_posix()}", "depth": 1})
res = client.get(
- f"/v1/studies/{study_id}/raw?{query_string}",
+ f"/v1/studies/{study_id}/raw",
+ params={"path": f"/{rel_path.as_posix()}", "depth": 1},
headers=headers,
)
assert res.status_code == http.HTTPStatus.UNPROCESSABLE_ENTITY
# We can access to the configuration the classic way,
# for instance, we can get the list of areas:
- query_string = urlencode({"path": "/input/areas/list", "depth": 1})
res = client.get(
- f"/v1/studies/{study_id}/raw?{query_string}",
+ f"/v1/studies/{study_id}/raw",
+ params={"path": "/input/areas/list", "depth": 1},
headers=headers,
)
res.raise_for_status()
diff --git a/tests/storage/web/test_studies_bp.py b/tests/storage/web/test_studies_bp.py
index 91fde089c1..05366331ca 100644
--- a/tests/storage/web/test_studies_bp.py
+++ b/tests/storage/web/test_studies_bp.py
@@ -72,7 +72,9 @@ def test_server() -> None:
client = TestClient(app)
client.get("/v1/studies/study1/raw?path=settings/general/params")
- mock_service.get.assert_called_once_with("study1", "settings/general/params", 3, True, PARAMS)
+ mock_service.get.assert_called_once_with(
+ "study1", "settings/general/params", depth=3, formatted=True, params=PARAMS
+ )
@pytest.mark.unit_test
@@ -121,7 +123,7 @@ def test_server_with_parameters() -> None:
parameters = RequestParameters(user=ADMIN)
assert result.status_code == HTTPStatus.OK
- mock_storage_service.get.assert_called_once_with("study1", "/", 4, True, parameters)
+ mock_storage_service.get.assert_called_once_with("study1", "/", depth=4, formatted=True, params=parameters)
result = client.get("/v1/studies/study2/raw?depth=WRONG_TYPE")
assert result.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
@@ -130,7 +132,7 @@ def test_server_with_parameters() -> None:
assert result.status_code == HTTPStatus.OK
excepted_parameters = RequestParameters(user=ADMIN)
- mock_storage_service.get.assert_called_with("study2", "/", 3, True, excepted_parameters)
+ mock_storage_service.get.assert_called_with("study2", "/", depth=3, formatted=True, params=excepted_parameters)
@pytest.mark.unit_test
diff --git a/webapp/package-lock.json b/webapp/package-lock.json
index 3ab3cca0b7..e179a3f1ca 100644
--- a/webapp/package-lock.json
+++ b/webapp/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "antares-web",
- "version": "2.15.2",
+ "version": "2.15.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
diff --git a/webapp/package.json b/webapp/package.json
index 506c324bf2..77204beb44 100644
--- a/webapp/package.json
+++ b/webapp/package.json
@@ -1,6 +1,6 @@
{
"name": "antares-web",
- "version": "2.15.2",
+ "version": "2.15.3",
"private": true,
"engines": {
"node": "18.16.1"
diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx
index 160020b096..c8111a2f25 100644
--- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx
+++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/common/ClusterRoot/index.tsx
@@ -19,6 +19,7 @@ import { AxiosError } from "axios";
import { useSnackbar } from "notistack";
import { FieldValues } from "react-hook-form";
import { Add } from "@mui/icons-material";
+import { usePrevious } from "react-use";
import {
Header,
ListContainer,
@@ -142,6 +143,10 @@ function ClusterRoot(props: ClusterRootProps) {
setCurrentCluster(undefined);
}, [currentArea]);
+ // Little fix, but the component must be rewritten...
+ const prevCurrentArea = usePrevious(currentArea);
+ const isAreaChanged = prevCurrentArea !== currentArea;
+
////////////////////////////////////////////////////////////////
// Event Handlers
////////////////////////////////////////////////////////////////
@@ -184,7 +189,7 @@ function ClusterRoot(props: ClusterRootProps) {
// JSX
////////////////////////////////////////////////////////////////
- return currentCluster === undefined ? (
+ return currentCluster === undefined || isAreaChanged ? (