v2.15.3

antarest/__init__.py

@@ -7,9 +7,9 @@
 
 # Standard project metadata
 
-__version__ = "2.15.2"
+__version__ = "2.15.3"
 __author__ = "RTE, Antares Web Team"
-__date__ = "2023-10-11"
+__date__ = "2023-10-12"
 # noinspection SpellCheckingInspection
 __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)"
 

antarest/study/storage/study_upgrader/__init__.py

@@ -6,7 +6,7 @@
 from http import HTTPStatus
 from http.client import HTTPException
 from pathlib import Path
-from typing import Callable, NamedTuple
+from typing import Callable, List, NamedTuple
 
 from antarest.core.exceptions import StudyValidationError
 
@@ -29,18 +29,34 @@ class UpgradeMethod(NamedTuple):
     old: str
     new: str
     method: Callable[[Path], None]
+    files: List[Path]
 
 
 UPGRADE_METHODS = [
-    UpgradeMethod("700", "710", upgrade_710),
-    UpgradeMethod("710", "720", upgrade_720),
-    UpgradeMethod("720", "800", upgrade_800),
-    UpgradeMethod("800", "810", upgrade_810),
-    UpgradeMethod("810", "820", upgrade_820),
-    UpgradeMethod("820", "830", upgrade_830),
-    UpgradeMethod("830", "840", upgrade_840),
-    UpgradeMethod("840", "850", upgrade_850),
-    UpgradeMethod("850", "860", upgrade_860),
+    UpgradeMethod("700", "710", upgrade_710, [Path("settings/generaldata.ini")]),
+    UpgradeMethod("710", "720", upgrade_720, []),
+    UpgradeMethod("720", "800", upgrade_800, [Path("settings/generaldata.ini")]),
+    UpgradeMethod(
+        "800",
+        "810",
+        upgrade_810,
+        [Path("settings/generaldata.ini"), Path("input")],
+    ),
+    UpgradeMethod("810", "820", upgrade_820, [Path("input/links")]),
+    UpgradeMethod(
+        "820",
+        "830",
+        upgrade_830,
+        [Path("settings/generaldata.ini"), Path("input/areas")],
+    ),
+    UpgradeMethod("830", "840", upgrade_840, [Path("settings/generaldata.ini")]),
+    UpgradeMethod("840", "850", upgrade_850, [Path("settings/generaldata.ini")]),
+    UpgradeMethod(
+        "850",
+        "860",
+        upgrade_860,
+        [Path("input"), Path("settings/generaldata.ini")],
+    ),
 ]
 
 
@@ -68,10 +84,10 @@ def find_next_version(from_version: str) -> str:
 
 def upgrade_study(study_path: Path, target_version: str) -> None:
     tmp_dir = Path(tempfile.mkdtemp(suffix=".upgrade.tmp", prefix="~", dir=study_path.parent))
-    shutil.copytree(study_path, tmp_dir, dirs_exist_ok=True)
     try:
-        src_version = get_current_version(tmp_dir)
-        can_upgrade_version(src_version, target_version)
+        src_version = get_current_version(study_path)
+        files_to_upgrade = can_upgrade_version(src_version, target_version)
+        files_to_retrieve = _copies_only_necessary_files(files_to_upgrade, study_path, tmp_dir)
         _do_upgrade(tmp_dir, src_version, target_version)
     except (StudyValidationError, InvalidUpgrade) as e:
         shutil.rmtree(tmp_dir)
@@ -82,11 +98,7 @@ def upgrade_study(study_path: Path, target_version: str) -> None:
         logger.error(f"Unhandled exception : {e}", exc_info=True)
         raise
     else:
-        backup_dir = Path(tempfile.mkdtemp(suffix=".backup.tmp", prefix="~", dir=study_path.parent))
-        backup_dir.rmdir()
-        study_path.rename(backup_dir)
-        tmp_dir.rename(study_path)
-        shutil.rmtree(backup_dir, ignore_errors=True)
+        _replace_safely_original_files(files_to_retrieve, study_path, tmp_dir)
 
 
 def get_current_version(study_path: Path) -> str:
@@ -116,17 +128,21 @@ def get_current_version(study_path: Path) -> str:
     )
 
 
-def can_upgrade_version(from_version: str, to_version: str) -> None:
+def can_upgrade_version(from_version: str, to_version: str) -> List[Path]:
     """
     Checks if upgrading from one version to another is possible.
 
     Args:
         from_version: The current version of the study.
         to_version: The target version of the study.
 
+    Returns:
+        If the upgrade is possible, the list of concerned folders and files
+
     Raises:
         InvalidUpgrade: If the upgrade is not possible.
     """
+    list_versions = []
     if from_version == to_version:
         raise InvalidUpgrade(f"Your study is already in version '{to_version}'")
 
@@ -138,12 +154,16 @@ def can_upgrade_version(from_version: str, to_version: str) -> None:
     if to_version not in targets:
         raise InvalidUpgrade(f"Version '{to_version}' unknown: possible versions are {', '.join(targets)}")
 
+    files = [u.files for u in UPGRADE_METHODS]
     curr_version = from_version
-    for src, dst in zip(sources, targets):
+    for src, dst, file in zip(sources, targets, files):
         if curr_version == src:
+            for path in file:
+                if path not in list_versions:
+                    list_versions.append(path)
             curr_version = dst
         if curr_version == to_version:
-            return
+            return list_versions
 
     # This code must be unreachable!
     raise InvalidUpgrade(
@@ -171,10 +191,66 @@ def _update_study_antares_file(target_version: str, study_path: Path) -> None:
     file.write_text(content, encoding="utf-8")
 
 
+def _copies_only_necessary_files(files_to_upgrade: List[Path], study_path: Path, tmp_path: Path) -> List[Path]:
+    """
+    Copies files concerned by the version upgrader into a temporary directory.
+    Args:
+        study_path: Path to the study.
+        tmp_path: Path to the temporary directory where the file modification will be performed.
+        files_to_upgrade: List[Path]: List of the files and folders concerned by the upgrade.
+    Returns:
+        The list of files and folders that were really copied. It's the same as files_to_upgrade but
+        without any children that has parents already in the list.
+    """
+    files_to_upgrade.append(Path("study.antares"))
+    files_to_retrieve = []
+    for path in files_to_upgrade:
+        entire_path = study_path / path
+        if entire_path.is_dir():
+            if not (tmp_path / path).exists():
+                shutil.copytree(entire_path, tmp_path / path, dirs_exist_ok=True)
+                files_to_retrieve.append(path)
+        elif len(path.parts) == 1:
+            shutil.copy(entire_path, tmp_path / path)
+            files_to_retrieve.append(path)
+        else:
+            parent_path = path.parent
+            (tmp_path / parent_path).mkdir(parents=True)
+            shutil.copy(entire_path, tmp_path / parent_path)
+            files_to_retrieve.append(path)
+    return files_to_retrieve
+
+
+def _replace_safely_original_files(files_to_replace: List[Path], study_path: Path, tmp_path: Path) -> None:
+    """
+    Replace files/folders of the study that should be upgraded by their copy already upgraded in the tmp directory.
+    It uses Path.rename() and an intermediary tmp directory to swap the folders safely.
+    In the end, all tmp directories are removed.
+    Args:
+        study_path: Path to the study.
+        tmp_path: Path to the temporary directory where the file modification will be performed.
+        files_to_replace: List[Path]: List of files and folders that were really copied
+        (cf. _copies_only_necessary_files's doc just above)
+    """
+    for k, path in enumerate(files_to_replace):
+        backup_dir = Path(
+            tempfile.mkdtemp(
+                suffix=f".backup_{k}.tmp",
+                prefix="~",
+                dir=study_path.parent,
+            )
+        )
+        backup_dir.rmdir()
+        original_path = study_path / path
+        original_path.rename(backup_dir)
+        (tmp_path / path).rename(original_path)
+        shutil.rmtree(backup_dir, ignore_errors=True)
+
+
 def _do_upgrade(study_path: Path, src_version: str, target_version: str) -> None:
     _update_study_antares_file(target_version, study_path)
     curr_version = src_version
-    for old, new, method in UPGRADE_METHODS:
+    for old, new, method, _ in UPGRADE_METHODS:
         if curr_version == old and curr_version != target_version:
             method(study_path)
             curr_version = new

antarest/study/web/raw_studies_blueprint.py

@@ -96,11 +96,16 @@ def get_study(
             extra={"user": current_user.id},
         )
         parameters = RequestParameters(user=current_user)
-        output = study_service.get(uuid, path, depth, formatted, parameters)
+
+        resource_path = pathlib.PurePosixPath(path)
+        output = study_service.get(uuid, str(resource_path), depth=depth, formatted=formatted, params=parameters)
 
         if isinstance(output, bytes):
-            resource_path = pathlib.Path(path)
-            suffix = resource_path.suffix.lower()
+            # Guess the suffix form the target data
+            parent_cfg = study_service.get(uuid, str(resource_path.parent), depth=2, formatted=True, params=parameters)
+            child = parent_cfg[resource_path.name]
+            suffix = pathlib.PurePosixPath(child).suffix
+
             content_type, encoding = CONTENT_TYPES.get(suffix, (None, None))
             if content_type == "application/json":
                 # Use `JSONResponse` to ensure to return a valid JSON response

docs/CHANGELOG.md

@@ -1,6 +1,21 @@
 Antares Web Changelog
 =====================
 
+v2.15.3 (2023-10-12)
+--------------------
+
+### Hotfix
+
+* **api-raw:** correct the `/studies/{uuid}/raw` endpoint to return "text/plain" content for matrices [`#1766`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1766)
+* **model:** error 500 in cluster list when area change [`0923c5e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/0923c5e990521e1a4543ec7e07ea4cab51d46162)
+
+
+### Performance
+
+* **storage:** make study_upgrader much faster [`#1533`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1533)
+
+
+
 v2.15.2 (2023-10-11)
 --------------------
 
@@ -33,6 +48,7 @@ v2.15.2 (2023-10-11)
 <a href="https://github.com/MartinBelthle">MartinBelthle</a>
 
 
+
 v2.15.1 (2023-10-05)
 --------------------
 

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name="AntaREST",
-    version="2.15.2",
+    version="2.15.3",
     description="Antares Server",
     long_description=Path("README.md").read_text(encoding="utf-8"),
     long_description_content_type="text/markdown",

sonar-project.properties

@@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.python.version=3.8
 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info
-sonar.projectVersion=2.15.2
+sonar.projectVersion=2.15.3
 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/*

tests/integration/raw_studies_blueprint/test_fetch_raw_data.py

@@ -2,7 +2,7 @@
 import json
 import pathlib
 import shutil
-from urllib.parse import urlencode
+from unittest.mock import ANY
 
 import numpy as np
 import pytest
@@ -54,9 +54,9 @@ def test_get_study(
         user_folder_dir = study_dir.joinpath("user/folder")
         for file_path in user_folder_dir.glob("*.*"):
             rel_path = file_path.relative_to(study_dir).as_posix()
-            query_string = urlencode({"path": f"/{rel_path}", "depth": 1})
             res = client.get(
-                f"/v1/studies/{study_id}/raw?{query_string}",
+                f"/v1/studies/{study_id}/raw",
+                params={"path": f"/{rel_path}", "depth": 1},
                 headers=headers,
             )
             res.raise_for_status()
@@ -80,32 +80,81 @@ def test_get_study(
         user_folder_dir = study_dir.joinpath("user/unknown")
         for file_path in user_folder_dir.glob("*.*"):
             rel_path = file_path.relative_to(study_dir)
-            query_string = urlencode({"path": f"/{rel_path.as_posix()}", "depth": 1})
             res = client.get(
-                f"/v1/studies/{study_id}/raw?{query_string}",
+                f"/v1/studies/{study_id}/raw",
+                params={"path": f"/{rel_path.as_posix()}", "depth": 1},
                 headers=headers,
             )
             res.raise_for_status()
             actual = res.content
             expected = file_path.read_bytes()
             assert actual == expected
 
+        # If we ask for properties, we should have a JSON content
+        rel_path = "/input/links/de/properties/fr"
+        res = client.get(
+            f"/v1/studies/{study_id}/raw",
+            params={"path": f"/{rel_path}", "depth": 2},
+            headers=headers,
+        )
+        res.raise_for_status()
+        actual = res.json()
+        assert actual == {
+            "asset-type": "ac",
+            "colorb": 112,
+            "colorg": 112,
+            "colorr": 112,
+            "display-comments": True,
+            "filter-synthesis": "",
+            "filter-year-by-year": "hourly",
+            "hurdles-cost": True,
+            "link-style": "plain",
+            "link-width": 1,
+            "loop-flow": False,
+            "transmission-capacities": "enabled",
+            "use-phase-shifter": False,
+        }
+
+        # If we ask for a matrix, we should have a JSON content if formatted is True
+        rel_path = "/input/links/de/fr"
+        res = client.get(
+            f"/v1/studies/{study_id}/raw",
+            params={"path": f"/{rel_path}", "formatted": True},
+            headers=headers,
+        )
+        res.raise_for_status()
+        actual = res.json()
+        assert actual == {"index": ANY, "columns": ANY, "data": ANY}
+
+        # If we ask for a matrix, we should have a CSV content if formatted is False
+        rel_path = "/input/links/de/fr"
+        res = client.get(
+            f"/v1/studies/{study_id}/raw",
+            params={"path": f"/{rel_path}", "formatted": False},
+            headers=headers,
+        )
+        res.raise_for_status()
+        actual = res.text
+        actual_lines = actual.splitlines()
+        first_row = [float(x) for x in actual_lines[0].split("\t")]
+        assert first_row == [100000, 100000, 0.010000, 0.010000, 0, 0, 0, 0]
+
         # Some files can be corrupted
         user_folder_dir = study_dir.joinpath("user/bad")
         for file_path in user_folder_dir.glob("*.*"):
             rel_path = file_path.relative_to(study_dir)
-            query_string = urlencode({"path": f"/{rel_path.as_posix()}", "depth": 1})
             res = client.get(
-                f"/v1/studies/{study_id}/raw?{query_string}",
+                f"/v1/studies/{study_id}/raw",
+                params={"path": f"/{rel_path.as_posix()}", "depth": 1},
                 headers=headers,
             )
             assert res.status_code == http.HTTPStatus.UNPROCESSABLE_ENTITY
 
         # We can access to the configuration the classic way,
         # for instance, we can get the list of areas:
-        query_string = urlencode({"path": "/input/areas/list", "depth": 1})
         res = client.get(
-            f"/v1/studies/{study_id}/raw?{query_string}",
+            f"/v1/studies/{study_id}/raw",
+            params={"path": "/input/areas/list", "depth": 1},
             headers=headers,
         )
         res.raise_for_status()