fix(study-factory): ignore non-existent files in archived studies during build

antarest/study/storage/rawstudy/model/filesystem/config/files.py

@@ -1,3 +1,4 @@
+import io
 import json
 import logging
 import re
@@ -8,7 +9,6 @@
 from pathlib import Path
 
 from antarest.core.model import JSON
-from antarest.core.utils.utils import extract_file_to_tmp_dir
 from antarest.study.storage.rawstudy.ini_reader import IniReader
 from antarest.study.storage.rawstudy.model.filesystem.config.binding_constraint import (
     BindingConstraintDTO,
@@ -91,30 +91,61 @@ def _extract_data_from_file(
 ) -> t.Any:
     """
     Extract and process data from various types of files.
+
+    Args:
+        root: Directory or ZIP file containing the study.
+        inside_root_path: Relative path to the file to extract.
+        file_type: Type of the file to extract: text, simple INI or multi INI.
+        multi_ini_keys: List of keys to use for multi INI files.
+
+    Returns:
+        The content of the file, processed according to its type:
+        - TXT: list of lines
+        - SIMPLE_INI or MULTI_INI: dictionary of keys/values
     """
 
-    tmp_dir = None
-    try:
-        if root.suffix.lower() == ".zip":
-            output_data_path, tmp_dir = extract_file_to_tmp_dir(root, inside_root_path)
+    is_zip_file: bool = root.suffix.lower() == ".zip"
+    posix_path: str = inside_root_path.as_posix()
+
+    if file_type == FileType.TXT:
+        # Parse the file as a list of lines, return an empty list if missing.
+        if is_zip_file:
+            with zipfile.ZipFile(root) as zf:
+                try:
+                    with zf.open(posix_path) as f:
+                        text = f.read().decode("utf-8")
+                        return text.splitlines(keepends=False)
+                except KeyError:
+                    # File not found in the ZIP archive
+                    return []
         else:
             output_data_path = root / inside_root_path
+            try:
+                return output_data_path.read_text(encoding="utf-8").splitlines(keepends=False)
+            except FileNotFoundError:
+                return []
+
+    elif file_type in {FileType.MULTI_INI, FileType.SIMPLE_INI}:
+        # Parse the file as a dictionary of keys/values, return an empty dictionary if missing.
+        reader = IniReader(multi_ini_keys)
+        if is_zip_file:
+            with zipfile.ZipFile(root) as zf:
+                try:
+                    with zf.open(posix_path) as f:
+                        buffer = io.StringIO(f.read().decode("utf-8"))
+                        return reader.read(buffer)
+                except KeyError:
+                    # File not found in the ZIP archive
+                    return {}
+        else:
+            output_data_path = root / inside_root_path
+            try:
+                return reader.read(output_data_path)
+            except FileNotFoundError:
+                return {}
 
-        if file_type == FileType.TXT:
-            text = output_data_path.read_text(encoding="utf-8")
-            return text.splitlines(keepends=False)
-        elif file_type == FileType.MULTI_INI:
-            multi_reader = IniReader(multi_ini_keys)
-            return multi_reader.read(output_data_path)
-        elif file_type == FileType.SIMPLE_INI:
-            ini_reader = IniReader()
-            return ini_reader.read(output_data_path)
-        else:  # pragma: no cover
-            raise NotImplementedError(file_type)
-
-    finally:
-        if tmp_dir:
-            tmp_dir.cleanup()
+    else:  # pragma: no cover
+        raise NotImplementedError(file_type)
 
 
 def _parse_version(path: Path) -> int:
@@ -363,7 +394,14 @@ def _parse_renewables(root: Path, area: str) -> t.List[RenewableConfigType]:
     """
     Parse the renewables INI file, return an empty list if missing.
     """
+
+    # Before version 8.1, we only have "Load", "Wind" and "Solar" objects.
+    # We can't use renewable clusters.
     version = _parse_version(root)
+    if version < 810:
+        return []
+
+    # Since version 8.1 of the solver, we can use "renewable clusters" objects.
     relpath = Path(f"input/renewables/clusters/{area}/list.ini")
     config_dict: t.Dict[str, t.Any] = _extract_data_from_file(
         root=root,

tests/integration/variant_blueprint/test_renewable_cluster.py

@@ -4,7 +4,9 @@
 import pytest
 from starlette.testclient import TestClient
 
+from antarest.core.tasks.model import TaskStatus
 from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
+from tests.integration.utils import wait_task_completion
 
 
 # noinspection SpellCheckingInspection
@@ -23,6 +25,22 @@ def test_lifecycle(
     ) -> None:
         # sourcery skip: extract-duplicate-method
 
+        # =======================
+        #  Study version upgrade
+        # =======================
+
+        # We have an "old" study that we need to upgrade to version 810
+        min_study_version = 810
+        res = client.put(
+            f"/v1/studies/{study_id}/upgrade",
+            headers={"Authorization": f"Bearer {user_access_token}"},
+            params={"target_version": min_study_version},
+        )
+        res.raise_for_status()
+        task_id = res.json()
+        task = wait_task_completion(client, user_access_token, task_id)
+        assert task.status == TaskStatus.COMPLETED, task
+
         # =====================
         #  General Data Update
         # =====================

tests/storage/rawstudies/samples/__init__.py

@@ -0,0 +1,3 @@
+from pathlib import Path
+
+ASSETS_DIR = Path(__file__).parent.resolve()

tests/storage/rawstudies/samples/v810/sample1/study.antares

@@ -1,5 +1,5 @@
 [antares]
-version = 800
+version = 810
 caption = renewable-2-clusters-ts-prod-factor
 created = 1618413128
 lastsave = 1625583204

tests/storage/rawstudies/test_factory.py

@@ -1,4 +1,3 @@
-from pathlib import Path
 from unittest.mock import Mock
 
 from antarest.core.interfaces.cache import CacheConstants
@@ -7,10 +6,11 @@
 from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
 from antarest.study.storage.rawstudy.model.filesystem.factory import StudyFactory
 from antarest.study.storage.rawstudy.model.filesystem.root.filestudytree import FileStudyTree
+from tests.storage.rawstudies.samples import ASSETS_DIR
 
 
-def test_renewable_subtree():
-    path = Path(__file__).parent / "samples/v810/sample1"
+def test_renewable_subtree() -> None:
+    path = ASSETS_DIR / "v810/sample1"
     context: ContextServer = Mock(specs=ContextServer)
     config = build(path, "")
     assert config.get_renewable_ids("area") == ["la_rochelle", "oleron"]
@@ -41,8 +41,8 @@ def test_renewable_subtree():
     }
 
 
-def test_factory_cache():
-    path = Path(__file__).parent / "samples/v810/sample1"
+def test_factory_cache() -> None:
+    path = ASSETS_DIR / "v810/sample1"
 
     cache = Mock()
     factory = StudyFactory(matrix=Mock(), resolver=Mock(), cache=cache)

tests/storage/repository/filesystem/config/test_config_files.py

@@ -363,7 +363,7 @@ def test_parse_thermal_860(tmp_path: Path, version, caplog) -> None:
 
 def test_parse_renewables(tmp_path: Path) -> None:
     study_path = build_empty_files(tmp_path)
-    study_path.joinpath("study.antares").write_text("[antares] \n version = 700")
+    study_path.joinpath("study.antares").write_text("[antares] \n version = 810")
     ini_path = study_path.joinpath("input/renewables/clusters/fr/list.ini")
 
     # Error case: `input/renewables/clusters/fr` directory is missing.