fix(outputs): build outputs config even when using cache (#1958)

ANT-1314

antarest/study/storage/rawstudy/model/filesystem/config/files.py

@@ -74,7 +74,7 @@ def build(study_path: Path, study_id: str, output_path: t.Optional[Path] = None)
         version=_parse_version(study_path),
         areas=_parse_areas(study_path),
         sets=_parse_sets(study_path),
-        outputs=_parse_outputs(outputs_dir),
+        outputs=parse_outputs(outputs_dir),
         bindings=_parse_bindings(study_path),
         store_new_set=sns,
         archive_input_series=asi,
@@ -232,7 +232,7 @@ def _parse_areas(root: Path) -> t.Dict[str, Area]:
     return {transform_name_to_id(a): parse_area(root, a) for a in areas}
 
 
-def _parse_outputs(output_path: Path) -> t.Dict[str, Simulation]:
+def parse_outputs(output_path: Path) -> t.Dict[str, Simulation]:
     if not output_path.is_dir():
         return {}
     sims = {}

antarest/study/storage/rawstudy/model/filesystem/factory.py

@@ -10,7 +10,7 @@
 from antarest.core.interfaces.cache import CacheConstants, ICache
 from antarest.matrixstore.service import ISimpleMatrixService
 from antarest.matrixstore.uri_resolver_service import UriResolverService
-from antarest.study.storage.rawstudy.model.filesystem.config.files import build
+from antarest.study.storage.rawstudy.model.filesystem.config.files import build, parse_outputs
 from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO
 from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
 from antarest.study.storage.rawstudy.model.filesystem.root.filestudytree import FileStudyTree
@@ -93,6 +93,9 @@ def _create_from_fs_unsafe(
             if from_cache is not None:
                 logger.info(f"Study {study_id} read from cache")
                 config = FileStudyTreeConfigDTO.parse_obj(from_cache).to_build_config()
+                if output_path:
+                    config.output_path = output_path
+                    config.outputs = parse_outputs(output_path)
                 return FileStudy(config, FileStudyTree(self.context, config))
         start_time = time.time()
         config = build(path, study_id, output_path)

tests/integration/variant_blueprint/test_variant_manager.py

@@ -1,10 +1,13 @@
+import io
 import logging
+import time
 import typing as t
 
 import pytest
 from starlette.testclient import TestClient
 
 from antarest.core.tasks.model import TaskDTO, TaskStatus
+from tests.integration.assets import ASSETS_DIR
 
 
 @pytest.fixture(name="base_study_id")
@@ -240,14 +243,77 @@ def test_comments(client: TestClient, admin_access_token: str, variant_id: str)
     assert res.json() == comment
 
 
-def test_recursive_variant_tree(client: TestClient, admin_access_token: str):
+def test_recursive_variant_tree(client: TestClient, admin_access_token: str, base_study_id: str) -> None:
     admin_headers = {"Authorization": f"Bearer {admin_access_token}"}
-    base_study_res = client.post("/v1/studies?name=foo", headers=admin_headers)
-    base_study_id = base_study_res.json()
-    parent_id = base_study_res.json()
-    for k in range(150):
-        res = client.post(f"/v1/studies/{base_study_id}/variants?name=variant_{k}", headers=admin_headers)
+    parent_id = base_study_id
+    for k in range(200):
+        res = client.post(
+            f"/v1/studies/{base_study_id}/variants",
+            headers=admin_headers,
+            params={"name": f"variant_{k}"},
+        )
         base_study_id = res.json()
+
     # Asserts that we do not trigger a Recursive Exception
     res = client.get(f"/v1/studies/{parent_id}/variants", headers=admin_headers)
-    assert res.status_code == 200
+    assert res.status_code == 200, res.json()
+
+
+def test_outputs(client: TestClient, admin_access_token: str, variant_id: str, tmp_path: str) -> None:
+    # =======================
+    #  SET UP
+    # =======================
+
+    admin_headers = {"Authorization": f"Bearer {admin_access_token}"}
+
+    # Only done to generate the variant folder
+    res = client.post(f"/v1/launcher/run/{variant_id}", headers=admin_headers)
+    res.raise_for_status()
+    job_id = res.json()["job_id"]
+
+    status = client.get(f"/v1/launcher/jobs/{job_id}", headers=admin_headers).json()["status"]
+    while status != "failed":
+        time.sleep(0.2)
+        status = client.get(f"/v1/launcher/jobs/{job_id}", headers=admin_headers).json()["status"]
+
+    # Import an output to the study folder
+    output_path_zip = ASSETS_DIR / "output_adq.zip"
+    res = client.post(
+        f"/v1/studies/{variant_id}/output",
+        headers=admin_headers,
+        files={"output": io.BytesIO(output_path_zip.read_bytes())},
+    )
+    res.raise_for_status()
+
+    # =======================
+    #  ASSERTS GENERATING THE VARIANT DOES NOT `HIDE` OUTPUTS FROM THE ENDPOINT
+    # =======================
+
+    # Get output
+    res = client.get(f"/v1/studies/{variant_id}/outputs", headers=admin_headers)
+    assert res.status_code == 200, res.json()
+    outputs = res.json()
+    assert len(outputs) == 1
+
+    # Generates the study
+    res = client.put(
+        f"/v1/studies/{variant_id}/generate",
+        headers=admin_headers,
+        params={"denormalize": False, "from_scratch": True},
+    )
+    res.raise_for_status()
+    task_id = res.json()
+
+    # Wait for task completion
+    res = client.get(f"/v1/tasks/{task_id}", headers=admin_headers, params={"wait_for_completion": True})
+    res.raise_for_status()
+    task_result = TaskDTO.parse_obj(res.json())
+    assert task_result.status == TaskStatus.COMPLETED
+    assert task_result.result is not None
+    assert task_result.result.success
+
+    # Get outputs again
+    res = client.get(f"/v1/studies/{variant_id}/outputs", headers=admin_headers)
+    assert res.status_code == 200, res.json()
+    outputs = res.json()
+    assert len(outputs) == 1

tests/storage/repository/filesystem/config/test_config_files.py

@@ -11,12 +11,12 @@
 )
 from antarest.study.storage.rawstudy.model.filesystem.config.files import (
     _parse_links,
-    _parse_outputs,
     _parse_renewables,
     _parse_sets,
     _parse_st_storage,
     _parse_thermal,
     build,
+    parse_outputs,
 )
 from antarest.study.storage.rawstudy.model.filesystem.config.model import (
     Area,
@@ -227,7 +227,7 @@ def test_parse_outputs__nominal(tmp_path: Path, assets_name: str, expected: Dict
     with ZipFile(pkg_dir) as zf:
         zf.extractall(tmp_path)
     output_path = tmp_path.joinpath("output")
-    actual = _parse_outputs(output_path)
+    actual = parse_outputs(output_path)
     assert actual == expected
 
 

tests/variantstudy/conftest.py

@@ -81,7 +81,11 @@ def get_matrix_id(matrix: t.Union[t.List[t.List[float]], str]) -> str:
         Get the matrix ID from a matrix or a matrix link.
         """
         if isinstance(matrix, str):
-            return matrix.lstrip("matrix://")
+            # str.removeprefix() is not available in Python 3.8
+            prefix = "matrix://"
+            if matrix.startswith(prefix):
+                return matrix[len(prefix) :]
+            return matrix
         elif isinstance(matrix, list):
             return create(matrix)
         else:

tests/variantstudy/test_command_factory.py

@@ -403,7 +403,11 @@ def setup_class(self):
     @pytest.mark.unit_test
     def test_command_factory(self, command_dto: CommandDTO):
         def get_matrix_id(matrix: str) -> str:
-            return matrix.lstrip("matrix://")
+            # str.removeprefix() is not available in Python 3.8
+            prefix = "matrix://"
+            if matrix.startswith(prefix):
+                return matrix[len(prefix) :]
+            return matrix
 
         command_factory = CommandFactory(
             generator_matrix_constants=Mock(spec=GeneratorMatrixConstants),