diff --git a/antarest/core/tasks/model.py b/antarest/core/tasks/model.py index 138344602a..a7ca9aedb9 100644 --- a/antarest/core/tasks/model.py +++ b/antarest/core/tasks/model.py @@ -37,6 +37,7 @@ class TaskType(str, Enum): SCAN = "SCAN" UPGRADE_STUDY = "UPGRADE_STUDY" THERMAL_CLUSTER_SERIES_GENERATION = "THERMAL_CLUSTER_SERIES_GENERATION" + SNAPSHOT_CLEARING = "SNAPSHOT_CLEARING" class TaskStatus(Enum): diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 1e26a20c8a..8737824696 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -15,7 +15,7 @@ import re import shutil import typing as t -from datetime import datetime +from datetime import datetime, timedelta from functools import reduce from pathlib import Path from uuid import uuid4 @@ -45,9 +45,11 @@ from antarest.core.serialization import to_json_string from antarest.core.tasks.model import CustomTaskEventMessages, TaskDTO, TaskResult, TaskType from antarest.core.tasks.service import DEFAULT_AWAIT_MAX_TIMEOUT, ITaskService, TaskUpdateNotifier, noop_notifier +from antarest.core.utils.fastapi_sqlalchemy import db from antarest.core.utils.utils import assert_this, suppress_exception from antarest.matrixstore.service import MatrixService from antarest.study.model import RawStudy, Study, StudyAdditionalData, StudyMetadataDTO, StudySimResultDTO +from antarest.study.repository import AccessPermissions, StudyFilter from antarest.study.storage.abstract_storage_service import AbstractStorageService from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO @@ -625,9 +627,9 @@ def callback(notifier: TaskUpdateNotifier) -> TaskResult: ) return TaskResult( success=generate_result.success, - message=f"{study_id} generated successfully" - if generate_result.success - else f"{study_id} not generated", + message=( + f"{study_id} generated successfully" if generate_result.success else f"{study_id} not generated" + ), return_value=generate_result.model_dump_json(), ) @@ -1053,3 +1055,67 @@ def initialize_additional_data(self, variant_study: VariantStudy) -> bool: exc_info=e, ) return False + + def clear_all_snapshots(self, retention_hours: timedelta, params: t.Optional[RequestParameters] = None) -> str: + """ + Admin command that clear all variant snapshots older than `retention_hours` (in hours). + Only available for admin users. + + Args: + retention_hours: number of retention hours + params: request parameters used to identify the user status + Returns: None + + Raises: + UserHasNotPermissionError + """ + if params is None or (params.user and not params.user.is_site_admin() and not params.user.is_admin_token()): + raise UserHasNotPermissionError() + + task_name = f"Cleaning all snapshot updated or accessed at least {retention_hours} hours ago." + + snapshot_clearing_task_instance = SnapshotCleanerTask( + variant_study_service=self, retention_hours=retention_hours + ) + + return self.task_service.add_task( + snapshot_clearing_task_instance, + task_name, + task_type=TaskType.SNAPSHOT_CLEARING, + ref_id="SNAPSHOT_CLEANING", + custom_event_messages=None, + request_params=params, + ) + + +class SnapshotCleanerTask: + def __init__( + self, + variant_study_service: VariantStudyService, + retention_hours: timedelta, + ) -> None: + self._variant_study_service = variant_study_service + self._retention_hours = retention_hours + + def _clear_all_snapshots(self) -> None: + with db(): + variant_list = self._variant_study_service.repository.get_all( + study_filter=StudyFilter( + variant=True, + access_permissions=AccessPermissions(is_admin=True), + ) + ) + for variant in variant_list: + if variant.updated_at and variant.updated_at < datetime.utcnow() - self._retention_hours: + if variant.last_access and variant.last_access < datetime.utcnow() - self._retention_hours: + self._variant_study_service.clear_snapshot(variant) + + def run_task(self, notifier: TaskUpdateNotifier) -> TaskResult: + msg = f"Start cleaning all snapshots updated or accessed {self._retention_hours} hours ago." + notifier(msg) + self._clear_all_snapshots() + msg = f"All selected snapshots were successfully cleared." + notifier(msg) + return TaskResult(success=True, message=msg) + + __call__ = run_task diff --git a/antarest/study/web/variant_blueprint.py b/antarest/study/web/variant_blueprint.py index 1c3c092a78..bc000fadf4 100644 --- a/antarest/study/web/variant_blueprint.py +++ b/antarest/study/web/variant_blueprint.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. - +import datetime import logging from typing import List, Optional, Union @@ -416,4 +416,33 @@ def create_from_variant( params = RequestParameters(user=current_user) raise NotImplementedError() + @bp.put( + "/studies/variants/clear-snapshots", + tags=[APITag.study_variant_management], + summary="Clear variant snapshots", + responses={ + 200: { + "description": "Delete snapshots older than a specific number of hours. By default, this number is 24." + } + }, + ) + def clear_variant_snapshots( + hours: int = 24, + current_user: JWTUser = Depends(auth.get_current_user), + ) -> str: + """ + Endpoint that clear `limit` hours old and older variant snapshots. + + Args: limit (int, optional): Number of hours to clear. Defaults to 24. + + Returns: ID of the task running the snapshot clearing. + """ + retention_hours = datetime.timedelta(hours=hours) + logger.info( + f"Delete all variant snapshots older than {retention_hours} hours.", + extra={"user": current_user.id}, + ) + params = RequestParameters(user=current_user) + return variant_study_service.clear_all_snapshots(retention_hours, params) + return bp diff --git a/installer b/installer index e7552a8a66..bd51be05c7 160000 --- a/installer +++ b/installer @@ -1 +1 @@ -Subproject commit e7552a8a66bef72f9e4b8fc6e1b274895f15a180 +Subproject commit bd51be05c7f0f1c634f2cd0c564686ed34b3a6f7 diff --git a/tests/integration/variant_blueprint/test_variant_manager.py b/tests/integration/variant_blueprint/test_variant_manager.py index 435f7565ac..00c75e515c 100644 --- a/tests/integration/variant_blueprint/test_variant_manager.py +++ b/tests/integration/variant_blueprint/test_variant_manager.py @@ -10,16 +10,19 @@ # # This file is part of the Antares project. +import datetime import io import logging import time import typing as t +from pathlib import Path import pytest from starlette.testclient import TestClient from antarest.core.tasks.model import TaskDTO, TaskStatus from tests.integration.assets import ASSETS_DIR +from tests.integration.utils import wait_task_completion @pytest.fixture(name="base_study_id") @@ -45,6 +48,68 @@ def variant_id_fixture( return t.cast(str, res.json()) +@pytest.fixture(name="generate_snapshots") +def generate_snapshot_fixture( + client: TestClient, + admin_access_token: str, + base_study_id: str, + monkeypatch: pytest.MonkeyPatch, + caplog: t.Any, +) -> t.List[str]: + """Generate some snapshots with different date of update and last access""" + + class FakeDatetime: + """ + Class that handle fake timestamp creation/update of variant + """ + + fake_time: datetime.datetime + + @classmethod + def now(cls) -> datetime.datetime: + """Method used to get the custom timestamp""" + return cls.fake_time + + @classmethod + def utcnow(cls) -> datetime.datetime: + """Method used while a variant is created""" + return cls.now() + + # Initialize variant_ids list + variant_ids = [] + + admin_headers = {"Authorization": f"Bearer {admin_access_token}"} + + with caplog.at_level(level=logging.WARNING): + # Generate three different timestamp + older_time = datetime.datetime.utcnow() - datetime.timedelta( + hours=25 + ) # older than the default value which is 24 + old_time = datetime.datetime.utcnow() - datetime.timedelta(hours=8) # older than 6 hours + recent_time = datetime.datetime.utcnow() - datetime.timedelta(hours=2) # older than 0 hours + + with monkeypatch.context() as m: + # Patch the datetime import instance of the variant_study_service package to hack + # the `created_at` and `updated_at` fields + # useful when a variant is created + m.setattr("antarest.study.storage.variantstudy.variant_study_service.datetime", FakeDatetime) + # useful when a study is accessed + m.setattr("antarest.study.service.datetime", FakeDatetime) + + for index, different_time in enumerate([older_time, old_time, recent_time]): + FakeDatetime.fake_time = different_time + res = client.post(f"/v1/studies/{base_study_id}/variants?name=variant{index}", headers=admin_headers) + variant_ids.append(res.json()) + + # Generate snapshot for each variant + task_id = client.put(f"/v1/studies/{variant_ids[index]}/generate", headers=admin_headers) + wait_task_completion( + client, admin_access_token, task_id.json() + ) # wait for the filesystem to be updated + client.get(f"v1/studies/{variant_ids[index]}", headers=admin_headers) + return t.cast(t.List[str], variant_ids) + + def test_variant_manager( client: TestClient, admin_access_token: str, @@ -330,3 +395,44 @@ def test_outputs(client: TestClient, admin_access_token: str, variant_id: str, t assert res.status_code == 200, res.json() outputs = res.json() assert len(outputs) == 1 + + +def test_clear_snapshots( + client: TestClient, + admin_access_token: str, + tmp_path: Path, + generate_snapshots: t.List[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + The `snapshot/` directory must not exist after a call to `clear-snapshot`. + """ + # Set up + admin_headers = {"Authorization": f"Bearer {admin_access_token}"} + + older = Path(tmp_path).joinpath(f"internal_workspace", generate_snapshots[0], "snapshot") + old = Path(tmp_path).joinpath(f"internal_workspace", generate_snapshots[1], "snapshot") + recent = Path(tmp_path).joinpath(f"internal_workspace", generate_snapshots[2], "snapshot") + + # Test + # Check initial data + assert older.exists() and old.exists() and recent.exists() + + # Delete the older snapshot (default retention hours implicitly equals to 24 hours) + # and check if it was successfully deleted + response = client.put(f"v1/studies/variants/clear-snapshots", headers=admin_headers) + task = response.json() + wait_task_completion(client, admin_access_token, task) + assert (not older.exists()) and old.exists() and recent.exists() + + # Delete the old snapshot and check if it was successfully deleted + response = client.put(f"v1/studies/variants/clear-snapshots?hours=6", headers=admin_headers) + task = response.json() + wait_task_completion(client, admin_access_token, task) + assert (not older.exists()) and (not old.exists()) and recent.exists() + + # Delete the recent snapshot and check if it was successfully deleted + response = client.put(f"v1/studies/variants/clear-snapshots?hours=-1", headers=admin_headers) + task = response.json() + wait_task_completion(client, admin_access_token, task) + assert not (older.exists() and old.exists() and recent.exists()) diff --git a/tests/study/storage/variantstudy/test_variant_study_service.py b/tests/study/storage/variantstudy/test_variant_study_service.py index 80a3de302d..cd8f9100c0 100644 --- a/tests/study/storage/variantstudy/test_variant_study_service.py +++ b/tests/study/storage/variantstudy/test_variant_study_service.py @@ -12,16 +12,19 @@ import datetime import re +import typing from pathlib import Path from unittest.mock import Mock import numpy as np import pytest +from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTUser from antarest.core.model import PublicMode -from antarest.core.requests import RequestParameters +from antarest.core.requests import RequestParameters, UserHasNotPermissionError from antarest.core.utils.fastapi_sqlalchemy import db -from antarest.login.model import Group, User +from antarest.core.utils.utils import sanitize_uuid +from antarest.login.model import ADMIN_ID, ADMIN_NAME, Group, User from antarest.matrixstore.service import SimpleMatrixService from antarest.study.business.utils import execute_or_add_commands from antarest.study.model import RawStudy, StudyAdditionalData @@ -239,3 +242,188 @@ def test_generate_task( else: expected = EXPECTED_DENORMALIZED assert res_study_files == expected + + @with_db_context + def test_clear_all_snapshots( + self, + tmp_path: Path, + variant_study_service: VariantStudyService, + raw_study_service: RawStudyService, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """ + - Test return value in case the user is not allowed to call the function, + - Test return value in case the user give a bad argument (negative + integer or other type than integer) + - Test deletion of an old snapshot and a recent one + + In order to test date and time of objects, a FakeDateTime class is defined and used + by a monkeypatch context + """ + + class FakeDatetime: + """ + Class that handle fake timestamp creation/update of variant + """ + + fake_time: datetime.datetime + + @classmethod + def now(cls) -> datetime.datetime: + """Method used to get the custom timestamp""" + return datetime.datetime(2023, 12, 31) + + @classmethod + def utcnow(cls) -> datetime.datetime: + """Method used while a variant is created""" + return cls.now() + + # ============================= + # SET UP + # ============================= + # Create two users + # an admin user + # noinspection PyArgumentList + admin_user = User(id=ADMIN_ID, name=ADMIN_NAME) + db.session.add(admin_user) + db.session.commit() + + regular_user = User(id=99, name="regular") + db.session.add(regular_user) + db.session.commit() + + # noinspection PyArgumentList + group = Group(id="my-group", name="group") + db.session.add(group) + db.session.commit() + + # Create a raw study (root of the variant) + raw_study_path = tmp_path / "My RAW Study" + # noinspection PyArgumentList + raw_study = RawStudy( + id="my_raw_study", + name=raw_study_path.name, + version="860", + author="John Smith", + created_at=datetime.datetime(2023, 7, 15, 16, 45), + updated_at=datetime.datetime(2023, 7, 19, 8, 15), + last_access=datetime.datetime.utcnow(), + public_mode=PublicMode.FULL, + owner=admin_user, + groups=[group], + path=str(raw_study_path), + additional_data=StudyAdditionalData(author="John Smith"), + ) + + db.session.add(raw_study) + db.session.commit() + + # Set up the Raw Study + raw_study_service.create(raw_study) + + # Variant studies + variant_list = [] + + # For each variant created + with monkeypatch.context() as m: + # Set the system date older to create older variants + m.setattr("antarest.study.storage.variantstudy.variant_study_service.datetime", FakeDatetime) + m.setattr("antarest.study.service.datetime", FakeDatetime) + + for index in range(3): + variant_list.append( + variant_study_service.create_variant_study( + raw_study.id, + "Variant{}".format(str(index)), + params=Mock( + spec=RequestParameters, + user=DEFAULT_ADMIN_USER, + ), + ) + ) + + # Generate a snapshot for each variant + variant_study_service.generate( + sanitize_uuid(variant_list[index].id), + False, + False, + params=Mock( + spec=RequestParameters, + user=Mock(spec=JWTUser, id=regular_user.id, impersonator=regular_user.id), + ), + ) + + variant_study_service.get(variant_list[index]) + + variant_study_path = Path(tmp_path).joinpath("internal_studies") + + # Check if everything was correctly initialized + assert len(list(variant_study_path.iterdir())) == 3 + + for variant in variant_study_path.iterdir(): + assert variant.is_dir() + assert list(variant.iterdir())[0].name == "snapshot" + + # ============================= + # TEST + # ============================= + # A user without rights cannot clear snapshots + with pytest.raises(UserHasNotPermissionError): + variant_study_service.clear_all_snapshots( + datetime.timedelta(1), + params=Mock( + spec=RequestParameters, + user=Mock( + spec=JWTUser, + id=regular_user.id, + is_site_admin=Mock(return_value=False), + is_admin_token=Mock(return_value=False), + ), + ), + ) + + # At this point, variants was not accessed yet + # Thus snapshot directories must exist still + for variant in variant_study_path.iterdir(): + assert variant.is_dir() + assert list(variant.iterdir()) + + # Simulate access for two old snapshots + variant_list[0].last_access = datetime.datetime.utcnow() - datetime.timedelta(days=60) + variant_list[1].last_access = datetime.datetime.utcnow() - datetime.timedelta(hours=6) + + # Simulate access for a recent one + variant_list[2].last_access = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + db.session.commit() + + # Clear old snapshots + task_id = variant_study_service.clear_all_snapshots( + datetime.timedelta(hours=5), + Mock( + spec=RequestParameters, + user=DEFAULT_ADMIN_USER, + ), + ) + variant_study_service.task_service.await_task(task_id) + + # Check if old snapshots was successfully cleared + nb_snapshot_dir = 0 # after the for iterations, must equal 1 + for variant_path in variant_study_path.iterdir(): + if variant_path.joinpath("snapshot").exists(): + nb_snapshot_dir += 1 + assert nb_snapshot_dir == 1 + + # Clear most recent snapshots + task_id = variant_study_service.clear_all_snapshots( + datetime.timedelta(hours=-1), + Mock( + spec=RequestParameters, + user=DEFAULT_ADMIN_USER, + ), + ) + variant_study_service.task_service.await_task(task_id) + + # Check if all snapshots were cleared + nb_snapshot_dir = 0 # after the for iterations, must equal 0 + for variant_path in variant_study_path.iterdir(): + assert not variant_path.joinpath("snapshot").exists()