feature(variants): allow to increase cleaning snapshot frequency (#2173)

Add the option `snapshot_retention_days` in configuration to allow for
shorter retention period for snapshots.

[ANT-1952] ticket.

antarest/core/config.py

@@ -155,6 +155,7 @@ class StorageConfig:
     auto_archive_dry_run: bool = False
     auto_archive_sleeping_time: int = 3600
     auto_archive_max_parallel: int = 5
+    snapshot_retention_days: int = 1
 
     @classmethod
     def from_dict(cls, data: JSON) -> "StorageConfig":
@@ -184,6 +185,10 @@ def from_dict(cls, data: JSON) -> "StorageConfig":
             auto_archive_dry_run=data.get("auto_archive_dry_run", defaults.auto_archive_dry_run),
             auto_archive_sleeping_time=data.get("auto_archive_sleeping_time", defaults.auto_archive_sleeping_time),
             auto_archive_max_parallel=data.get("auto_archive_max_parallel", defaults.auto_archive_max_parallel),
+            snapshot_retention_days=data.get(
+                "snapshot_retention_days",
+                defaults.snapshot_retention_days,
+            ),
         )
 
 

antarest/study/storage/auto_archive_service.py

@@ -38,6 +38,10 @@ def __init__(self, study_service: StudyService, config: Config):
         self.max_parallel = self.config.storage.auto_archive_max_parallel
 
     def _try_archive_studies(self) -> None:
+        """
+        Archive old studies
+        Clear old variant snapshots
+        """
         old_date = datetime.datetime.utcnow() - datetime.timedelta(days=self.config.storage.auto_archive_threshold_days)
         with db():
             # in this part full `Read` rights over studies are granted to this function
@@ -73,9 +77,6 @@ def _try_archive_studies(self) -> None:
                                 study_id,
                                 params=RequestParameters(DEFAULT_ADMIN_USER),
                             )
-                            self.study_service.storage_service.variant_study_service.clear_snapshot(
-                                self.study_service.get_study(study_id)
-                            )
             except TaskAlreadyRunning:
                 pass
             except Exception as e:
@@ -84,6 +85,10 @@ def _try_archive_studies(self) -> None:
                     exc_info=e,
                 )
 
+        self.study_service.storage_service.variant_study_service.clear_all_snapshots(
+            datetime.timedelta(days=self.config.storage.snapshot_retention_days)
+        )
+
     def _loop(self) -> None:
         while True:
             try:

docs/install/1-CONFIG.md

@@ -304,6 +304,12 @@ default:
 - **Default value:** 5
 - **Description:** Max auto archival tasks in parallel.
 
+## **snapshot_retention_days**
+
+- **Type:** Integer
+- **Default value:** 1
+- **Description:** Snapshots of variant not updated or accessed for **snapshot_retention_days** days will be cleared.
+
 ## **watcher_lock**
 
 - **Type:** Boolean

tests/storage/business/test_autoarchive_service.py

@@ -69,6 +69,10 @@ def test_auto_archival(tmp_path: Path):
                 id="e",
                 updated_at=now - datetime.timedelta(days=61),
             ),
+            VariantStudy(
+                id="f",
+                updated_at=now - datetime.timedelta(days=1),
+            ),
         ]
     )
     db_session.commit()
@@ -86,13 +90,10 @@ def test_auto_archival(tmp_path: Path):
     # Check that the raw study "d" was about to be archived but failed because the task was already running
     study_service.archive.assert_called_once_with("d", params=RequestParameters(DEFAULT_ADMIN_USER))
 
-    # Check that the snapshot of the variant study "e" is cleared
-    study_service.storage_service.variant_study_service.clear_snapshot.assert_called_once()
-    calls = study_service.storage_service.variant_study_service.clear_snapshot.call_args_list
-    assert len(calls) == 1
-    clear_snapshot_call = calls[0]
-    actual_study = clear_snapshot_call[0][0]
-    assert actual_study.id == "e"
-
     # Check that the variant outputs are deleted for the variant study "e"
     study_service.archive_outputs.assert_called_once_with("e", params=RequestParameters(DEFAULT_ADMIN_USER))
+
+    # Check if the `clear_all_snapshots` method was called with default values
+    study_service.storage_service.variant_study_service.clear_all_snapshots.assert_called_once_with(
+        datetime.timedelta(days=1),
+    )