OS-4886. obsolete_snapshot_chains archive

bumiworker/bumiworker/modules/archive/obsolete_snapshot_chains.py

@@ -0,0 +1,46 @@
+from bumiworker.bumiworker.consts import ArchiveReason
+from bumiworker.bumiworker.modules.recommendations.obsolete_snapshot_chains import (
+    ObsoleteSnapshotChains as ObsoleteSnapshotChainsRecommendation,
+    SUPPORTED_CLOUDS,
+)
+from bumiworker.bumiworker.modules.obsolete_snapshots_base import (
+    ObsoleteSnapshotsArchiveBase
+)
+
+
+class ObsoleteSnapshotChains(ObsoleteSnapshotsArchiveBase,
+                             ObsoleteSnapshotChainsRecommendation):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.reason_description_map[ArchiveReason.RECOMMENDATION_APPLIED] = (
+            'snapshot chain deleted')
+
+    @property
+    def supported_cloud_types(self):
+        return SUPPORTED_CLOUDS
+
+    @property
+    def resource_type(self):
+        return 'Snapshot Chain'
+
+    def get_used_resources(self, now, cloud_account_id, cloud_config,
+                           obsolete_threshold):
+        snapshots_to_chains = self._get_snapshots_to_chains(
+            cloud_account_id)
+        snapshots_used_by_images = self.get_snapshots_used_by_images(
+            now, cloud_config)
+        chains_used_by_images = self._group_by_chains(
+            snapshots_to_chains, snapshots_used_by_images)
+        snapshots_used_by_volumes = self.get_snapshots_used_by_volumes(
+            now, cloud_account_id, obsolete_threshold)
+        chains_used_by_volumes = self._group_by_chains(
+            snapshots_to_chains, snapshots_used_by_volumes)
+        chains_using_volumes = self._get_chains_using_volumes(
+            now, cloud_account_id)
+        return (chains_used_by_images, chains_using_volumes,
+                chains_used_by_volumes)
+
+
+def main(organization_id, config_client, created_at, **kwargs):
+    return ObsoleteSnapshotChains(
+        organization_id, config_client, created_at).get()

bumiworker/bumiworker/modules/archive/obsolete_snapshots.py

@@ -1,87 +1,40 @@
-import logging
-
-from collections import defaultdict
-from datetime import datetime, timedelta
-
 from bumiworker.bumiworker.consts import ArchiveReason
-from bumiworker.bumiworker.modules.base import ArchiveBase
+from bumiworker.bumiworker.modules.obsolete_snapshots_base import (
+    ObsoleteSnapshotsArchiveBase
+)
 from bumiworker.bumiworker.modules.recommendations.obsolete_snapshots import (
     ObsoleteSnapshots as ObsoleteSnapshotsRecommendation, SUPPORTED_CLOUDS,
-    AWS_CLOUD)
-
-LOG = logging.getLogger(__name__)
+    AWS_CLOUD
+)
 
 
-class ObsoleteSnapshots(ArchiveBase, ObsoleteSnapshotsRecommendation):
+class ObsoleteSnapshots(ObsoleteSnapshotsArchiveBase,
+                        ObsoleteSnapshotsRecommendation):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.reason_description_map[ArchiveReason.RECOMMENDATION_APPLIED] = (
             'snapshot deleted')
 
+    @property
+    def resource_type(self):
+        return 'Snapshot'
+
     @property
     def supported_cloud_types(self):
         return SUPPORTED_CLOUDS
 
-    def _get_active_snapshots(self, cloud_account_id):
-        return self.mongo_client.restapi.resources.distinct(
-            'cloud_resource_id', {'$and': [
-                {'active': True},
-                {'deleted_at': 0},
-                {'resource_type': 'Snapshot'},
-                {'cloud_account_id': cloud_account_id},
-            ]})
-
-    def _get(self, previous_options, optimizations, cloud_accounts_map,
-             **kwargs):
-        now = datetime.utcnow()
-        days_threshold = previous_options['days_threshold']
-        obsolete_threshold = timedelta(days_threshold)
-
-        account_optimizations_map = defaultdict(list)
-        for optimization in optimizations:
-            account_optimizations_map[optimization['cloud_account_id']].append(
-                optimization)
-
-        result = []
-        for cloud_account_id, optimizations_ in account_optimizations_map.items():
-            if cloud_account_id not in cloud_accounts_map:
-                for optimization in optimizations_:
-                    self._set_reason_properties(
-                        optimization, ArchiveReason.CLOUD_ACCOUNT_DELETED)
-                    result.append(optimization)
-                continue
-
-            cloud_config = cloud_accounts_map[cloud_account_id]
-            cloud_config.update(cloud_config.get('config', {}))
-
-            snapshots_used_by_images = {}
-            if cloud_config.get('type') == AWS_CLOUD:
-                snapshots_used_by_images = self.get_snapshots_used_by_images(
-                    now, cloud_config)
-            snapshots_using_volumes = self._get_snapshots_using_volumes(
-                now, cloud_account_id)
-            snapshots_used_by_volumes = self.get_snapshots_used_by_volumes(
-                now, cloud_account_id, obsolete_threshold)
-            not_deleted_snapshots = set(self._get_active_snapshots(
-                cloud_account_id))
-            for optimization in optimizations_:
-                if optimization['cloud_resource_id'] in snapshots_used_by_images:
-                    self._set_reason_properties(
-                        optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT,
-                        'used by AMI source')
-                elif (optimization['cloud_resource_id'] in snapshots_using_volumes or
-                      optimization['cloud_resource_id'] in snapshots_used_by_volumes):
-                    self._set_reason_properties(
-                        optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT,
-                        'used by volume')
-                elif optimization['cloud_resource_id'] not in not_deleted_snapshots:
-                    self._set_reason_properties(
-                        optimization, ArchiveReason.RECOMMENDATION_APPLIED)
-                else:
-                    self._set_reason_properties(
-                        optimization, ArchiveReason.OPTIONS_CHANGED)
-                result.append(optimization)
-        return result
+    def get_used_resources(self, now, cloud_account_id, cloud_config,
+                           obsolete_threshold):
+        snapshots_used_by_images = {}
+        if cloud_config.get('type') == AWS_CLOUD:
+            snapshots_used_by_images = self.get_snapshots_used_by_images(
+                now, cloud_config)
+        snapshots_using_volumes = self._get_snapshots_using_volumes(
+            now, cloud_account_id)
+        snapshots_used_by_volumes = self.get_snapshots_used_by_volumes(
+            now, cloud_account_id, obsolete_threshold)
+        return (snapshots_used_by_images, snapshots_using_volumes,
+                snapshots_used_by_volumes)
 
 
 def main(organization_id, config_client, created_at, **kwargs):

bumiworker/bumiworker/modules/obsolete_snapshots_base.py

@@ -1,11 +1,13 @@
 from concurrent.futures.thread import ThreadPoolExecutor
+from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from kombu.log import get_logger
 from pymongo import UpdateOne
 
 from tools.cloud_adapter.cloud import Cloud as CloudAdapter
 
-from bumiworker.bumiworker.modules.base import ModuleBase
+from bumiworker.bumiworker.consts import ArchiveReason
+from bumiworker.bumiworker.modules.base import ArchiveBase, ModuleBase
 
 BULK_SIZE = 2000
 DAYS_IN_MONTH = 30
@@ -205,3 +207,70 @@ def _collect_resources(self, configs):
                             'snapshots')
                     })
         return res
+
+
+class ObsoleteSnapshotsArchiveBase(ArchiveBase):
+
+    @property
+    def resource_type(self):
+        return None
+
+    def _get_active_resources(self, cloud_account_id):
+        return self.mongo_client.restapi.resources.distinct(
+            'cloud_resource_id', {'$and': [
+                {'active': True},
+                {'deleted_at': 0},
+                {'resource_type': self.resource_type},
+                {'cloud_account_id': cloud_account_id},
+            ]})
+
+    def get_used_resources(self, now, cloud_account_id, cloud_config,
+                           obsolete_threshold):
+        raise NotImplementedError
+
+    def _get(self, previous_options, optimizations, cloud_accounts_map,
+             **kwargs):
+        now = datetime.utcnow()
+        days_threshold = previous_options['days_threshold']
+        obsolete_threshold = timedelta(days_threshold)
+
+        account_optimizations_map = defaultdict(list)
+        for optimization in optimizations:
+            account_optimizations_map[optimization['cloud_account_id']].append(
+                optimization)
+
+        result = []
+        for cloud_account_id, optimizations_ in account_optimizations_map.items():
+            if cloud_account_id not in cloud_accounts_map:
+                for optimization in optimizations_:
+                    self._set_reason_properties(
+                        optimization, ArchiveReason.CLOUD_ACCOUNT_DELETED)
+                    result.append(optimization)
+                continue
+
+            cloud_config = cloud_accounts_map[cloud_account_id]
+            cloud_config.update(cloud_config.get('config', {}))
+
+            (res_used_by_images, res_using_volumes,
+             res_used_by_volumes) = self.get_used_resources(
+                now, cloud_account_id, cloud_config, obsolete_threshold)
+
+            not_deleted_res = set(self._get_active_resources(cloud_account_id))
+            for optimization in optimizations_:
+                if optimization['cloud_resource_id'] in res_used_by_images:
+                    self._set_reason_properties(
+                        optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT,
+                        'used by AMI source')
+                elif (optimization['cloud_resource_id'] in res_using_volumes or
+                      optimization['cloud_resource_id'] in res_used_by_volumes):
+                    self._set_reason_properties(
+                        optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT,
+                        'used by volume')
+                elif optimization['cloud_resource_id'] not in not_deleted_res:
+                    self._set_reason_properties(
+                        optimization, ArchiveReason.RECOMMENDATION_APPLIED)
+                else:
+                    self._set_reason_properties(
+                        optimization, ArchiveReason.OPTIONS_CHANGED)
+                result.append(optimization)
+        return result