Find and remove image duplicates

src/.gitignore

@@ -13,4 +13,5 @@ deployment/aws/
 test_data/
 celerybeat-schedule.db
 *.rdb*
-venv
+*.venv
+venv

src/_main_/utils/common.py

@@ -121,24 +121,33 @@ def rename_fields(args, pairs):
 
 def serialize_all(data, full=False, **kwargs):
   #medium = (kwargs or {}).get("medium", False)
+  info = (kwargs or {}).get("info", False)
   if not data:
     return []
 
   if isinstance(data[0], dict):
     return data
 
+
   if full:
     return [d.full_json() for d in data]
+  elif info:
+    return [d.info() for d in data]
   #elif medium: 
   #  return [d.medium_json() for d in data]
   return [d.simple_json() for d in data]
 
 
-def serialize(data, full=False):
+def serialize(data, full=False, **kwargs):
+  info = (kwargs or {}).get("info", False)
   if not data:
     return {}
+
   if full:
     return data.full_json()
+  elif info: 
+    return data.info()
+
   return data.simple_json()
 
 def check_length(args, field,  min_length=5, max_length=40):

src/api/constants.py

@@ -13,4 +13,13 @@
 STANDARD_USER = 'standard_user'
 GUEST_USER = 'guest_user'
 INVITED_USER = 'invited_user'  
-WHEN_USER_AUTHENTICATED_SESSION_EXPIRES = "WHEN_USER_AUTHENTICATED_SESSION_EXPIRES"
+WHEN_USER_AUTHENTICATED_SESSION_EXPIRES = "WHEN_USER_AUTHENTICATED_SESSION_EXPIRES"
+
+CSV_FIELD_NAMES = [
+    "media_url",
+    "primary_media_id",
+    "usage_stats",
+    "usage_summary",
+    "ids_of_duplicates",
+    "duplicates",
+]

src/api/handlers/media_library.py

@@ -22,6 +22,68 @@ def registerRoutes(self):
         self.add("/gallery.find", self.find_images)
         self.add("/gallery.item.edit", self.edit_details)
 
+        self.add("/gallery.generate.hashes", self.generate_hashes) # A temporary route that we will need to run to generate hashes of already uploaded content (ONCE!)
+        self.add("/gallery.duplicates.summarize", self.summarize_duplicates) # Generates a CSV of duplicate images with other useful attributes
+        self.add("/gallery.duplicates.clean", self.clean_duplicates) # Allows you to clean all/some duplicates and transfer relationships to only one record
+        self.add("/gallery.duplicates.summary.print", self.print_duplicates) # Allows you to clean all/some duplicates and transfer relationships to only one record
+
+    # @admins_only
+    def print_duplicates(self, request):
+        """ Creates a downloadable file that contains the summary of duplicate media"""
+        context: Context = request.context
+        args: dict = context.args
+        self.validator.expect("type", str) #  Future Enhancement: provide type as 'csv' or 'pdf' or other formats. But currently, only CSV
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        response, error = self.service.print_duplicates(args, context)
+        if error:
+            return error
+        # return MassenergizeResponse(data=images)
+        return response
+
+    # @admins_only
+    def clean_duplicates(self, request):
+        """Based on requests params, this route can remove duplicates and re-assign relationships for a specific group of similar duplicate items, or do so for all groups """
+        context: Context = request.context
+        args: dict = context.args
+        self.validator.expect("hash", str, is_required=True)
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        images, error = self.service.clean_duplicates(args, context)
+        if error:
+            return error
+        return MassenergizeResponse(data=images)
+
+
+    def summarize_duplicates(self, request):
+        """Creates a summary of duplicate images and a combined list of wherever they are being used on the platform"""
+        context: Context = request.context
+        args: dict = context.args
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        images, error = self.service.summarize_duplicates(args, context)
+        if error:
+            return error
+        return MassenergizeResponse(data=images)
+
+    # @admins_only
+    def generate_hashes(self, request):
+        """Generates hashes of media images in the database that dont have hashes yet"""
+        context: Context = request.context
+        args: dict = context.args
+
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        images, error = self.service.generate_hashes(args, context)
+        if error:
+            return error
+        return MassenergizeResponse(data=images)
+
+
     @admins_only
     def fetch_content(self, request):
         """Fetches image content related communities that admins can browse through"""

src/api/services/media_library.py

@@ -7,6 +7,33 @@ class MediaLibraryService:
     def __init__(self):
         self.store = MediaLibraryStore()
 
+    def print_duplicates(self, args, context):
+        response, error = self.store.print_duplicates(args, context)
+        if error:
+            return None, error
+        # return serialize_all(response), None
+        return response, None
+
+    def clean_duplicates(self, args, context):
+        response, error = self.store.clean_duplicates(args, context)
+        if error:
+            return None, error
+        # return serialize_all(response), None
+        return response, None
+
+    def summarize_duplicates(self, args, context):
+        response, error = self.store.summarize_duplicates(args, context)
+        if error:
+            return None, error
+        # return serialize_all(response), None
+        return response, None
+
+    def generate_hashes(self, args, context):
+        response, error = self.store.generate_hashes(args, context)
+        if error:
+            return None, error
+        return response, None
+
     def fetch_content(self, args):
         images, error = self.store.fetch_content(args)
         if error: