add a tool to find objects with few versions (#2457)

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import logging
+import argparse
+import time
+
+from qcrepocleaner import binUtils
+from qcrepocleaner.Ccdb import Ccdb
+import sys
+import datetime
+
+def parseArgs():
+    """Parse the arguments passed to the script."""
+    logging.info("Parsing arguments")
+    parser = argparse.ArgumentParser(description='Identify the objects that have less than X versions.')
+    parser.add_argument('--url', dest='url', action='store', help='URL of the CCDB, with http[s]://', required=True)
+    parser.add_argument('--log-level', dest='log_level', action='store', default="20",
+                        help='Log level (CRITICAL->50, ERROR->40, WARNING->30, INFO->20,DEBUG->10)')
+    parser.add_argument('--path', dest='path', action='store', default="",
+                        help='The path to work with (without initial slash and without .* at the end, e.g. qc/TST/MO/Bob).', required=True)
+    parser.add_argument('--number-versions', dest='threshold', action='store', default=10,
+                        help='The threshold under which we report an object if it has less or equal versions than that')
+    args = parser.parse_args()
+    logging.info(args)
+    return args
+
+
+def run(args):
+    ccdb = Ccdb(args.url)
+    ccdb.logger = logging.getLogger
+
+    list_results = {}
+    threshold = int(args.threshold)
+
+    path = args.path + ".*"
+    objects_paths = ccdb.getObjectsList(path=path)
+    counter = 1
+
+    for obj_path in objects_paths:
+        number_of_versions = len(ccdb.getVersionsList(object_path=obj_path))
+        logging.debug(f"Number versions for {obj_path} : {number_of_versions}")
+        print(f"{counter}/{len(objects_paths)}", end='\r')
+        counter += 1
+        if number_of_versions <= threshold:
+            list_results[obj_path] = number_of_versions
+
+    logging.info(f"List of the objects in {args.path} with <= versions than {args.threshold} :")
+    for obj_path, number_of_versions in list_results.items():
+        logging.info(f"   {obj_path} ({number_of_versions})")
+
+
+# ****************
+# We start here !
+# ****************
+
+def main():
+    binUtils.prepare_main_logger()
+
+    # Parse arguments
+    args = parseArgs()
+    logging.getLogger().setLevel(int(args.log_level))
+
+    run(args)
+
+
+if __name__ == "__main__":
+    main()

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated

@@ -3,6 +3,8 @@
 import logging
 import argparse
 import time
+
+from qcrepocleaner import binUtils
 from qcrepocleaner.Ccdb import Ccdb
 import sys
 import datetime
@@ -24,21 +26,6 @@ def parseArgs():
     return args
 
 
-def prepare_main_logger():
-    logger = logging.getLogger()
-    # Logging (split between stderr and stdout)
-    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
-    h1 = logging.StreamHandler(sys.stdout)
-    h1.setLevel(logging.DEBUG)
-    h1.addFilter(lambda record: record.levelno <= logging.INFO) # filter out everything that is above INFO level
-    h1.setFormatter(formatter)
-    logger.addHandler(h1)
-    h2 = logging.StreamHandler(sys.stderr)
-    h2.setLevel(logging.WARNING)     # take only warnings and error logs
-    h2.setFormatter(formatter)
-    logger.addHandler(h2)
-
-
 def days_ago_timestamp(days):
     """
     Returns the timestamp (milliseconds) corresponding to the number of days in the past.
@@ -77,7 +64,7 @@ def run(args):
 # ****************
 
 def main():
-    prepare_main_logger()
+    binUtils.prepare_main_logger()
 
     # Parse arguments
     args = parseArgs()