Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add a tool to find objects with few versions
Browse files Browse the repository at this point in the history
Barthelemy committed Oct 21, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent b9e95b3 commit e170be8
Showing 2 changed files with 70 additions and 16 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env python3

import logging
import argparse
import time

from qcrepocleaner import binUtils
from qcrepocleaner.Ccdb import Ccdb
import sys
import datetime

def parseArgs():
"""Parse the arguments passed to the script."""
logging.info("Parsing arguments")
parser = argparse.ArgumentParser(description='Identify the objects that have less than X versions.')
parser.add_argument('--url', dest='url', action='store', help='URL of the CCDB, with http[s]://', required=True)
parser.add_argument('--log-level', dest='log_level', action='store', default="20",
help='Log level (CRITICAL->50, ERROR->40, WARNING->30, INFO->20,DEBUG->10)')
parser.add_argument('--path', dest='path', action='store', default="",
help='The path to work with (without initial slash and without .* at the end, e.g. qc/TST/MO/Bob).', required=True)
parser.add_argument('--number-versions', dest='threshold', action='store', default=10,
help='The threshold under which we report an object if it has less or equal versions than that')
args = parser.parse_args()
logging.info(args)
return args


def run(args):
ccdb = Ccdb(args.url)
ccdb.logger = logging.getLogger

list_results = {}
threshold = int(args.threshold)

path = args.path + ".*"
objects_paths = ccdb.getObjectsList(path=path)
counter = 1

for obj_path in objects_paths:
number_of_versions = len(ccdb.getVersionsList(object_path=obj_path))
logging.debug(f"Number versions for {obj_path} : {number_of_versions}")
print(f"{counter}/{len(objects_paths)}", end='\r')
counter += 1
if number_of_versions <= threshold:
list_results[obj_path] = number_of_versions

logging.info(f"List of the objects in {args.path} with <= versions than {args.threshold} :")
for obj_path, number_of_versions in list_results.items():
logging.info(f" {obj_path} ({number_of_versions})")


# ****************
# We start here !
# ****************

def main():
binUtils.prepare_main_logger()

# Parse arguments
args = parseArgs()
logging.getLogger().setLevel(int(args.log_level))

run(args)


if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
@@ -3,6 +3,8 @@
import logging
import argparse
import time

from qcrepocleaner import binUtils
from qcrepocleaner.Ccdb import Ccdb
import sys
import datetime
@@ -24,21 +26,6 @@ def parseArgs():
return args


def prepare_main_logger():
logger = logging.getLogger()
# Logging (split between stderr and stdout)
formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
h1 = logging.StreamHandler(sys.stdout)
h1.setLevel(logging.DEBUG)
h1.addFilter(lambda record: record.levelno <= logging.INFO) # filter out everything that is above INFO level
h1.setFormatter(formatter)
logger.addHandler(h1)
h2 = logging.StreamHandler(sys.stderr)
h2.setLevel(logging.WARNING) # take only warnings and error logs
h2.setFormatter(formatter)
logger.addHandler(h2)


def days_ago_timestamp(days):
"""
Returns the timestamp (milliseconds) corresponding to the number of days in the past.
@@ -77,7 +64,7 @@ def run(args):
# ****************

def main():
prepare_main_logger()
binUtils.prepare_main_logger()

# Parse arguments
args = parseArgs()

0 comments on commit e170be8

Please sign in to comment.