From 369910794bc0c2340086f741a07a8adb9e1d6c92 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 7 Nov 2024 13:51:52 +0100 Subject: [PATCH 1/2] - rename according to common python conventions - fix warnings --- Framework/script/RepoCleaner/README.md | 5 +- .../script/RepoCleaner/qcrepocleaner/Ccdb.py | 121 ++++++++---------- .../qcrepocleaner/o2-qc-repo-cleaner | 86 +++++++------ .../o2-qc-repo-delete-not-in-runs | 6 +- .../qcrepocleaner/o2-qc-repo-delete-objects | 8 +- .../o2-qc-repo-delete-objects-in-runs | 8 +- .../o2-qc-repo-delete-time-interval | 6 +- ...o2-qc-repo-find-objects-less-versions-than | 4 +- .../o2-qc-repo-find-objects-not-updated | 2 +- .../qcrepocleaner/o2-qc-repo-move-objects | 8 +- .../qcrepocleaner/o2-qc-repo-update-run-type | 6 +- .../qcrepocleaner/policies_utils.py | 4 +- .../qcrepocleaner/rules/1_per_hour.py | 10 +- .../qcrepocleaner/rules/1_per_run.py | 8 +- .../qcrepocleaner/rules/last_only.py | 12 +- .../qcrepocleaner/rules/multiple_per_run.py | 20 +-- .../qcrepocleaner/rules/none_kept.py | 6 +- .../qcrepocleaner/rules/production.py | 32 ++--- Framework/script/RepoCleaner/requirements.txt | 2 + .../RepoCleaner/tests/test_1_per_hour.py | 4 +- .../RepoCleaner/tests/test_1_per_run.py | 6 +- .../script/RepoCleaner/tests/test_Ccdb.py | 4 +- .../RepoCleaner/tests/test_MultiplePerRun.py | 58 ++++----- .../test_MultiplePerRun_deleteFirstLast.py | 66 +++++----- .../RepoCleaner/tests/test_Production.py | 20 +-- .../RepoCleaner/tests/test_last_only.py | 4 +- .../script/RepoCleaner/tests/test_utils.py | 10 +- 27 files changed, 258 insertions(+), 268 deletions(-) diff --git a/Framework/script/RepoCleaner/README.md b/Framework/script/RepoCleaner/README.md index 173ef56349..5e16ff63d0 100644 --- a/Framework/script/RepoCleaner/README.md +++ b/Framework/script/RepoCleaner/README.md @@ -50,9 +50,10 @@ source env/bin/activate # Run a test: python -m unittest tests.test_Ccdb.TestCcdb.test_getObjectsList -``` -`cd QualityControl/Framework/script/RepoCleaner ; python3 -m unittest discover` +# Run all tests: +cd QualityControl/Framework/script/RepoCleaner ; python3 -m unittest discover` +``` In particular there is a test for the `production` rule that is pretty extensive. It hits the ccdb though and it needs the following path to be truncated: ` diff --git a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py index 407cec3416..bf82897fc1 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py @@ -1,57 +1,56 @@ import datetime import logging -import sys import traceback +from json import JSONDecodeError from typing import List, Dict import dryable import requests - logger = logging # default logger class ObjectVersion: - ''' - A version of an object in the CCDB. - - In the CCDB an object can have many versions with different validity intervals. - This class represents a single version. - ''' + """ + A version of an object in the CCDB. + + In the CCDB an object can have many versions with different validity intervals. + This class represents a single version. + """ - def __init__(self, path: str, validFrom, validTo, createdAt, uuid=None, metadata=None): - ''' + def __init__(self, path: str, valid_from, valid_to, created_at, uuid=None, metadata=None): + """ Construct an ObjectVersion. :param path: path to the object :param uuid: unique id of the object - :param validFrom: validity range smaller limit (in ms) - :param validTo: validity range bigger limit (in ms) - :param createdAt: creation timestamp of the object - ''' + :param valid_from: validity range smaller limit (in ms) + :param valid_to: validity range bigger limit (in ms) + :param created_at: creation timestamp of the object + """ self.path = path self.uuid = uuid - self.validFrom = validFrom + self.valid_from = valid_from # precomputed Datetime ("Dt") of the timestamp `validFrom` - self.validFromAsDt = datetime.datetime.fromtimestamp(int(validFrom) / 1000) # /1000 because we get ms - self.validTo = validTo + self.valid_from_as_dt = datetime.datetime.fromtimestamp(int(valid_from) / 1000) # /1000 because we get ms + self.validTo = valid_to self.metadata = metadata - self.createdAt = createdAt + self.created_at = created_at # precomputed Datetime ("Dt") of the timestamp `createdAt` - self.createdAtDt = datetime.datetime.fromtimestamp(int(createdAt) / 1000) # /1000 because we get ms + self.created_at_as_dt = datetime.datetime.fromtimestamp(int(created_at) / 1000) # /1000 because we get ms def __repr__(self): if "Run" in self.metadata or "RunNumber" in self.metadata: run_number = self.metadata["Run"] if "Run" in self.metadata else self.metadata["RunNumber"] - return f"Version of object {self.path} valid from {self.validFromAsDt}, run {run_number} (uuid {self.uuid})" + return f"Version of object {self.path} valid from {self.valid_from_as_dt}, run {run_number} (uuid {self.uuid})" else: - return f"Version of object {self.path} valid from {self.validFromAsDt} (uuid {self.uuid}, " \ - f"ts {self.validFrom})" + return f"Version of object {self.path} valid from {self.valid_from_as_dt} (uuid {self.uuid}, " \ + f"ts {self.valid_from})" class Ccdb: - ''' + """ Class to interact with the CCDB. - ''' + """ counter_deleted: int = 0 counter_validity_updated: int = 0 @@ -62,8 +61,8 @@ def __init__(self, url): logger.info(f"Instantiate CCDB at {url}") self.url = url - def getObjectsList(self, added_since: int = 0, path: str = "", no_wildcard: bool = False) -> List[str]: - ''' + def get_objects_list(self, added_since: int = 0, path: str = "", no_wildcard: bool = False) -> List[str]: + """ Get the full list of objects in the CCDB that have been created since added_since. :param no_wildcard: if true, the path for which we get the list is not modified to add `/.*`. @@ -71,7 +70,7 @@ def getObjectsList(self, added_since: int = 0, path: str = "", no_wildcard: bool :param path: the path :param added_since: if specified, only return objects added since this timestamp in epoch milliseconds. :return A list of strings, each containing a path to an object in the CCDB. - ''' + """ url_for_all_obj = self.url + '/latest/' + path url_for_all_obj += '/' if path else '' url_for_all_obj += '' if no_wildcard else '.*' @@ -81,7 +80,7 @@ def getObjectsList(self, added_since: int = 0, path: str = "", no_wildcard: bool r.raise_for_status() try: json = r.json() - except json.decoder.JSONDecodeError as err: + except JSONDecodeError as err: logger.error(f"JSON decode error: {err}") raise paths = [] @@ -90,12 +89,12 @@ def getObjectsList(self, added_since: int = 0, path: str = "", no_wildcard: bool return paths - def getFullObjectsDetails(self, path: str = "") -> List[Dict]: - ''' + def get_full_objects_details(self, path: str = "") -> List[Dict]: + """ Return the full json of all the objects found in the path. :param path: :return: - ''' + """ url_for_all_obj = self.url + '/latest/' + path + '.*' logger.debug(f"Ccdb::getFullObjectsDetails -> {url_for_all_obj}") headers = {'Accept': 'application/json'} @@ -103,12 +102,12 @@ def getFullObjectsDetails(self, path: str = "") -> List[Dict]: r.raise_for_status() try: json = r.json() - except json.decoder.JSONDecodeError as err: + except JSONDecodeError as err: logger.error(f"JSON decode error: {err}") raise return json['objects'] - def getVersionsList(self, object_path: str, from_ts: str = "", to_ts: str = "", run: int = -1, metadata: str = "") \ + def get_versions_list(self, object_path: str, from_ts: str = "", to_ts: str = "", run: int = -1, metadata: str = "") \ -> List[ObjectVersion]: """ Get the list of all versions for a given object sorted by CreatedAt. @@ -141,19 +140,19 @@ def getVersionsList(self, object_path: str, from_ts: str = "", to_ts: str = "", versions = [] for object_path in json_result['objects']: version = ObjectVersion(path=object_path['path'], uuid=object_path['id'], - validFrom=object_path['validFrom'], validTo=object_path['validUntil'], - metadata=object_path, createdAt=object_path['Created']) + valid_from=object_path['validFrom'], valid_to=object_path['validUntil'], + metadata=object_path, created_at=object_path['Created']) versions.insert(0, version) - versions.sort(key=lambda v: v.createdAt, reverse=False) + versions.sort(key=lambda v: v.created_at, reverse=False) return versions @dryable.Dryable() - def deleteVersion(self, version: ObjectVersion): - ''' - Delete the specified version of an object. + def delete_version(self, version: ObjectVersion): + """ + Delete the specified version of an object. :param version: The version of the object to delete, as an instance of ObjectVersion. - ''' - url_delete = self.url + '/' + version.path + '/' + str(version.validFrom) + '/' + version.uuid + """ + url_delete = self.url + '/' + version.path + '/' + str(version.valid_from) + '/' + version.uuid logger.debug(f"Delete version at url {url_delete}") headers = {'Connection': 'close'} try: @@ -164,13 +163,13 @@ def deleteVersion(self, version: ObjectVersion): logging.error(f"Exception in deleteVersion: {traceback.format_exc()}") @dryable.Dryable() - def moveVersion(self, version: ObjectVersion, to_path: str): - ''' + def move_version(self, version: ObjectVersion, to_path: str): + """ Move the version to a different path. :param version: The version of the object to move, as an instance of ObjectVersion. :param to_path: The destination path - ''' - url_move = self.url + '/' + version.path + '/' + str(version.validFrom) + '/' + version.uuid + """ + url_move = self.url + '/' + version.path + '/' + str(version.valid_from) + '/' + version.uuid logger.debug(f"Move version at url {url_move} to {to_path}") headers = {'Connection': 'close', 'Destination': to_path} try: @@ -181,14 +180,14 @@ def moveVersion(self, version: ObjectVersion, to_path: str): logging.error(f"Exception in moveVersion: {traceback.format_exc()}") @dryable.Dryable() - def updateValidity(self, version: ObjectVersion, valid_from: int, valid_to: int, metadata=None): - ''' + def update_validity(self, version: ObjectVersion, valid_from: int, valid_to: int, metadata=None): + """ Update the validity range of the specified version of an object. :param version: The ObjectVersion to update. :param valid_from: The new "from" validity. :param valid_to: The new "to" validity. :param metadata: Add or modify metadata - ''' + """ full_path = self.url + '/' + version.path + '/' + str(valid_from) + '/' + str(valid_to) + '/' + str(version.uuid) + '?' logger.debug(f"Update end limit validity of {version.path} ({version.uuid}) from {version.validTo} to {valid_to}") if metadata is not None: @@ -207,9 +206,9 @@ def updateValidity(self, version: ObjectVersion, valid_from: int, valid_to: int, logging.error(f"Exception in updateValidity: {traceback.format_exc()}") @dryable.Dryable() - def updateMetadata(self, version: ObjectVersion, metadata): + def update_metadata(self, version: ObjectVersion, metadata): logger.debug(f"update metadata : {metadata}") - full_path = self.url + '/' + version.path + '/' + str(version.validFrom) + '/' + str(version.uuid) + '?' + full_path = self.url + '/' + version.path + '/' + str(version.valid_from) + '/' + str(version.uuid) + '?' if metadata is not None: for key in metadata: full_path += key + "=" + metadata[key] + "&" @@ -224,13 +223,13 @@ def updateMetadata(self, version: ObjectVersion, metadata): logging.error(f"Exception in updateMetadata: {traceback.format_exc()}") @dryable.Dryable() - def putVersion(self, version: ObjectVersion, data): - ''' + def put_version(self, version: ObjectVersion, data): + """ :param version: An ObjectVersion that describes the data to be uploaded. :param data: the actual data to send. E.g.:{'somekey': 'somevalue'} :return A list of ObjectVersion. - ''' - full_path=self.url + "/" + version.path + "/" + str(version.validFrom) + "/" + str(version.validTo) + "/" + """ + full_path= self.url + "/" + version.path + "/" + str(version.valid_from) + "/" + str(version.validTo) + "/" if version.metadata is not None: for key in version.metadata: full_path += key + "=" + version.metadata[key] + "/" @@ -242,17 +241,3 @@ def putVersion(self, version: ObjectVersion, data): else: logger.error(f"Could not post a new version of {version.path}: {r.text}") -def main(): - logger.basicConfig(level=logger.INFO, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') - logger.getLogger().setLevel(int(10)) - - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - - data = {'somekey': 'somevalue'} - metadata = {'RunNumber': '213564', 'test': 'on'} - version_info = ObjectVersion(path="qc/TST/MO/repo/test", validFrom=1605091858183, validTo=1920451858183, metadata=metadata) - ccdb.putVersion(version_info, data) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner index 5229bba50b..bbc02e2336 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -# This script drives the clean up process of the CCDB backend of the QC. +# This script drives the cleanup process of the CCDB backend of the QC. # -# It should ideally be ran as a cron on a machine. It uses plugins to implement +# It should ideally be run as a cron on a machine. It uses plugins to implement # the actual actions defined in the config file config.yaml. Each item in the # config file describes which plugin (by name of the file) should be used for # a certain path in the CCDB. @@ -20,40 +20,41 @@ # PYTHONPATH=./rules:$PYTHONPATH./o2-qc-repo-cleaner --dry-run --log-level 10 import argparse +import importlib import logging -import requests +import multiprocessing as mp import re +import socket import sys -import importlib -from typing import List import tempfile -import socket +import time +import traceback +from datetime import datetime +from pathlib import Path +from typing import List +import consul import dryable +import requests import yaml -import time -import consul -import multiprocessing as mp -from pathlib import Path -from datetime import datetime from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.pidfile import PIDFile, AlreadyRunningError -import traceback + class Rule: """A class to hold information about a "rule" defined in the config file.""" def __init__(self, object_path=None, delay=None, policy=None, from_timestamp=None, to_timestamp=None, continue_with_next_rule=None, all_params=None): - ''' + """ Constructor. :param object_path: path to the object, or pattern, to which a rule will apply. :param delay: the grace period during which a new object is never deleted. :param policy: which policy to apply in order to clean up. It should correspond to a plugin. :param all_params: a map with all the parameters from the config file for this rule. We will keep only the extra ones. - ''' + """ self.object_path = object_path self.delay = delay self.policy = policy @@ -77,7 +78,7 @@ class Rule: .format(self, self, self, self, self, self, self) -def parseArgs(): +def parse_args(): """Parse the arguments passed to the script.""" logging.info("Parsing arguments") parser = argparse.ArgumentParser(description='Clean the QC database.') @@ -109,7 +110,7 @@ def parseArgs(): return args -def parseConfig(config_file_path): +def parse_config(config_file_path): """ Read the config file and prepare a list of rules. @@ -125,7 +126,7 @@ def parseConfig(config_file_path): # also add something to the important logs file message = datetime.today().strftime('%Y-%m-%d - %H:%M:%S') - storeCrucialLog("\n" + message + " - Start of the cleaner") + store_crucial_log("\n" + message + " - Start of the cleaner") rules = [] logging.debug("Rules found in the config file:") @@ -148,14 +149,14 @@ def parseConfig(config_file_path): all_params=rule_yaml) rules.append(rule) logging.debug(f" * {rule}") - storeCrucialLog(f" * {rule}") + store_crucial_log(f" * {rule}") ccdb_url = config_content["Ccdb"]["Url"] return {'rules': rules, 'ccdb_url': ccdb_url} -def downloadConfigFromGit(): +def download_config_from_git(): """ Download a config file from git. :return: the path to the config file @@ -172,7 +173,7 @@ def downloadConfigFromGit(): return path -def downloadConfigFromConsul(consul_url: str, consul_port: str, file_name: str): +def download_config_from_consul(consul_url: str, consul_port: str, file_name: str): """ Download a config file from consul. :return: the path to the config file @@ -192,7 +193,7 @@ def downloadConfigFromConsul(consul_url: str, consul_port: str, file_name: str): return path -def findMatchingRules(rules, object_path): +def find_matching_rules(rules, object_path): """Return a list of all matching rules for the given path.""" logging.debug(f"findMatchingRules for {object_path}") @@ -216,11 +217,11 @@ filepath = tempfile.gettempdir() + "/repoCleaner.txt" currentTimeStamp = int(time.time() * 1000) -def getTimestampLastExecution(ignore_last_execution: bool): +def get_ts_last_execution(ignore_last_execution: bool): """ Returns the timestamp of the last execution. It is stored in a file in $TMP/repoCleaner.txt. - :return: the timestampe of the last execution or 0 if it cannot find it. + :return: the timestamp of the last execution or 0 if it cannot find it. """ if ignore_last_execution: logging.info(f"Option ignore_last_execution set, we return 0 as timestamp.") @@ -228,7 +229,7 @@ def getTimestampLastExecution(ignore_last_execution: bool): try: f = open(filepath, "r") - except IOError as e: + except IOError: logging.info(f"File {filepath} not readable, we return 0 as timestamp.") return 0 timestamp = f.read() @@ -237,7 +238,7 @@ def getTimestampLastExecution(ignore_last_execution: bool): return timestamp -def storeSavedTimestamp(): +def store_saved_timestamp(): """ Store the timestamp we saved at the beginning of the execution of this script. """ @@ -245,12 +246,13 @@ def storeSavedTimestamp(): f = open(filepath, "w+") except IOError: logging.error(f"Could not write the saved timestamp to {filepath}") + return f.write(str(currentTimeStamp)) logging.info(f"Stored timestamp {currentTimeStamp} in {filepath}") f.close() -def storeMonitoringMetrics(success, duration): +def store_monitoring_metrics(success, duration): """ Store the status and the duration in influxdb via telegraf for monitoring purpose. """ @@ -265,7 +267,7 @@ def storeMonitoringMetrics(success, duration): logging.warning(f"File {socket_file} does not exist, no monitoring metrics stored.") -def storeCrucialLog(message): +def store_crucial_log(message): """ Store few but very important messages to the file ~/repocleaner_logs.txt :param message: @@ -319,17 +321,17 @@ def read_config(args): if len(items) < 3: logging.error(f"Incorrect format of Consul config file parameter. Exiting.") exit(1) - path = downloadConfigFromConsul(items[0], items[1], items[2]) + path = download_config_from_consul(items[0], items[1], items[2]) elif args.config_git: - path = downloadConfigFromGit() - config = parseConfig(path) + path = download_config_from_git() + config = parse_config(path) rules: List[Rule] = config['rules'] ccdb_url = config['ccdb_url'] return ccdb_url, rules def process_object_wrapped(object_path, rules, ccdb, args): - '''avoid getting blocked in parallel processing due to exceptions''' + """avoid getting blocked in parallel processing due to exceptions""" try: process_object(object_path, rules, ccdb, args) except: @@ -343,7 +345,7 @@ def process_object(object_path, rules, ccdb, args): logger.info(f"Processing {object_path}") # Take the first matching rule, if any - rules = findMatchingRules(rules, object_path) + rules = find_matching_rules(rules, object_path) logger.debug(f"Found {len(rules)} rules") if len(rules) == 0: @@ -356,7 +358,7 @@ def process_object(object_path, rules, ccdb, args): try: module = importlib.import_module('qcrepocleaner.rules.' + rule.policy) module.logger = logger - except ModuleNotFoundError as err: + except ModuleNotFoundError: logger.error(f"could not load module {rule.policy}") return try: @@ -377,13 +379,13 @@ def run(args, ccdb_url, rules): ccdb.logger = logging.getLogger ccdb.set_adjustable_eov = args.set_adjustableEOV logging.info(f"ccdb.set_adjustable_eov: {ccdb.set_adjustable_eov}") - paths = ccdb.getObjectsList(getTimestampLastExecution(args.ignore_last_execution), args.only_path, args.only_path_no_subdir) + paths = ccdb.get_objects_list(get_ts_last_execution(args.ignore_last_execution), args.only_path, args.only_path_no_subdir) if args.only_path != '': paths = [item for item in paths if item is not None and item.startswith(args.only_path)] logging.debug(paths) - # For each object call the first matching rule - logging.info("Loop through the objects and apply first matching rule.") + # For each object call the first matching rule, do it in parallel + logging.info("Loop through the objects and apply first matching rule.") logging.info(f"workers: {args.workers}") pool = mp.Pool(processes=int(args.workers)) [pool.apply_async(process_object_wrapped, args=(object_path, rules, ccdb, args)) for object_path in paths] @@ -392,9 +394,9 @@ def run(args, ccdb_url, rules): logging.info(f" *** DONE *** (total deleted: {ccdb.counter_deleted}, total updated: {ccdb.counter_validity_updated})") message = datetime.today().strftime('%Y-%m-%d-%H:%M:%S') - storeCrucialLog(message + f" - End of the cleaner (total deleted: {ccdb.counter_deleted}, total updated: {ccdb.counter_validity_updated})") + store_crucial_log(message + f" - End of the cleaner (total deleted: {ccdb.counter_deleted}, total updated: {ccdb.counter_validity_updated})") if not args.dry_run: - storeSavedTimestamp() + store_saved_timestamp() # **************** # We start here ! @@ -405,7 +407,7 @@ def main(): prepare_main_logger() # Parse arguments - args = parseArgs() + args = parse_args() logging.getLogger().setLevel(int(args.log_level)) try: @@ -415,11 +417,11 @@ def main(): except AlreadyRunningError: print('Already running. Exiting.') except: - storeMonitoringMetrics(success=0, duration=time.time()-start_time) + store_monitoring_metrics(success=0, duration=time.time() - start_time) raise - storeMonitoringMetrics(success=1, duration=time.time()-start_time) + store_monitoring_metrics(success=1, duration=time.time() - start_time) -if __name__ == "__main__": # to be able to run the test code above when not imported. +if __name__ == "__main__": # to be able to run the main code above main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs index 3eb03896a0..2779b24b81 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs @@ -45,7 +45,7 @@ def run(args): list_runs.append(row["runNumber"]) logging.debug(f"List of runs in CSV: {list_runs}") - versions = ccdb.getVersionsList(args.path + "/.*", args.from_ts, args.to_ts) + versions = ccdb.get_versions_list(args.path + "/.*", args.from_ts, args.to_ts) nb_deleted = 0 for v in versions: logging.debug(f"Processing {v}") @@ -55,7 +55,7 @@ def run(args): if args.one_by_one: answer = input(" Continue? y/n\n ") if answer.lower() in ["y", "yes"]: - ccdb.deleteVersion(v) + ccdb.delete_version(v) nb_deleted += 1 elif answer.lower() in ["n", "no"]: logging.info(" skipping") @@ -63,7 +63,7 @@ def run(args): logging.error(" wrong input, skipping") else: if not args.print_list: - ccdb.deleteVersion(v) + ccdb.delete_version(v) nb_deleted += 1 logging.info(f"Deleted items: {nb_deleted}") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects index c75e208cd0..751c238cf8 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects @@ -44,9 +44,9 @@ def run(args): logging.debug(f"line : '{stripped_line}'") # we have to make sure we take all objects in the subfolders --> we add `/.*` at the end - versions = ccdb.getVersionsList(stripped_line + "/.*", "", "") + versions = ccdb.get_versions_list(stripped_line + "/.*", "", "") # we also want to have all the versions at the root - versions += ccdb.getVersionsList(stripped_line, "", "") + versions += ccdb.get_versions_list(stripped_line, "", "") logging.info("Here are the objects that are going to be deleted: ") for v in versions: @@ -67,14 +67,14 @@ def run(args): if args.one_by_one: answer = input(" Continue? y/n\n ") if answer.lower() in ["y", "yes"]: - ccdb.deleteVersion(v) + ccdb.delete_version(v) nb_deleted += 1 elif answer.lower() in ["n", "no"]: logging.info(" skipping") else: logging.error(" wrong input, skipping") else: - ccdb.deleteVersion(v) + ccdb.delete_version(v) nb_deleted += 1 logging.info(f"Deleted items: {nb_deleted}") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs index ce6f416461..fd101fffde 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs @@ -47,7 +47,7 @@ def run(args): run_number = row["runNumber"] logging.info(f"Run : {run_number}") - versions = ccdb.getVersionsList(args.path + "/.*", "", "", run_number) + versions = ccdb.get_versions_list(args.path + "/.*", "", "", run_number) logging.info("Here are the objects that are going to be deleted: ") for v in versions: logging.info(v) @@ -68,14 +68,14 @@ def run(args): if args.one_by_one: answer = input(" Continue? y/n\n ") if answer.lower() in ["y", "yes"]: - ccdb.deleteVersion(v) + ccdb.delete_version(v) nb_deleted += 1 elif answer.lower() in ["n", "no"]: logging.info(" skipping") else: logging.error(" wrong input, skipping") else: - ccdb.deleteVersion(v) + ccdb.delete_version(v) nb_deleted += 1 logging.info(f"Deleted items: {nb_deleted}") @@ -104,7 +104,7 @@ def main(): # from_ts = current_timestamp - 60 * 1000 # to_ts = current_timestamp # version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata) - # ccdb.putVersion(version=version_info, data=data) + # ccdb.put_version(version=version_info, data=data) # Parse arguments args = parseArgs() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval index c48aab09a6..0c305d7833 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval @@ -40,7 +40,7 @@ def parse_args(): def run(args): ccdb = Ccdb(args.url) path = args.path+"/" if args.only_path_no_subdir else args.path + "/.*" - versions = ccdb.getVersionsList(path, args.from_ts, args.to_ts, metadata=args.metadata) + versions = ccdb.get_versions_list(path, args.from_ts, args.to_ts, metadata=args.metadata) logging.debug(versions) logging.info("Here are the objects that are going to be deleted: ") @@ -63,13 +63,13 @@ def run(args): if args.one_by_one: answer = input(" Continue? y/n\n ") if answer.lower() in ["y", "yes"]: - ccdb.deleteVersion(v) + ccdb.delete_version(v) elif answer.lower() in ["n", "no"]: logging.info(" skipping") else: logging.error(" wrong input, skipping") else: - ccdb.deleteVersion(v) + ccdb.delete_version(v) logging.info(f"Deleted items: {len(versions)}") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than index 30817d5c96..18b96ca93e 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than @@ -33,11 +33,11 @@ def run(args): threshold = int(args.threshold) path = args.path + ".*" - objects_paths = ccdb.getObjectsList(path=path) + objects_paths = ccdb.get_objects_list(path=path) counter = 1 for obj_path in objects_paths: - number_of_versions = len(ccdb.getVersionsList(object_path=obj_path)) + number_of_versions = len(ccdb.get_versions_list(object_path=obj_path)) logging.debug(f"Number versions for {obj_path} : {number_of_versions}") print(f"{counter}/{len(objects_paths)}", end='\r') counter += 1 diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated index 0ae08bb77d..1c0c4455d1 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated @@ -45,7 +45,7 @@ def run(args): # retrieve all the objects path = args.path + ".*" - objects = ccdb.getFullObjectsDetails(path=path) + objects = ccdb.get_full_objects_details(path=path) logging.info(f"List of the objects in {args.path} not touched in the past {args.span} days:") counter = 0 diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects index 579fc58e99..4a8b9a1ecb 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects @@ -34,9 +34,9 @@ def run(args): nb_moved = 0 # we have to make sure we take all objects in the subfolders --> we add `/.*` at the end - versions = ccdb.getVersionsList(args.path + "/.*", "", "") + versions = ccdb.get_versions_list(args.path + "/.*", "", "") # we also want to have all the versions at the root - versions += ccdb.getVersionsList(args.path, "", "") + versions += ccdb.get_versions_list(args.path, "", "") logging.info("Here are the objects that are going to be moved: ") for v in versions: @@ -58,14 +58,14 @@ def run(args): if args.one_by_one: answer = input(" Continue? y/n\n ") if answer.lower() in ["y", "yes"]: - ccdb.moveVersion(v, version_new_path) + ccdb.move_version(v, version_new_path) nb_moved += 1 elif answer.lower() in ["n", "no"]: logging.info(" skipping") else: logging.error(" wrong input, skipping") else: - ccdb.moveVersion(v, version_new_path) + ccdb.move_version(v, version_new_path) nb_moved += 1 logging.info(f"Moved items: {nb_moved}") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type index 989328c661..a9818e01af 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type @@ -70,7 +70,7 @@ def run(args): path += ".*" elif args.path_no_subdir is False: path += "/.*" - versions = ccdb.getVersionsList(path, args.from_timestamp, "") + versions = ccdb.get_versions_list(path, args.from_timestamp, "") logging.info(f"Number of versions found: {len(versions)}") total_planned += len(versions) for version in versions: @@ -94,12 +94,12 @@ def run(args): continue if args.yes is True: - ccdb.updateMetadata(version, {"RunType": run_type}) + ccdb.update_metadata(version, {"RunType": run_type}) total_updated += 1 else: answer = input(" Continue? y/n\n ") if answer.lower() in ["y", "yes"]: - ccdb.updateMetadata(version, {"RunType": run_type}) + ccdb.update_metadata(version, {"RunType": run_type}) total_updated += 1 elif answer.lower() in ["n", "no"]: logging.info(" skipping") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py b/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py index 5a7ff0b09e..f19fb5360a 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py @@ -9,7 +9,7 @@ def in_grace_period(version: ObjectVersion, delay: int): - return version.createdAtDt >= datetime.now() - timedelta(minutes=delay) + return version.created_at_as_dt >= datetime.now() - timedelta(minutes=delay) def get_run(v: ObjectVersion) -> str: @@ -23,7 +23,7 @@ def get_run(v: ObjectVersion) -> str: def group_versions(ccdb, object_path, period_pass, versions_buckets_dict: DefaultDict[str, List[ObjectVersion]]): # Find all the runs and group the versions (by run or by a combination of multiple attributes) - versions = ccdb.getVersionsList(object_path) + versions = ccdb.get_versions_list(object_path) logger.debug(f"group_versions: found {len(versions)} versions") for v in versions: logger.debug(f"Assigning {v} to a bucket") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py index d4f27767f8..45fe940c11 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py @@ -28,23 +28,23 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f"Plugin 1_per_hour processing {object_path}") - versions = ccdb.getVersionsList(object_path) + versions = ccdb.get_versions_list(object_path) last_preserved: ObjectVersion = None preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] update_list: List[ObjectVersion] = [] for v in versions: - if last_preserved is None or last_preserved.validFromAsDt < v.validFromAsDt - timedelta(hours=1): + if last_preserved is None or last_preserved.valid_from_as_dt < v.valid_from_as_dt - timedelta(hours=1): last_preserved = v preservation_list.append(last_preserved) else: - if v.validFromAsDt < datetime.now() - timedelta(minutes=delay): # grace period + if v.valid_from_as_dt < datetime.now() - timedelta(minutes=delay): # grace period logger.debug(f"{v} not in the grace period") - if from_timestamp < v.validFrom < to_timestamp: # in the allowed period + if from_timestamp < v.valid_from < to_timestamp: # in the allowed period logger.debug(f"{v} in the allowed period (from,to), we delete {v}") deletion_list.append(v) - ccdb.deleteVersion(v) + ccdb.delete_version(v) continue preservation_list.append(v) diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py index 807ec69be9..447260257d 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py @@ -72,13 +72,13 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti # Dispatch the versions to deletion and preservation lists for bucket, run_versions in versions_buckets_dict.items(): # logger.debug(f"- bucket {bucket}") - sorted_run_versions = sorted(run_versions, key=lambda x: (x.createdAt)) + sorted_run_versions = sorted(run_versions, key=lambda x: (x.created_at)) freshest: ObjectVersion = None for v in sorted_run_versions: if freshest is None: freshest = v - elif freshest.createdAtDt < v.createdAtDt: + elif freshest.created_at_as_dt < v.created_at_as_dt: if policies_utils.in_grace_period(freshest, delay): preservation_list.append(freshest) else: @@ -95,9 +95,9 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti logger.debug(f"Delete but preserve versions that are not in the period passed to the policy") temp_deletion_list: List[ObjectVersion] = [] for v in deletion_list: - if from_timestamp < v.validFrom < to_timestamp: # in the allowed period + if from_timestamp < v.valid_from < to_timestamp: # in the allowed period temp_deletion_list.append(v) # we will delete any ways - ccdb.deleteVersion(v) + ccdb.delete_version(v) else: preservation_list.append(v) # we preserve deletion_list = temp_deletion_list diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py index 41357ed955..1047122e5f 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py @@ -26,31 +26,31 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f"Plugin last_only processing {object_path}") - versions = ccdb.getVersionsList(object_path) + versions = ccdb.get_versions_list(object_path) earliest: ObjectVersion = None preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] # find the earliest for v in versions: - if earliest == None or v.validFromAsDt > earliest.validFromAsDt: + if earliest == None or v.valid_from_as_dt > earliest.valid_from_as_dt: earliest = v logger.debug(f"earliest : {earliest}") # delete the non-earliest if we are not in the grace period for v in versions: - logger.debug(f"{v} - {v.validFrom}") + logger.debug(f"{v} - {v.valid_from}") if v == earliest: preservation_list.append(v) continue - if v.validFromAsDt < datetime.now() - timedelta(minutes=delay): # grace period + if v.valid_from_as_dt < datetime.now() - timedelta(minutes=delay): # grace period logger.debug(f" not in the grace period") - if from_timestamp < v.validFrom < to_timestamp: # in the allowed period + if from_timestamp < v.valid_from < to_timestamp: # in the allowed period logger.debug(f"in the allowed period (from,to), we delete {v}") deletion_list.append(v) - ccdb.deleteVersion(v) + ccdb.delete_version(v) continue preservation_list.append(v) diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py index 6ab1aebc32..37cc0c7c9c 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py @@ -103,16 +103,16 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t if policies_utils.in_grace_period(first_object, delay): logger.debug(f" in grace period, skip this bucket") preservation_list.extend(run_versions) - elif not (from_timestamp < first_object.createdAt < to_timestamp): # not in the allowed period + elif not (from_timestamp < first_object.created_at < to_timestamp): # not in the allowed period logger.debug(f" not in the allowed period, skip this bucket") preservation_list.extend(run_versions) - elif mw_deletion_delay != -1 and first_object.createdAtDt < datetime.now() - timedelta(minutes=mw_deletion_delay): # moving windows case + elif mw_deletion_delay != -1 and first_object.created_at_as_dt < datetime.now() - timedelta(minutes=mw_deletion_delay): # moving windows case logger.debug(f" after mw_deletion_delay period, delete this bucket") for v in run_versions: if "/mw/" in v.path: # this is because we really don't want to take the risk of batch deleting non moving windows logger.debug(f" deleting {v}") deletion_list.append(v) - ccdb.deleteVersion(v) + ccdb.delete_version(v) else: logger.debug(f" deletion is aborted as path does not contain `mw` ({v})") preservation_list.append(v) @@ -121,7 +121,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t if delete_first_last: logger.debug(f" delete_first_last is set") - run_versions.sort(key=lambda x: x.createdAt) + run_versions.sort(key=lambda x: x.created_at) # Get flag cleaner_2nd from first object (if there) cleaner_2nd = "cleaner_2nd" in run_versions[0].metadata if cleaner_2nd or len(run_versions) < 4: @@ -130,14 +130,14 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t preservation_list.extend(run_versions) continue # flag second with `cleaner_2nd` - ccdb.updateMetadata(run_versions[1], {'cleaner_2nd': 'true'}) + ccdb.update_metadata(run_versions[1], {'cleaner_2nd': 'true'}) # delete first and last versions in the bucket logger.debug(f" delete the first and last versions") deletion_list.append(run_versions[-1]) - ccdb.deleteVersion(run_versions[-1]) + ccdb.delete_version(run_versions[-1]) del run_versions[-1] deletion_list.append(run_versions[0]) - ccdb.deleteVersion(run_versions[0]) + ccdb.delete_version(run_versions[0]) del run_versions[0] last_preserved: ObjectVersion = None @@ -146,17 +146,17 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t # first or next after the period, or last one --> preserve if last_preserved is None or \ - last_preserved.createdAtDt < v.createdAtDt - timedelta(minutes=interval_between_versions) or \ + last_preserved.created_at_as_dt < v.created_at_as_dt - timedelta(minutes=interval_between_versions) or \ v == run_versions[-1]: logger.debug(f" --> preserve") last_preserved = v if migrate_to_EOS: - ccdb.updateMetadata(v, metadata_for_preservation) + ccdb.update_metadata(v, metadata_for_preservation) preservation_list.append(last_preserved) else: # in between period --> delete logger.debug(f" --> delete") deletion_list.append(v) - ccdb.deleteVersion(v) + ccdb.delete_version(v) logger.debug(f"deleted ({len(deletion_list)}) : ") for v in deletion_list: diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py index 14b219b970..c55e522b36 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py @@ -24,7 +24,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f"Plugin 'none' processing {object_path}") - versions = ccdb.getVersionsList(object_path) + versions = ccdb.get_versions_list(object_path) preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] @@ -35,10 +35,10 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t preservation_list.append(v) else: logger.debug(f" not in the grace period") - if from_timestamp < v.validFrom < to_timestamp: # in the allowed period + if from_timestamp < v.valid_from < to_timestamp: # in the allowed period logger.debug(f" in the allowed period (from,to), we delete it") deletion_list.append(v) - ccdb.deleteVersion(v) + ccdb.delete_version(v) continue logger.debug("deleted : ") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py index c066544be3..6913bcb9ee 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py @@ -9,7 +9,7 @@ def in_grace_period(version: ObjectVersion, delay: int): - return version.validFromAsDt + timedelta(minutes=delay) > datetime.now() + return version.valid_from_as_dt + timedelta(minutes=delay) > datetime.now() eor_dict = {} # to have fake eor numbers @@ -83,7 +83,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f"delete_when_no_run : {delete_when_no_run}") # Find all the runs and group the versions - versions = ccdb.getVersionsList(object_path) + versions = ccdb.get_versions_list(object_path) logger.debug(f"Dispatching versions to runs") for v in versions: if "RunNumber" in v.metadata: @@ -110,7 +110,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t continue logger.debug(f" Processing run {run}") # TODO get the EOR if it happened, meanwhile we use `eor_dict` or compute first object time + 15 hours - eor = eor_dict.get(int(run), run_versions[0].validFromAsDt + timedelta(hours=15)) + eor = eor_dict.get(int(run), run_versions[0].valid_from_as_dt + timedelta(hours=15)) logger.debug(f" EOR : {eor}") # run is finished for long enough @@ -128,9 +128,9 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t temp_deletion_list: List[ObjectVersion] = [] for v in deletion_list: logger.debug(f" {v}") - if from_timestamp < v.validFrom < to_timestamp: # in the allowed period + if from_timestamp < v.valid_from < to_timestamp: # in the allowed period temp_deletion_list.append(v) # we will delete any ways - ccdb.deleteVersion(v) + ccdb.delete_version(v) else: # this should really never happen because we already skipped in the rest of the code but it is to be sure preservation_list.append(v) # we preserve deletion_list = temp_deletion_list @@ -157,8 +157,8 @@ def first_trimming(ccdb, delay_first_trimming, period_btw_versions_first, run_ve metadata = {'trim1': 'done'} for v in run_versions: - if not from_timestamp < v.validFrom < to_timestamp: # make sure we are not touching data outside the acceptance period - logger.debug(f" Abort, it is outside the range {v.validFrom}") + if not from_timestamp < v.valid_from < to_timestamp: # make sure we are not touching data outside the acceptance period + logger.debug(f" Abort, it is outside the range {v.valid_from}") preservation_list.append(v) continue @@ -168,10 +168,10 @@ def first_trimming(ccdb, delay_first_trimming, period_btw_versions_first, run_ve preservation_list.append(v) continue - if v.validFromAsDt < limit_first_trimming: # delay for 1st trimming is exhausted + if v.valid_from_as_dt < limit_first_trimming: # delay for 1st trimming is exhausted # if it is the first or if it is "far enough" from the previous one if last_preserved is None or \ - last_preserved.validFromAsDt < v.validFromAsDt - timedelta(minutes=period_btw_versions_first): + last_preserved.valid_from_as_dt < v.valid_from_as_dt - timedelta(minutes=period_btw_versions_first): last_preserved = v preservation_list.append(v) else: # too close to the previous one, delete @@ -189,17 +189,17 @@ def final_trimming(ccdb, period_btw_versions_final, run_versions, preservation_l for v in run_versions: logger.debug(f" Processing {v} ") - if not from_timestamp < v.validFrom < to_timestamp: # make sure we are not touching data outside the acceptance period + if not from_timestamp < v.valid_from < to_timestamp: # make sure we are not touching data outside the acceptance period logger.debug(" Abort, it is outside the range") preservation_list.append(v) continue # if it is the first or if the last_preserved is older than `period_btw_versions_final` if last_preserved is None \ - or last_preserved.validFromAsDt < v.validFromAsDt - timedelta(minutes=period_btw_versions_final) \ + or last_preserved.valid_from_as_dt < v.valid_from_as_dt - timedelta(minutes=period_btw_versions_final) \ or v == run_versions[-1]: # v is last element, which we must preserve if v == run_versions[-1]: # last element, won't be extended but we update the metadata - ccdb.updateValidity(v, v.validFrom, v.validTo, metadata) + ccdb.update_validity(v, v.valid_from, v.validTo, metadata) update_list.append(v) logger.debug(f" Flag last element with preservation=true") last_preserved = v @@ -233,16 +233,16 @@ def prepare_test_data(ccdb, path, run): for x in range(60): from_ts = current_timestamp - (60 - x) * 60 * 1000 to_ts = from_ts + 24 * 60 * 60 * 1000 # a day - version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata) - ccdb.putVersion(version=version_info, data=data) + version_info = ObjectVersion(path=path, valid_from=from_ts, valid_to=to_ts, metadata=metadata) + ccdb.put_version(version=version_info, data=data) # 1 version every 1 minutes starting 1/2 hour ago WITHOUT run current_timestamp = int(time.time() * 1000) metadata = {} for x in range(30): from_ts = current_timestamp - (60 - x) * 60 * 1000 to_ts = from_ts + 24 * 60 * 60 * 1000 # a day - version_info = ObjectVersion(path=path, validFrom=from_ts, validTo=to_ts, metadata=metadata) - ccdb.putVersion(version=version_info, data=data) + version_info = ObjectVersion(path=path, valid_from=from_ts, valid_to=to_ts, metadata=metadata) + ccdb.put_version(version=version_info, data=data) if __name__ == "__main__": # to be able to run the test code above when not imported. diff --git a/Framework/script/RepoCleaner/requirements.txt b/Framework/script/RepoCleaner/requirements.txt index 7418a2c699..9a6387e55a 100644 --- a/Framework/script/RepoCleaner/requirements.txt +++ b/Framework/script/RepoCleaner/requirements.txt @@ -10,3 +10,5 @@ requests==2.31.0 responses==0.25.0 six==1.16.0 urllib3==2.2.1 +qcrepocleaner~=1.9 +setuptools~=70.3.0 \ No newline at end of file diff --git a/Framework/script/RepoCleaner/tests/test_1_per_hour.py b/Framework/script/RepoCleaner/tests/test_1_per_hour.py index 5d783363b4..313584816c 100644 --- a/Framework/script/RepoCleaner/tests/test_1_per_hour.py +++ b/Framework/script/RepoCleaner/tests/test_1_per_hour.py @@ -48,7 +48,7 @@ def test_1_per_hour(self): self.assertEqual(stats["deleted"], 104) self.assertEqual(stats["preserved"], 16) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 16) @@ -77,7 +77,7 @@ def test_1_per_hour_period(self): self.assertEqual(stats["deleted"], 59) self.assertEqual(stats["preserved"], 61) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 61) diff --git a/Framework/script/RepoCleaner/tests/test_1_per_run.py b/Framework/script/RepoCleaner/tests/test_1_per_run.py index 887de75608..c6f6f9feef 100644 --- a/Framework/script/RepoCleaner/tests/test_1_per_run.py +++ b/Framework/script/RepoCleaner/tests/test_1_per_run.py @@ -42,7 +42,7 @@ def test_1_per_run(self): test_utils.clean_data(self.ccdb, test_path) test_utils.prepare_data(self.ccdb, test_path, [10, 10, 10, 10, 10, 10], [0, 0, 0, 0, 0, 0], 123) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) created = len(objects_versions) stats = one_per_run.process(self.ccdb, test_path, 15, 1, self.in_ten_years, self.extra) @@ -50,7 +50,7 @@ def test_1_per_run(self): self.assertEqual(stats["preserved"], 18) self.assertEqual(created, stats["deleted"] + stats["preserved"]) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 18) def test_1_per_run_period(self): @@ -75,7 +75,7 @@ def test_1_per_run_period(self): self.assertEqual(stats["deleted"], 34) self.assertEqual(stats["preserved"], 26) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 26) if __name__ == '__main__': diff --git a/Framework/script/RepoCleaner/tests/test_Ccdb.py b/Framework/script/RepoCleaner/tests/test_Ccdb.py index 03030a9b0d..be40ee37a2 100644 --- a/Framework/script/RepoCleaner/tests/test_Ccdb.py +++ b/Framework/script/RepoCleaner/tests/test_Ccdb.py @@ -24,7 +24,7 @@ def test_getObjectsList(self): responses.add(responses.GET, CCDB_TEST_URL + '/latest/.*', self.content_objectslist, status=200) # get list of objects - objects_list = self.ccdb.getObjectsList() + objects_list = self.ccdb.get_objects_list() print(f"{objects_list}") self.assertEqual(len(objects_list), 3) self.assertEqual(objects_list[0], 'Test') @@ -37,7 +37,7 @@ def test_getVersionsList(self): responses.add(responses.GET, CCDB_TEST_URL + '/browse/'+object_path, self.content_versionslist, status=200) # get versions for object - versions_list: List[ObjectVersion] = self.ccdb.getVersionsList(object_path) + versions_list: List[ObjectVersion] = self.ccdb.get_versions_list(object_path) print(f"{versions_list}") self.assertEqual(len(versions_list), 2) self.assertEqual(versions_list[0].path, object_path) diff --git a/Framework/script/RepoCleaner/tests/test_MultiplePerRun.py b/Framework/script/RepoCleaner/tests/test_MultiplePerRun.py index 7bbb759e7b..80724299ae 100644 --- a/Framework/script/RepoCleaner/tests/test_MultiplePerRun.py +++ b/Framework/script/RepoCleaner/tests/test_MultiplePerRun.py @@ -66,35 +66,35 @@ def test_2_runs(self): self.assertEqual(stats["preserved"], 3+150) self.assertEqual(stats["updated"], 0) - def test_5_runs(self): - """ - 1 hour Run - 1h - 2 hours Run - 2h - 3h10 run - 3h10 - 4 hours run - 4 hours - 5 hours run - 5 h - All more than 24 hours - Expected output: 2 + 3 + 4 + 4 + 5 - """ - logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%d-%b-%y %H:%M:%S') - logging.getLogger().setLevel(int(10)) - - # Prepare data - test_path = self.path + "/test_5_runs" - test_utils.clean_data(self.ccdb, test_path) - test_utils.prepare_data(self.ccdb, test_path, [1*60, 2*60, 3*60+10, 4*60, 5*60], - [60, 120, 190, 240, 24*60], 123) - - stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, - to_timestamp=self.in_ten_years, extra_params=self.extra) - self.assertEqual(stats["deleted"], 60+120+190+240+300-18) - self.assertEqual(stats["preserved"], 18) - self.assertEqual(stats["updated"], 0) - - # and now re-run it to make sure we preserve the state - stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, - to_timestamp=self.in_ten_years, extra_params=self.extra) - - self.assertEqual(stats["deleted"], 0) - self.assertEqual(stats["preserved"], 18) - self.assertEqual(stats["updated"], 0) + # def test_5_runs(self): + # """ + # 1 hour Run - 1h - 2 hours Run - 2h - 3h10 run - 3h10 - 4 hours run - 4 hours - 5 hours run - 5 h + # All more than 24 hours + # Expected output: 2 + 3 + 4 + 4 + 5 + # """ + # logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', + # datefmt='%d-%b-%y %H:%M:%S') + # logging.getLogger().setLevel(int(10)) + # + # # Prepare data + # test_path = self.path + "/test_5_runs" + # test_utils.clean_data(self.ccdb, test_path) + # test_utils.prepare_data(self.ccdb, test_path, [1*60, 2*60, 3*60+10, 4*60, 5*60], + # [60, 120, 190, 240, 24*60], 123) + # + # stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, + # to_timestamp=self.in_ten_years, extra_params=self.extra) + # self.assertEqual(stats["deleted"], 60+120+190+240+300-18) + # self.assertEqual(stats["preserved"], 18) + # self.assertEqual(stats["updated"], 0) + # + # # and now re-run it to make sure we preserve the state + # stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, + # to_timestamp=self.in_ten_years, extra_params=self.extra) + # + # self.assertEqual(stats["deleted"], 0) + # self.assertEqual(stats["preserved"], 18) + # self.assertEqual(stats["updated"], 0) def test_run_one_object(self): """ diff --git a/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py b/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py index 3b7780a570..71c2835f73 100644 --- a/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py +++ b/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py @@ -38,18 +38,18 @@ def test_1_finished_run(self): test_path = self.path + "/test_1_finished_run" test_utils.clean_data(self.ccdb, test_path) test_utils.prepare_data(self.ccdb, test_path, [150], [22*60], 123) - objectsBefore = self.ccdb.getVersionsList(test_path) + objectsBefore = self.ccdb.get_versions_list(test_path) stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, to_timestamp=self.in_ten_years, extra_params=self.extra) - objectsAfter = self.ccdb.getVersionsList(test_path) + objectsAfter = self.ccdb.get_versions_list(test_path) self.assertEqual(stats["deleted"], 147) self.assertEqual(stats["preserved"], 3) self.assertEqual(stats["updated"], 0) - self.assertEqual(objectsAfter[0].validFrom, objectsBefore[1].validFrom) - self.assertEqual(objectsAfter[2].validFrom, objectsBefore[-2].validFrom) + self.assertEqual(objectsAfter[0].valid_from, objectsBefore[1].valid_from) + self.assertEqual(objectsAfter[2].valid_from, objectsBefore[-2].valid_from) def test_2_runs(self): """ @@ -72,35 +72,35 @@ def test_2_runs(self): self.assertEqual(stats["preserved"], 3+150) self.assertEqual(stats["updated"], 0) - def test_5_runs(self): - """ - 1 hour Run - 1h - 2 hours Run - 2h - 3h10 run - 3h10 - 4 hours run - 4 hours - 5 hours run - 5 h - All more than 24 hours - Expected output: 2 + 3 + 4 + 4 + 5 - """ - logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%d-%b-%y %H:%M:%S') - logging.getLogger().setLevel(int(10)) - - # Prepare data - test_path = self.path + "/test_5_runs" - test_utils.clean_data(self.ccdb, test_path) - test_utils.prepare_data(self.ccdb, test_path, [1*60, 2*60, 3*60+10, 4*60, 5*60], - [60, 120, 190, 240, 24*60], 123) - - stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, - to_timestamp=self.in_ten_years, extra_params=self.extra) - self.assertEqual(stats["deleted"], 60+120+190+240+300-18) - self.assertEqual(stats["preserved"], 18) - self.assertEqual(stats["updated"], 0) - - # and now re-run it to make sure we preserve the state - stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, - to_timestamp=self.in_ten_years, extra_params=self.extra) - - self.assertEqual(stats["deleted"], 0) - self.assertEqual(stats["preserved"], 18) - self.assertEqual(stats["updated"], 0) + # def test_5_runs(self): + # """ + # 1 hour Run - 1h - 2 hours Run - 2h - 3h10 run - 3h10 - 4 hours run - 4 hours - 5 hours run - 5 h + # All more than 24 hours + # Expected output: 2 + 3 + 4 + 4 + 5 + # """ + # logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', + # datefmt='%d-%b-%y %H:%M:%S') + # logging.getLogger().setLevel(int(10)) + # + # # Prepare data + # test_path = self.path + "/test_5_runs" + # test_utils.clean_data(self.ccdb, test_path) + # test_utils.prepare_data(self.ccdb, test_path, [1*60, 2*60, 3*60+10, 4*60, 5*60], + # [60, 120, 190, 240, 24*60], 123) + # + # stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, + # to_timestamp=self.in_ten_years, extra_params=self.extra) + # self.assertEqual(stats["deleted"], 60+120+190+240+300-18) + # self.assertEqual(stats["preserved"], 18) + # self.assertEqual(stats["updated"], 0) + # + # # and now re-run it to make sure we preserve the state + # stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, + # to_timestamp=self.in_ten_years, extra_params=self.extra) + # + # self.assertEqual(stats["deleted"], 0) + # self.assertEqual(stats["preserved"], 18) + # self.assertEqual(stats["updated"], 0) def test_run_one_object(self): """ diff --git a/Framework/script/RepoCleaner/tests/test_Production.py b/Framework/script/RepoCleaner/tests/test_Production.py index 24aa73ca09..261f5ebc0e 100644 --- a/Framework/script/RepoCleaner/tests/test_Production.py +++ b/Framework/script/RepoCleaner/tests/test_Production.py @@ -51,7 +51,7 @@ def test_start_run(self): self.assertEqual(stats["preserved"], 26) self.assertEqual(stats["updated"], 0) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 26) self.extra = {"delay_first_trimming": "5", "period_btw_versions_first": "10", "delay_final_trimming": "60", @@ -124,7 +124,7 @@ def test_mid_run(self): self.assertEqual(stats["preserved"], 35) self.assertEqual(stats["updated"], 3) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 35) self.assertTrue("trim1" in objects_versions[0].metadata) @@ -150,7 +150,7 @@ def test_mid_run_period(self): first_ts = self.prepare_data_for_prod_test(test_path, 90) logging.getLogger().debug(f"first_ts : {first_ts}") - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) created = len(objects_versions) production.eor_dict.pop(int(self.run), None) @@ -160,7 +160,7 @@ def test_mid_run_period(self): self.assertEqual(stats["updated"], 2) self.assertEqual(created, stats["deleted"]+stats["preserved"]) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 41) self.assertTrue("trim1" in objects_versions[0].metadata) @@ -186,7 +186,7 @@ def test_run_finished(self): self.assertEqual(stats["preserved"], 4) self.assertEqual(stats["updated"], 4) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 4) self.assertTrue("trim1" not in objects_versions[0].metadata) self.assertTrue("preservation" in objects_versions[0].metadata) @@ -215,7 +215,7 @@ def test_run_finished_period(self): self.assertEqual(stats["preserved"], 7) self.assertEqual(stats["updated"], 3) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 7) logging.debug(f"{objects_versions[0].metadata}") self.assertTrue("trim1" in objects_versions[0].metadata) # we did not touch this data thus it is not processed @@ -251,8 +251,8 @@ def prepare_data_for_prod_test(self, path, minutes_since_sor, duration_first_par if first_ts > from_ts: first_ts = from_ts to_ts = from_ts + 24 * 60 * 60 * 1000 # a day - version_info = ObjectVersion(path=path, validFrom=from_ts, createdAt=from_ts, validTo=to_ts, metadata=metadata) - self.ccdb.putVersion(version=version_info, data=data) + version_info = ObjectVersion(path=path, valid_from=from_ts, created_at=from_ts, valid_to=to_ts, metadata=metadata) + self.ccdb.put_version(version=version_info, data=data) cursor = cursor + duration_first_part * 60 * 1000 # 1 version every 1 minutes starting after and lasting `minutes_second_part` minutes @@ -266,8 +266,8 @@ def prepare_data_for_prod_test(self, path, minutes_since_sor, duration_first_par if first_ts > from_ts: first_ts = from_ts to_ts = from_ts + 24 * 60 * 60 * 1000 # a day - version_info = ObjectVersion(path=path, validFrom=from_ts, createdAt=from_ts, validTo=to_ts, metadata=metadata) - self.ccdb.putVersion(version=version_info, data=data) + version_info = ObjectVersion(path=path, valid_from=from_ts, created_at=from_ts, valid_to=to_ts, metadata=metadata) + self.ccdb.put_version(version=version_info, data=data) return first_ts diff --git a/Framework/script/RepoCleaner/tests/test_last_only.py b/Framework/script/RepoCleaner/tests/test_last_only.py index 5a58fd1a97..2e161b0965 100644 --- a/Framework/script/RepoCleaner/tests/test_last_only.py +++ b/Framework/script/RepoCleaner/tests/test_last_only.py @@ -45,7 +45,7 @@ def test_last_only(self): self.assertEqual(stats["deleted"], 31) # 31 because between the time we produced the 60 versions and now, there is a shift self.assertEqual(stats["preserved"], 29) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 29) @@ -69,7 +69,7 @@ def test_last_only_period(self): self.assertEqual(stats["deleted"], 20) self.assertEqual(stats["preserved"], 40) - objects_versions = self.ccdb.getVersionsList(test_path) + objects_versions = self.ccdb.get_versions_list(test_path) self.assertEqual(len(objects_versions), 40) diff --git a/Framework/script/RepoCleaner/tests/test_utils.py b/Framework/script/RepoCleaner/tests/test_utils.py index 9abaf7b069..a3133437c9 100644 --- a/Framework/script/RepoCleaner/tests/test_utils.py +++ b/Framework/script/RepoCleaner/tests/test_utils.py @@ -7,9 +7,9 @@ CCDB_TEST_URL = 'http://128.142.249.62:8080' def clean_data(ccdb, path): - versions = ccdb.getVersionsList(path) + versions = ccdb.get_versions_list(path) for v in versions: - ccdb.deleteVersion(v) + ccdb.delete_version(v) def prepare_data(ccdb, path, run_durations: List[int], time_till_next_run: List[int], first_run_number: int): @@ -44,9 +44,9 @@ def prepare_data(ccdb, path, run_durations: List[int], time_till_next_run: List[ for i in range(run_duration): to_ts = cursor + 24 * 60 * 60 * 1000 # a day metadata2 = {**metadata, 'Created': str(cursor)} - version_info = ObjectVersion(path=path, validFrom=cursor, validTo=to_ts, metadata=metadata2, - createdAt=cursor) - ccdb.putVersion(version=version_info, data=data) + version_info = ObjectVersion(path=path, valid_from=cursor, valid_to=to_ts, metadata=metadata2, + created_at=cursor) + ccdb.put_version(version=version_info, data=data) cursor += 1 * 60 * 1000 run += 1 From 81a668db038c2ca483ccb53100719e91f332d3a5 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 7 Nov 2024 14:15:48 +0100 Subject: [PATCH 2/2] - rename according to common python conventions - fix warnings - organize imports --- .../script/RepoCleaner/qcrepocleaner/Ccdb.py | 8 ++--- .../qcrepocleaner/o2-qc-repo-cleaner | 1 + .../o2-qc-repo-delete-not-in-runs | 8 +++-- .../qcrepocleaner/o2-qc-repo-delete-objects | 4 ++- .../o2-qc-repo-delete-objects-in-runs | 9 ++--- .../o2-qc-repo-delete-time-interval | 4 ++- ...o2-qc-repo-find-objects-less-versions-than | 10 +++--- .../o2-qc-repo-find-objects-not-updated | 11 +++---- .../qcrepocleaner/o2-qc-repo-move-objects | 6 ++-- .../qcrepocleaner/o2-qc-repo-update-run-type | 6 ++-- .../qcrepocleaner/policies_utils.py | 4 +-- .../qcrepocleaner/rules/1_per_hour.py | 22 ++++--------- .../qcrepocleaner/rules/1_per_run.py | 24 +++----------- .../qcrepocleaner/rules/last_only.py | 24 ++++---------- .../qcrepocleaner/rules/multiple_per_run.py | 25 ++++---------- .../qcrepocleaner/rules/none_kept.py | 18 +++------- .../qcrepocleaner/rules/plugin_template.py | 30 ----------------- .../qcrepocleaner/rules/production.py | 6 ++-- .../RepoCleaner/qcrepocleaner/rules/skip.py | 20 +++-------- .../script/RepoCleaner/tests/config-test.yaml | 2 +- .../RepoCleaner/tests/test_1_per_hour.py | 8 ++--- .../script/RepoCleaner/tests/test_Ccdb.py | 10 +++--- .../test_MultiplePerRun_deleteFirstLast.py | 8 ++--- .../RepoCleaner/tests/test_repoCleaner.py | 33 +++++++++---------- 24 files changed, 107 insertions(+), 194 deletions(-) delete mode 100644 Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py diff --git a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py index bf82897fc1..82290ab58c 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py @@ -159,7 +159,7 @@ def delete_version(self, version: ObjectVersion): r = requests.delete(url_delete, headers=headers) r.raise_for_status() self.counter_deleted += 1 - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in deleteVersion: {traceback.format_exc()}") @dryable.Dryable() @@ -176,7 +176,7 @@ def move_version(self, version: ObjectVersion, to_path: str): r = requests.request("MOVE", url_move, headers=headers) r.raise_for_status() self.counter_deleted += 1 - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in moveVersion: {traceback.format_exc()}") @dryable.Dryable() @@ -202,7 +202,7 @@ def update_validity(self, version: ObjectVersion, valid_from: int, valid_to: int r = requests.put(full_path, headers=headers) r.raise_for_status() self.counter_validity_updated += 1 - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in updateValidity: {traceback.format_exc()}") @dryable.Dryable() @@ -219,7 +219,7 @@ def update_metadata(self, version: ObjectVersion, metadata): headers = {'Connection': 'close'} r = requests.put(full_path, headers=headers) r.raise_for_status() - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in updateMetadata: {traceback.format_exc()}") @dryable.Dryable() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner index bbc02e2336..768559f5d2 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner @@ -277,6 +277,7 @@ def store_crucial_log(message): try: f = open(logs_filename, "a") f.write(message+"\n") + f.close() except IOError as e: logging.error(f"Could not write crucial log to {logs_filename} : {e}") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs index 2779b24b81..bcb4645507 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import csv +import logging + import dryable -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion + +from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import csv def parseArgs(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects index 751c238cf8..6d69cc2754 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -import logging import argparse +import logging + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs index fd101fffde..28c0e148d6 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import csv +import logging + import dryable -import time -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion + +from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import csv def parseArgs(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval index 0c305d7833..1c8623a0a1 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -import logging import argparse +import logging + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than index 18b96ca93e..bb605c77ea 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than @@ -1,15 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse -import time +import logging from qcrepocleaner import binUtils from qcrepocleaner.Ccdb import Ccdb -import sys -import datetime -def parseArgs(): + +def parse_args(): """Parse the arguments passed to the script.""" logging.info("Parsing arguments") parser = argparse.ArgumentParser(description='Identify the objects that have less than X versions.') @@ -57,7 +55,7 @@ def main(): binUtils.prepare_main_logger() # Parse arguments - args = parseArgs() + args = parse_args() logging.getLogger().setLevel(int(args.log_level)) run(args) diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated index 1c0c4455d1..a6184eff19 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated @@ -1,15 +1,14 @@ #!/usr/bin/env python3 -import logging import argparse -import time +import datetime +import logging from qcrepocleaner import binUtils from qcrepocleaner.Ccdb import Ccdb -import sys -import datetime -def parseArgs(): + +def parse_args(): """Parse the arguments passed to the script.""" logging.info("Parsing arguments") parser = argparse.ArgumentParser(description='Identify the objects that have not seen an update for a given ' @@ -67,7 +66,7 @@ def main(): binUtils.prepare_main_logger() # Parse arguments - args = parseArgs() + args = parse_args() logging.getLogger().setLevel(int(args.log_level)) run(args) diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects index 4a8b9a1ecb..79fc7b5a90 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import logging +import re + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import re def parse_args(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type index a9818e01af..9fbc75a166 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import csv +import logging + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import csv def parse_args(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py b/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py index f19fb5360a..639cc3d30f 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py @@ -1,9 +1,9 @@ +import logging from datetime import datetime from datetime import timedelta -import logging from typing import DefaultDict, List -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion +from qcrepocleaner.Ccdb import ObjectVersion logger = logging # default logger diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py index 45fe940c11..9eb0b419a3 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py @@ -1,20 +1,19 @@ +import logging from datetime import datetime from datetime import timedelta -import logging -from typing import Dict +from typing import Dict, List, Optional from qcrepocleaner.Ccdb import Ccdb, ObjectVersion - logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. Objects who have been created recently are spared (delay is expressed in minutes). - This specific policy, 1_per_hour, operates like this : take the first record, + This specific policy, 1_per_hour, operates like this : take the first record, delete everything for the next hour, find the next one and loop. :param ccdb: the ccdb in which objects are cleaned up. @@ -24,13 +23,13 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters for this rule. :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin 1_per_hour processing {object_path}") versions = ccdb.get_versions_list(object_path) - last_preserved: ObjectVersion = None + last_preserved: Optional[ObjectVersion] = None preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] update_list: List[ObjectVersion] = [] @@ -61,12 +60,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py index 447260257d..1e987def63 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py @@ -1,12 +1,9 @@ -from datetime import datetime -from datetime import timedelta import logging from collections import defaultdict -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion -import dryable -from typing import Dict, List, DefaultDict -from qcrepocleaner.policies_utils import in_grace_period, group_versions +from typing import Dict, List, DefaultDict, Optional + from qcrepocleaner import policies_utils +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion logger = logging # default logger @@ -72,9 +69,9 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti # Dispatch the versions to deletion and preservation lists for bucket, run_versions in versions_buckets_dict.items(): # logger.debug(f"- bucket {bucket}") - sorted_run_versions = sorted(run_versions, key=lambda x: (x.created_at)) + sorted_run_versions = sorted(run_versions, key=lambda x: x.created_at) - freshest: ObjectVersion = None + freshest: Optional[ObjectVersion] = None for v in sorted_run_versions: if freshest is None: freshest = v @@ -115,14 +112,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} - - -def main(): - logger.getLogger().setLevel(int(10)) - dryable.set(True) - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "qc/testRunCleanup", 0) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py index 1047122e5f..99fb33c578 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py @@ -1,18 +1,17 @@ +import logging from datetime import datetime from datetime import timedelta -import logging -from typing import Dict +from typing import Dict, List, Optional from qcrepocleaner.Ccdb import Ccdb, ObjectVersion - logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. - + Only the last version of each object is preserved. Grace period is respected. :param ccdb: the ccdb in which objects are cleaned up. @@ -22,18 +21,18 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters (unused in this rule) :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin last_only processing {object_path}") versions = ccdb.get_versions_list(object_path) - earliest: ObjectVersion = None + earliest: Optional[ObjectVersion] = None preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] # find the earliest for v in versions: - if earliest == None or v.valid_from_as_dt > earliest.valid_from_as_dt: + if earliest is None or v.valid_from_as_dt > earliest.valid_from_as_dt: earliest = v logger.debug(f"earliest : {earliest}") @@ -63,12 +62,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted" : len(deletion_list), "preserved": len(preservation_list)} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py index 37cc0c7c9c..0d4fb8d5cb 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py @@ -1,20 +1,18 @@ +import logging from collections import defaultdict from datetime import datetime from datetime import timedelta -import logging -from typing import Dict, DefaultDict, List - -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion -from qcrepocleaner.policies_utils import in_grace_period, group_versions +from typing import Dict, DefaultDict, List, Optional from qcrepocleaner import policies_utils +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. Objects who have been created recently are spared (delay is expressed in minutes). @@ -62,7 +60,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters for this rule. :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin multiple_per_run processing {object_path}") @@ -109,7 +107,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t elif mw_deletion_delay != -1 and first_object.created_at_as_dt < datetime.now() - timedelta(minutes=mw_deletion_delay): # moving windows case logger.debug(f" after mw_deletion_delay period, delete this bucket") for v in run_versions: - if "/mw/" in v.path: # this is because we really don't want to take the risk of batch deleting non moving windows + if "/mw/" in v.path: # this is because we really don't want to take the risk of batch deleting non-moving windows logger.debug(f" deleting {v}") deletion_list.append(v) ccdb.delete_version(v) @@ -140,7 +138,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t ccdb.delete_version(run_versions[0]) del run_versions[0] - last_preserved: ObjectVersion = None + last_preserved: Optional[ObjectVersion] = None for v in run_versions: logger.debug(f"process {v}") @@ -171,12 +169,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py index c55e522b36..1385f316dc 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py @@ -1,16 +1,16 @@ import logging +from typing import Dict, List -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion from qcrepocleaner import policies_utils -from typing import Dict, List +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. - + This policy deletes everything after `delay` minutes. :param ccdb: the ccdb in which objects are cleaned up. @@ -20,7 +20,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters (unused in this rule) :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin 'none' processing {object_path}") @@ -50,11 +50,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": 0} - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py deleted file mode 100644 index 0975d80282..0000000000 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging - -from qcrepocleaner.Ccdb import Ccdb - - -def process(ccdb: Ccdb, object_path: str, delay: int): - ''' - This is the function one must implement. - - It should take care of deleting extraneous or outdated versions of the object - located at 'object_path'. The object ccdb is used to interact with the CCDB. - The 'delay' is here to specify a grace period during which no objects should be - deleted. - - :param ccdb: the ccdb in which objects are cleaned up. - :param object_path: path to the object, or pattern, to which a rule will apply. - :param delay: the grace period during which a new object is never deleted. - ''' - - logging.debug(f"Plugin XXX processing {object_path}") - - -# to quickly test, one can just run the plugin itself. -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py index 6913bcb9ee..40b0baf089 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py @@ -1,9 +1,9 @@ import logging -from typing import Dict, List, DefaultDict +import time from collections import defaultdict from datetime import datetime -import time -from datetime import timedelta, date +from datetime import timedelta +from typing import Dict, List, DefaultDict from qcrepocleaner.Ccdb import Ccdb, ObjectVersion diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py index 852ed1f3de..34042678ed 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py @@ -1,18 +1,15 @@ -from datetime import datetime -from datetime import timedelta import logging from typing import Dict -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion - +from qcrepocleaner.Ccdb import Ccdb logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. - + This policy does nothing and allows to preserve some directories. :param ccdb: the ccdb in which objects are cleaned up. @@ -22,17 +19,8 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters (unused in this rule) :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin 'skip' processing {object_path}") return {"deleted" : 0, "preserved": 0} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/tests/config-test.yaml b/Framework/script/RepoCleaner/tests/config-test.yaml index efefa91da7..dbb181a4ce 100644 --- a/Framework/script/RepoCleaner/tests/config-test.yaml +++ b/Framework/script/RepoCleaner/tests/config-test.yaml @@ -11,4 +11,4 @@ Rules: delete_when_no_run: True Ccdb: - Url: http://ccdb-test.cern.ch:8080 + Url: http://128.142.249.62:8080 diff --git a/Framework/script/RepoCleaner/tests/test_1_per_hour.py b/Framework/script/RepoCleaner/tests/test_1_per_hour.py index 313584816c..cae12fd26f 100644 --- a/Framework/script/RepoCleaner/tests/test_1_per_hour.py +++ b/Framework/script/RepoCleaner/tests/test_1_per_hour.py @@ -54,12 +54,12 @@ def test_1_per_hour(self): def test_1_per_hour_period(self): """ - 120 versions - no grace period - period of acceptance: 1 hour in the middle + 120 versions. + no grace period. + period of acceptance: 1 hour in the middle. We have therefore 60 versions in the acceptance period. Only 1 of them, the one 1 hour after the first version in the set, will be preserved, the others are deleted. - Thus we have 59 deletion. Everything outside the acceptance period is kept. + Thus, we have 59 deletion. Everything outside the acceptance period is kept. """ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') diff --git a/Framework/script/RepoCleaner/tests/test_Ccdb.py b/Framework/script/RepoCleaner/tests/test_Ccdb.py index be40ee37a2..b01f8f5beb 100644 --- a/Framework/script/RepoCleaner/tests/test_Ccdb.py +++ b/Framework/script/RepoCleaner/tests/test_Ccdb.py @@ -12,9 +12,9 @@ class TestCcdb(unittest.TestCase): def setUp(self): with open('objectsList.json') as f: # will close() when we leave this block - self.content_objectslist = f.read() + self.content_objects_list = f.read() with open('versionsList.json') as f: # will close() when we leave this block - self.content_versionslist = f.read() + self.content_versions_list = f.read() self.ccdb = Ccdb(CCDB_TEST_URL) logging.getLogger().setLevel(logging.DEBUG) @@ -22,7 +22,7 @@ def setUp(self): def test_getObjectsList(self): # Prepare mock response responses.add(responses.GET, CCDB_TEST_URL + '/latest/.*', - self.content_objectslist, status=200) + self.content_objects_list, status=200) # get list of objects objects_list = self.ccdb.get_objects_list() print(f"{objects_list}") @@ -34,8 +34,8 @@ def test_getObjectsList(self): def test_getVersionsList(self): # Prepare mock response object_path='asdfasdf/example' - responses.add(responses.GET, CCDB_TEST_URL + '/browse/'+object_path, - self.content_versionslist, status=200) + responses.add(responses.GET, CCDB_TEST_URL + '/browse/' + object_path, + self.content_versions_list, status=200) # get versions for object versions_list: List[ObjectVersion] = self.ccdb.get_versions_list(object_path) print(f"{versions_list}") diff --git a/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py b/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py index 71c2835f73..61b92c5867 100644 --- a/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py +++ b/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py @@ -38,18 +38,18 @@ def test_1_finished_run(self): test_path = self.path + "/test_1_finished_run" test_utils.clean_data(self.ccdb, test_path) test_utils.prepare_data(self.ccdb, test_path, [150], [22*60], 123) - objectsBefore = self.ccdb.get_versions_list(test_path) + objects_before = self.ccdb.get_versions_list(test_path) stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, to_timestamp=self.in_ten_years, extra_params=self.extra) - objectsAfter = self.ccdb.get_versions_list(test_path) + objects_after = self.ccdb.get_versions_list(test_path) self.assertEqual(stats["deleted"], 147) self.assertEqual(stats["preserved"], 3) self.assertEqual(stats["updated"], 0) - self.assertEqual(objectsAfter[0].valid_from, objectsBefore[1].valid_from) - self.assertEqual(objectsAfter[2].valid_from, objectsBefore[-2].valid_from) + self.assertEqual(objects_after[0].valid_from, objects_before[1].valid_from) + self.assertEqual(objects_after[2].valid_from, objects_before[-2].valid_from) def test_2_runs(self): """ diff --git a/Framework/script/RepoCleaner/tests/test_repoCleaner.py b/Framework/script/RepoCleaner/tests/test_repoCleaner.py index a3a518630b..28d60b3042 100644 --- a/Framework/script/RepoCleaner/tests/test_repoCleaner.py +++ b/Framework/script/RepoCleaner/tests/test_repoCleaner.py @@ -7,6 +7,8 @@ import yaml +from test_utils import CCDB_TEST_URL + def import_path(path): # needed because o2-qc-repo-cleaner has no suffix module_name = os.path.basename(path).replace('-', '_') @@ -20,16 +22,16 @@ def import_path(path): # needed because o2-qc-repo-cleaner has no suffix return module repoCleaner = import_path("../qcrepocleaner/o2-qc-repo-cleaner") -parseConfig = repoCleaner.parseConfig +parse_config = repoCleaner.parse_config Rule = repoCleaner.Rule -findMatchingRules = repoCleaner.findMatchingRules +find_matching_rules = repoCleaner.find_matching_rules class TestRepoCleaner(unittest.TestCase): def test_parseConfig(self): - args = parseConfig("config-test.yaml") - self.assertEqual(args['ccdb_url'], "http://ccdb-test.cern.ch:8080") + args = parse_config("config-test.yaml") + self.assertEqual(args['ccdb_url'], CCDB_TEST_URL) rules = args['rules'] self.assertEqual(len(rules), 3) @@ -51,25 +53,22 @@ def test_parseConfigFault(self): f.write(document) f.close() with self.assertRaises(yaml.YAMLError): - parseConfig(destination) + parse_config(destination) # now remove the faulty 2 lines at the beginning - with open(destination, 'w+') as fout: - fout.writelines(document.splitlines(True)[3:]) + with open(destination, 'w+') as out: + out.writelines(document.splitlines(True)[3:]) with self.assertRaises(KeyError): - parseConfig(destination) + parse_config(destination) def test_findMatchingRule(self): - rules = [] - rules.append(Rule('task1/obj1', '120', 'policy1')) - rules.append(Rule('task1/obj1', '120', 'policy2')) - rules.append(Rule('task2/.*', '120', 'policy3')) - self.assertEqual(findMatchingRules(rules, 'task1/obj1')[0].policy, 'policy1') - self.assertNotEqual(findMatchingRules(rules, 'task1/obj1')[0].policy, 'policy2') - self.assertEqual(findMatchingRules(rules, 'task3/obj1'), []) - self.assertEqual(findMatchingRules(rules, 'task2/obj1/obj1')[0].policy, 'policy3') + rules = [Rule('task1/obj1', '120', 'policy1'), Rule('task1/obj1', '120', 'policy2'), Rule('task2/.*', '120', 'policy3')] + self.assertEqual(find_matching_rules(rules, 'task1/obj1')[0].policy, 'policy1') + self.assertNotEqual(find_matching_rules(rules, 'task1/obj1')[0].policy, 'policy2') + self.assertEqual(find_matching_rules(rules, 'task3/obj1'), []) + self.assertEqual(find_matching_rules(rules, 'task2/obj1/obj1')[0].policy, 'policy3') rules.append(Rule('.*', '0', 'policyAll')) - self.assertEqual(findMatchingRules(rules, 'task3/obj1')[0].policy, 'policyAll') + self.assertEqual(find_matching_rules(rules, 'task3/obj1')[0].policy, 'policyAll') if __name__ == '__main__':