From 4a13d48f0bae776a99276aaa3c5b88b67b7bdae9 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 7 Nov 2024 14:15:48 +0100 Subject: [PATCH] - rename according to common python conventions - fix warnings - organize imports --- .../script/RepoCleaner/qcrepocleaner/Ccdb.py | 8 ++--- .../qcrepocleaner/o2-qc-repo-cleaner | 1 + .../o2-qc-repo-delete-not-in-runs | 8 +++-- .../qcrepocleaner/o2-qc-repo-delete-objects | 4 ++- .../o2-qc-repo-delete-objects-in-runs | 9 ++--- .../o2-qc-repo-delete-time-interval | 4 ++- ...o2-qc-repo-find-objects-less-versions-than | 10 +++--- .../o2-qc-repo-find-objects-not-updated | 11 +++---- .../qcrepocleaner/o2-qc-repo-move-objects | 6 ++-- .../qcrepocleaner/o2-qc-repo-update-run-type | 6 ++-- .../qcrepocleaner/policies_utils.py | 4 +-- .../qcrepocleaner/rules/1_per_hour.py | 22 ++++--------- .../qcrepocleaner/rules/1_per_run.py | 24 +++----------- .../qcrepocleaner/rules/last_only.py | 24 ++++---------- .../qcrepocleaner/rules/multiple_per_run.py | 25 ++++---------- .../qcrepocleaner/rules/none_kept.py | 18 +++------- .../qcrepocleaner/rules/plugin_template.py | 30 ----------------- .../qcrepocleaner/rules/production.py | 6 ++-- .../RepoCleaner/qcrepocleaner/rules/skip.py | 22 +++---------- .../script/RepoCleaner/tests/config-test.yaml | 2 +- .../RepoCleaner/tests/test_1_per_hour.py | 8 ++--- .../script/RepoCleaner/tests/test_Ccdb.py | 10 +++--- .../test_MultiplePerRun_deleteFirstLast.py | 8 ++--- .../RepoCleaner/tests/test_repoCleaner.py | 33 +++++++++---------- 24 files changed, 108 insertions(+), 195 deletions(-) delete mode 100644 Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py diff --git a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py index 3d48202d87..a74c13cb58 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py @@ -159,7 +159,7 @@ def delete_version(self, version: ObjectVersion): r = requests.delete(url_delete, headers=headers) r.raise_for_status() self.counter_deleted += 1 - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in deleteVersion: {traceback.format_exc()}") @dryable.Dryable() @@ -176,7 +176,7 @@ def move_version(self, version: ObjectVersion, to_path: str): r = requests.request("MOVE", url_move, headers=headers) r.raise_for_status() self.counter_deleted += 1 - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in moveVersion: {traceback.format_exc()}") @dryable.Dryable() @@ -202,7 +202,7 @@ def update_validity(self, version: ObjectVersion, valid_from: int, valid_to: int r = requests.put(full_path, headers=headers) r.raise_for_status() self.counter_validity_updated += 1 - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in updateValidity: {traceback.format_exc()}") @dryable.Dryable() @@ -219,7 +219,7 @@ def update_metadata(self, version: ObjectVersion, metadata): headers = {'Connection': 'close'} r = requests.put(full_path, headers=headers) r.raise_for_status() - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: logging.error(f"Exception in updateMetadata: {traceback.format_exc()}") @dryable.Dryable() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner index 576f8c3558..f62425731c 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner @@ -277,6 +277,7 @@ def store_crucial_log(message): try: f = open(logs_filename, "a") f.write(message+"\n") + f.close() except IOError as e: logging.error(f"Could not write crucial log to {logs_filename} : {e}") diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs index 2779b24b81..bcb4645507 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import csv +import logging + import dryable -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion + +from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import csv def parseArgs(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects index 751c238cf8..6d69cc2754 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -import logging import argparse +import logging + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs index 6770dc5f3f..eafe9c043e 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import csv +import logging + import dryable -import time -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion + +from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import csv def parseArgs(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval index 0c305d7833..1c8623a0a1 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -import logging import argparse +import logging + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than index 18b96ca93e..bb605c77ea 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than @@ -1,15 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse -import time +import logging from qcrepocleaner import binUtils from qcrepocleaner.Ccdb import Ccdb -import sys -import datetime -def parseArgs(): + +def parse_args(): """Parse the arguments passed to the script.""" logging.info("Parsing arguments") parser = argparse.ArgumentParser(description='Identify the objects that have less than X versions.') @@ -57,7 +55,7 @@ def main(): binUtils.prepare_main_logger() # Parse arguments - args = parseArgs() + args = parse_args() logging.getLogger().setLevel(int(args.log_level)) run(args) diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated index 1c0c4455d1..a6184eff19 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated @@ -1,15 +1,14 @@ #!/usr/bin/env python3 -import logging import argparse -import time +import datetime +import logging from qcrepocleaner import binUtils from qcrepocleaner.Ccdb import Ccdb -import sys -import datetime -def parseArgs(): + +def parse_args(): """Parse the arguments passed to the script.""" logging.info("Parsing arguments") parser = argparse.ArgumentParser(description='Identify the objects that have not seen an update for a given ' @@ -67,7 +66,7 @@ def main(): binUtils.prepare_main_logger() # Parse arguments - args = parseArgs() + args = parse_args() logging.getLogger().setLevel(int(args.log_level)) run(args) diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects index 4a8b9a1ecb..79fc7b5a90 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import logging +import re + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import re def parse_args(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type index a9818e01af..9fbc75a166 100755 --- a/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type +++ b/Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import logging import argparse +import csv +import logging + import dryable + from qcrepocleaner.Ccdb import Ccdb from qcrepocleaner.binUtils import prepare_main_logger -import csv def parse_args(): diff --git a/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py b/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py index f19fb5360a..639cc3d30f 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py @@ -1,9 +1,9 @@ +import logging from datetime import datetime from datetime import timedelta -import logging from typing import DefaultDict, List -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion +from qcrepocleaner.Ccdb import ObjectVersion logger = logging # default logger diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py index 45fe940c11..9eb0b419a3 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py @@ -1,20 +1,19 @@ +import logging from datetime import datetime from datetime import timedelta -import logging -from typing import Dict +from typing import Dict, List, Optional from qcrepocleaner.Ccdb import Ccdb, ObjectVersion - logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. Objects who have been created recently are spared (delay is expressed in minutes). - This specific policy, 1_per_hour, operates like this : take the first record, + This specific policy, 1_per_hour, operates like this : take the first record, delete everything for the next hour, find the next one and loop. :param ccdb: the ccdb in which objects are cleaned up. @@ -24,13 +23,13 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters for this rule. :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin 1_per_hour processing {object_path}") versions = ccdb.get_versions_list(object_path) - last_preserved: ObjectVersion = None + last_preserved: Optional[ObjectVersion] = None preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] update_list: List[ObjectVersion] = [] @@ -61,12 +60,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py index 447260257d..1e987def63 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py @@ -1,12 +1,9 @@ -from datetime import datetime -from datetime import timedelta import logging from collections import defaultdict -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion -import dryable -from typing import Dict, List, DefaultDict -from qcrepocleaner.policies_utils import in_grace_period, group_versions +from typing import Dict, List, DefaultDict, Optional + from qcrepocleaner import policies_utils +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion logger = logging # default logger @@ -72,9 +69,9 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti # Dispatch the versions to deletion and preservation lists for bucket, run_versions in versions_buckets_dict.items(): # logger.debug(f"- bucket {bucket}") - sorted_run_versions = sorted(run_versions, key=lambda x: (x.created_at)) + sorted_run_versions = sorted(run_versions, key=lambda x: x.created_at) - freshest: ObjectVersion = None + freshest: Optional[ObjectVersion] = None for v in sorted_run_versions: if freshest is None: freshest = v @@ -115,14 +112,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} - - -def main(): - logger.getLogger().setLevel(int(10)) - dryable.set(True) - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "qc/testRunCleanup", 0) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py index 1047122e5f..99fb33c578 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py @@ -1,18 +1,17 @@ +import logging from datetime import datetime from datetime import timedelta -import logging -from typing import Dict +from typing import Dict, List, Optional from qcrepocleaner.Ccdb import Ccdb, ObjectVersion - logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. - + Only the last version of each object is preserved. Grace period is respected. :param ccdb: the ccdb in which objects are cleaned up. @@ -22,18 +21,18 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters (unused in this rule) :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin last_only processing {object_path}") versions = ccdb.get_versions_list(object_path) - earliest: ObjectVersion = None + earliest: Optional[ObjectVersion] = None preservation_list: List[ObjectVersion] = [] deletion_list: List[ObjectVersion] = [] # find the earliest for v in versions: - if earliest == None or v.valid_from_as_dt > earliest.valid_from_as_dt: + if earliest is None or v.valid_from_as_dt > earliest.valid_from_as_dt: earliest = v logger.debug(f"earliest : {earliest}") @@ -63,12 +62,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted" : len(deletion_list), "preserved": len(preservation_list)} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py index 37cc0c7c9c..0d4fb8d5cb 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/multiple_per_run.py @@ -1,20 +1,18 @@ +import logging from collections import defaultdict from datetime import datetime from datetime import timedelta -import logging -from typing import Dict, DefaultDict, List - -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion -from qcrepocleaner.policies_utils import in_grace_period, group_versions +from typing import Dict, DefaultDict, List, Optional from qcrepocleaner import policies_utils +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. Objects who have been created recently are spared (delay is expressed in minutes). @@ -62,7 +60,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters for this rule. :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin multiple_per_run processing {object_path}") @@ -109,7 +107,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t elif mw_deletion_delay != -1 and first_object.created_at_as_dt < datetime.now() - timedelta(minutes=mw_deletion_delay): # moving windows case logger.debug(f" after mw_deletion_delay period, delete this bucket") for v in run_versions: - if "/mw/" in v.path: # this is because we really don't want to take the risk of batch deleting non moving windows + if "/mw/" in v.path: # this is because we really don't want to take the risk of batch deleting non-moving windows logger.debug(f" deleting {v}") deletion_list.append(v) ccdb.delete_version(v) @@ -140,7 +138,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t ccdb.delete_version(run_versions[0]) del run_versions[0] - last_preserved: ObjectVersion = None + last_preserved: Optional[ObjectVersion] = None for v in run_versions: logger.debug(f"process {v}") @@ -171,12 +169,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py index c55e522b36..1385f316dc 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/none_kept.py @@ -1,16 +1,16 @@ import logging +from typing import Dict, List -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion from qcrepocleaner import policies_utils -from typing import Dict, List +from qcrepocleaner.Ccdb import Ccdb, ObjectVersion logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. - + This policy deletes everything after `delay` minutes. :param ccdb: the ccdb in which objects are cleaned up. @@ -20,7 +20,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters (unused in this rule) :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin 'none' processing {object_path}") @@ -50,11 +50,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t logger.debug(f" {v}") return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": 0} - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py deleted file mode 100644 index 0975d80282..0000000000 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/plugin_template.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging - -from qcrepocleaner.Ccdb import Ccdb - - -def process(ccdb: Ccdb, object_path: str, delay: int): - ''' - This is the function one must implement. - - It should take care of deleting extraneous or outdated versions of the object - located at 'object_path'. The object ccdb is used to interact with the CCDB. - The 'delay' is here to specify a grace period during which no objects should be - deleted. - - :param ccdb: the ccdb in which objects are cleaned up. - :param object_path: path to the object, or pattern, to which a rule will apply. - :param delay: the grace period during which a new object is never deleted. - ''' - - logging.debug(f"Plugin XXX processing {object_path}") - - -# to quickly test, one can just run the plugin itself. -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py index 6913bcb9ee..40b0baf089 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/production.py @@ -1,9 +1,9 @@ import logging -from typing import Dict, List, DefaultDict +import time from collections import defaultdict from datetime import datetime -import time -from datetime import timedelta, date +from datetime import timedelta +from typing import Dict, List, DefaultDict from qcrepocleaner.Ccdb import Ccdb, ObjectVersion diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py index 8767d13514..390c6f3fae 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/skip.py @@ -1,18 +1,15 @@ -from datetime import datetime -from datetime import timedelta import logging from typing import Dict -from qcrepocleaner.Ccdb import Ccdb, ObjectVersion - +from qcrepocleaner.Ccdb import Ccdb logger = logging # default logger def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' + """ Process this deletion rule on the object. We use the CCDB passed by argument. - + This policy does nothing and allows to preserve some directories. :param ccdb: the ccdb in which objects are cleaned up. @@ -22,19 +19,10 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters (unused in this rule) :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' + """ logger.debug(f"Plugin 'skip' processing {object_path}") versions = ccdb.getVersionsList(object_path) - return {"deleted": 0, "preserved": len(versions), "updated": 0} - - -def main(): - ccdb = Ccdb('http://ccdb-test.cern.ch:8080') - process(ccdb, "asdfasdf/example", 60) - - -if __name__ == "__main__": # to be able to run the test code above when not imported. - main() + return {"deleted": 0, "preserved": len(versions), "updated": 0} \ No newline at end of file diff --git a/Framework/script/RepoCleaner/tests/config-test.yaml b/Framework/script/RepoCleaner/tests/config-test.yaml index efefa91da7..dbb181a4ce 100644 --- a/Framework/script/RepoCleaner/tests/config-test.yaml +++ b/Framework/script/RepoCleaner/tests/config-test.yaml @@ -11,4 +11,4 @@ Rules: delete_when_no_run: True Ccdb: - Url: http://ccdb-test.cern.ch:8080 + Url: http://128.142.249.62:8080 diff --git a/Framework/script/RepoCleaner/tests/test_1_per_hour.py b/Framework/script/RepoCleaner/tests/test_1_per_hour.py index 313584816c..cae12fd26f 100644 --- a/Framework/script/RepoCleaner/tests/test_1_per_hour.py +++ b/Framework/script/RepoCleaner/tests/test_1_per_hour.py @@ -54,12 +54,12 @@ def test_1_per_hour(self): def test_1_per_hour_period(self): """ - 120 versions - no grace period - period of acceptance: 1 hour in the middle + 120 versions. + no grace period. + period of acceptance: 1 hour in the middle. We have therefore 60 versions in the acceptance period. Only 1 of them, the one 1 hour after the first version in the set, will be preserved, the others are deleted. - Thus we have 59 deletion. Everything outside the acceptance period is kept. + Thus, we have 59 deletion. Everything outside the acceptance period is kept. """ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') diff --git a/Framework/script/RepoCleaner/tests/test_Ccdb.py b/Framework/script/RepoCleaner/tests/test_Ccdb.py index be40ee37a2..b01f8f5beb 100644 --- a/Framework/script/RepoCleaner/tests/test_Ccdb.py +++ b/Framework/script/RepoCleaner/tests/test_Ccdb.py @@ -12,9 +12,9 @@ class TestCcdb(unittest.TestCase): def setUp(self): with open('objectsList.json') as f: # will close() when we leave this block - self.content_objectslist = f.read() + self.content_objects_list = f.read() with open('versionsList.json') as f: # will close() when we leave this block - self.content_versionslist = f.read() + self.content_versions_list = f.read() self.ccdb = Ccdb(CCDB_TEST_URL) logging.getLogger().setLevel(logging.DEBUG) @@ -22,7 +22,7 @@ def setUp(self): def test_getObjectsList(self): # Prepare mock response responses.add(responses.GET, CCDB_TEST_URL + '/latest/.*', - self.content_objectslist, status=200) + self.content_objects_list, status=200) # get list of objects objects_list = self.ccdb.get_objects_list() print(f"{objects_list}") @@ -34,8 +34,8 @@ def test_getObjectsList(self): def test_getVersionsList(self): # Prepare mock response object_path='asdfasdf/example' - responses.add(responses.GET, CCDB_TEST_URL + '/browse/'+object_path, - self.content_versionslist, status=200) + responses.add(responses.GET, CCDB_TEST_URL + '/browse/' + object_path, + self.content_versions_list, status=200) # get versions for object versions_list: List[ObjectVersion] = self.ccdb.get_versions_list(object_path) print(f"{versions_list}") diff --git a/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py b/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py index 71c2835f73..61b92c5867 100644 --- a/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py +++ b/Framework/script/RepoCleaner/tests/test_MultiplePerRun_deleteFirstLast.py @@ -38,18 +38,18 @@ def test_1_finished_run(self): test_path = self.path + "/test_1_finished_run" test_utils.clean_data(self.ccdb, test_path) test_utils.prepare_data(self.ccdb, test_path, [150], [22*60], 123) - objectsBefore = self.ccdb.get_versions_list(test_path) + objects_before = self.ccdb.get_versions_list(test_path) stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1, to_timestamp=self.in_ten_years, extra_params=self.extra) - objectsAfter = self.ccdb.get_versions_list(test_path) + objects_after = self.ccdb.get_versions_list(test_path) self.assertEqual(stats["deleted"], 147) self.assertEqual(stats["preserved"], 3) self.assertEqual(stats["updated"], 0) - self.assertEqual(objectsAfter[0].valid_from, objectsBefore[1].valid_from) - self.assertEqual(objectsAfter[2].valid_from, objectsBefore[-2].valid_from) + self.assertEqual(objects_after[0].valid_from, objects_before[1].valid_from) + self.assertEqual(objects_after[2].valid_from, objects_before[-2].valid_from) def test_2_runs(self): """ diff --git a/Framework/script/RepoCleaner/tests/test_repoCleaner.py b/Framework/script/RepoCleaner/tests/test_repoCleaner.py index a3a518630b..28d60b3042 100644 --- a/Framework/script/RepoCleaner/tests/test_repoCleaner.py +++ b/Framework/script/RepoCleaner/tests/test_repoCleaner.py @@ -7,6 +7,8 @@ import yaml +from test_utils import CCDB_TEST_URL + def import_path(path): # needed because o2-qc-repo-cleaner has no suffix module_name = os.path.basename(path).replace('-', '_') @@ -20,16 +22,16 @@ def import_path(path): # needed because o2-qc-repo-cleaner has no suffix return module repoCleaner = import_path("../qcrepocleaner/o2-qc-repo-cleaner") -parseConfig = repoCleaner.parseConfig +parse_config = repoCleaner.parse_config Rule = repoCleaner.Rule -findMatchingRules = repoCleaner.findMatchingRules +find_matching_rules = repoCleaner.find_matching_rules class TestRepoCleaner(unittest.TestCase): def test_parseConfig(self): - args = parseConfig("config-test.yaml") - self.assertEqual(args['ccdb_url'], "http://ccdb-test.cern.ch:8080") + args = parse_config("config-test.yaml") + self.assertEqual(args['ccdb_url'], CCDB_TEST_URL) rules = args['rules'] self.assertEqual(len(rules), 3) @@ -51,25 +53,22 @@ def test_parseConfigFault(self): f.write(document) f.close() with self.assertRaises(yaml.YAMLError): - parseConfig(destination) + parse_config(destination) # now remove the faulty 2 lines at the beginning - with open(destination, 'w+') as fout: - fout.writelines(document.splitlines(True)[3:]) + with open(destination, 'w+') as out: + out.writelines(document.splitlines(True)[3:]) with self.assertRaises(KeyError): - parseConfig(destination) + parse_config(destination) def test_findMatchingRule(self): - rules = [] - rules.append(Rule('task1/obj1', '120', 'policy1')) - rules.append(Rule('task1/obj1', '120', 'policy2')) - rules.append(Rule('task2/.*', '120', 'policy3')) - self.assertEqual(findMatchingRules(rules, 'task1/obj1')[0].policy, 'policy1') - self.assertNotEqual(findMatchingRules(rules, 'task1/obj1')[0].policy, 'policy2') - self.assertEqual(findMatchingRules(rules, 'task3/obj1'), []) - self.assertEqual(findMatchingRules(rules, 'task2/obj1/obj1')[0].policy, 'policy3') + rules = [Rule('task1/obj1', '120', 'policy1'), Rule('task1/obj1', '120', 'policy2'), Rule('task2/.*', '120', 'policy3')] + self.assertEqual(find_matching_rules(rules, 'task1/obj1')[0].policy, 'policy1') + self.assertNotEqual(find_matching_rules(rules, 'task1/obj1')[0].policy, 'policy2') + self.assertEqual(find_matching_rules(rules, 'task3/obj1'), []) + self.assertEqual(find_matching_rules(rules, 'task2/obj1/obj1')[0].policy, 'policy3') rules.append(Rule('.*', '0', 'policyAll')) - self.assertEqual(findMatchingRules(rules, 'task3/obj1')[0].policy, 'policyAll') + self.assertEqual(find_matching_rules(rules, 'task3/obj1')[0].policy, 'policyAll') if __name__ == '__main__':