- rename according to common python conventions

- fix warnings
- organize imports

Framework/script/RepoCleaner/qcrepocleaner/Ccdb.py

@@ -159,7 +159,7 @@ def delete_version(self, version: ObjectVersion):
             r = requests.delete(url_delete, headers=headers)
             r.raise_for_status()
             self.counter_deleted += 1
-        except requests.exceptions.RequestException as e:
+        except requests.exceptions.RequestException:
             logging.error(f"Exception in deleteVersion: {traceback.format_exc()}")
 
     @dryable.Dryable()
@@ -176,7 +176,7 @@ def move_version(self, version: ObjectVersion, to_path: str):
             r = requests.request("MOVE", url_move, headers=headers)
             r.raise_for_status()
             self.counter_deleted += 1
-        except requests.exceptions.RequestException as e:
+        except requests.exceptions.RequestException:
             logging.error(f"Exception in moveVersion: {traceback.format_exc()}")
 
     @dryable.Dryable()
@@ -202,7 +202,7 @@ def update_validity(self, version: ObjectVersion, valid_from: int, valid_to: int
             r = requests.put(full_path, headers=headers)
             r.raise_for_status()
             self.counter_validity_updated += 1
-        except requests.exceptions.RequestException as e:  
+        except requests.exceptions.RequestException:
             logging.error(f"Exception in updateValidity: {traceback.format_exc()}")
 
     @dryable.Dryable()
@@ -219,7 +219,7 @@ def update_metadata(self, version: ObjectVersion, metadata):
             headers = {'Connection': 'close'}
             r = requests.put(full_path, headers=headers)
             r.raise_for_status()
-        except requests.exceptions.RequestException as e:
+        except requests.exceptions.RequestException:
             logging.error(f"Exception in updateMetadata: {traceback.format_exc()}")
 
     @dryable.Dryable()

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-cleaner

@@ -277,6 +277,7 @@ def store_crucial_log(message):
     try:
         f = open(logs_filename, "a")
         f.write(message+"\n")
+        f.close()
     except IOError as e:
         logging.error(f"Could not write crucial log to {logs_filename} : {e}")
 

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
+import csv
+import logging
+
 import dryable
-from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
+
+from qcrepocleaner.Ccdb import Ccdb
 from qcrepocleaner.binUtils import prepare_main_logger
-import csv
 
 
 def parseArgs():

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
+import logging
+
 import dryable
+
 from qcrepocleaner.Ccdb import Ccdb
 from qcrepocleaner.binUtils import prepare_main_logger
 

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
+import csv
+import logging
+
 import dryable
-import time
-from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
+
+from qcrepocleaner.Ccdb import Ccdb
 from qcrepocleaner.binUtils import prepare_main_logger
-import csv
 
 
 def parseArgs():

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
+import logging
+
 import dryable
+
 from qcrepocleaner.Ccdb import Ccdb
 from qcrepocleaner.binUtils import prepare_main_logger
 

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-less-versions-than

@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
-import time
+import logging
 
 from qcrepocleaner import binUtils
 from qcrepocleaner.Ccdb import Ccdb
-import sys
-import datetime
 
-def parseArgs():
+
+def parse_args():
     """Parse the arguments passed to the script."""
     logging.info("Parsing arguments")
     parser = argparse.ArgumentParser(description='Identify the objects that have less than X versions.')
@@ -57,7 +55,7 @@ def main():
     binUtils.prepare_main_logger()
 
     # Parse arguments
-    args = parseArgs()
+    args = parse_args()
     logging.getLogger().setLevel(int(args.log_level))
 
     run(args)

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-find-objects-not-updated

@@ -1,15 +1,14 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
-import time
+import datetime
+import logging
 
 from qcrepocleaner import binUtils
 from qcrepocleaner.Ccdb import Ccdb
-import sys
-import datetime
 
-def parseArgs():
+
+def parse_args():
     """Parse the arguments passed to the script."""
     logging.info("Parsing arguments")
     parser = argparse.ArgumentParser(description='Identify the objects that have not seen an update for a given '
@@ -67,7 +66,7 @@ def main():
     binUtils.prepare_main_logger()
 
     # Parse arguments
-    args = parseArgs()
+    args = parse_args()
     logging.getLogger().setLevel(int(args.log_level))
 
     run(args)

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
+import logging
+import re
+
 import dryable
+
 from qcrepocleaner.Ccdb import Ccdb
 from qcrepocleaner.binUtils import prepare_main_logger
-import re
 
 
 def parse_args():

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-update-run-type

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
-import logging
 import argparse
+import csv
+import logging
+
 import dryable
+
 from qcrepocleaner.Ccdb import Ccdb
 from qcrepocleaner.binUtils import prepare_main_logger
-import csv
 
 
 def parse_args():

Framework/script/RepoCleaner/qcrepocleaner/policies_utils.py

@@ -1,9 +1,9 @@
+import logging
 from datetime import datetime
 from datetime import timedelta
-import logging
 from typing import DefaultDict, List
 
-from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
+from qcrepocleaner.Ccdb import ObjectVersion
 
 logger = logging  # default logger
 

Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_hour.py

@@ -1,20 +1,19 @@
+import logging
 from datetime import datetime
 from datetime import timedelta
-import logging
-from typing import Dict
+from typing import Dict, List, Optional
 
 from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
 
-
 logger = logging  # default logger
 
 
 def process(ccdb: Ccdb, object_path: str, delay: int,  from_timestamp: int, to_timestamp: int,
             extra_params: Dict[str, str]):
-    '''
+    """
     Process this deletion rule on the object. We use the CCDB passed by argument.
     Objects who have been created recently are spared (delay is expressed in minutes).
-    This specific policy, 1_per_hour, operates like this : take the first record, 
+    This specific policy, 1_per_hour, operates like this : take the first record,
     delete everything for the next hour, find the next one and loop.
 
     :param ccdb: the ccdb in which objects are cleaned up.
@@ -24,13 +23,13 @@ def process(ccdb: Ccdb, object_path: str, delay: int,  from_timestamp: int, to_t
     :param to_timestamp: only objects created before this timestamp are considered.
     :param extra_params: a dictionary containing extra parameters for this rule.
     :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved.
-    '''
+    """
 
     logger.debug(f"Plugin 1_per_hour processing {object_path}")
 
     versions = ccdb.get_versions_list(object_path)
 
-    last_preserved: ObjectVersion = None
+    last_preserved: Optional[ObjectVersion] = None
     preservation_list: List[ObjectVersion] = []
     deletion_list: List[ObjectVersion] = []
     update_list: List[ObjectVersion] = []
@@ -61,12 +60,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int,  from_timestamp: int, to_t
         logger.debug(f"   {v}")
 
     return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)}
-
-
-def main():
-    ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
-    process(ccdb, "asdfasdf/example", 60)
-
-
-if __name__ == "__main__":  # to be able to run the test code above when not imported.
-    main()

Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py

@@ -1,12 +1,9 @@
-from datetime import datetime
-from datetime import timedelta
 import logging
 from collections import defaultdict
-from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
-import dryable
-from typing import Dict, List, DefaultDict
-from qcrepocleaner.policies_utils import in_grace_period, group_versions
+from typing import Dict, List, DefaultDict, Optional
+
 from qcrepocleaner import policies_utils
+from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
 
 logger = logging  # default logger
 
@@ -72,9 +69,9 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti
     # Dispatch the versions to deletion and preservation lists
     for bucket, run_versions in versions_buckets_dict.items():
         # logger.debug(f"- bucket {bucket}")
-        sorted_run_versions = sorted(run_versions, key=lambda x: (x.created_at))
+        sorted_run_versions = sorted(run_versions, key=lambda x: x.created_at)
 
-        freshest: ObjectVersion = None
+        freshest: Optional[ObjectVersion] = None
         for v in sorted_run_versions:
             if freshest is None:
                 freshest = v
@@ -115,14 +112,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_ti
         logger.debug(f"   {v}")
 
     return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)}
-
-
-def main():
-    logger.getLogger().setLevel(int(10))
-    dryable.set(True)
-    ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
-    process(ccdb, "qc/testRunCleanup", 0)
-
-
-if __name__ == "__main__":  # to be able to run the test code above when not imported.
-    main()

Framework/script/RepoCleaner/qcrepocleaner/rules/last_only.py

@@ -1,18 +1,17 @@
+import logging
 from datetime import datetime
 from datetime import timedelta
-import logging
-from typing import Dict
+from typing import Dict, List, Optional
 
 from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
 
-
 logger = logging  # default logger
 
 
 def process(ccdb: Ccdb, object_path: str, delay: int,  from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]):
-    '''
+    """
     Process this deletion rule on the object. We use the CCDB passed by argument.
-    
+
     Only the last version of each object is preserved. Grace period is respected.
 
     :param ccdb: the ccdb in which objects are cleaned up.
@@ -22,18 +21,18 @@ def process(ccdb: Ccdb, object_path: str, delay: int,  from_timestamp: int, to_t
     :param to_timestamp: only objects created before this timestamp are considered.
     :param extra_params: a dictionary containing extra parameters (unused in this rule)
     :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved.
-    '''
+    """
 
     logger.debug(f"Plugin last_only processing {object_path}")
 
     versions = ccdb.get_versions_list(object_path)
 
-    earliest: ObjectVersion = None
+    earliest: Optional[ObjectVersion] = None
     preservation_list: List[ObjectVersion] = []
     deletion_list: List[ObjectVersion] = []
     # find the earliest
     for v in versions:
-        if earliest == None or v.valid_from_as_dt > earliest.valid_from_as_dt:
+        if earliest is None or v.valid_from_as_dt > earliest.valid_from_as_dt:
             earliest = v
     logger.debug(f"earliest : {earliest}")
 
@@ -63,12 +62,3 @@ def process(ccdb: Ccdb, object_path: str, delay: int,  from_timestamp: int, to_t
         logger.debug(f"   {v}")
 
     return {"deleted" : len(deletion_list), "preserved": len(preservation_list)}
-
-
-def main():
-    ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
-    process(ccdb, "asdfasdf/example", 60)
-
-
-if __name__ == "__main__":  # to be able to run the test code above when not imported.
-    main()