HPCC-20686 Add regression suite option to delete workunits at end of run

Signed-off-by: Goutami Sooda <[email protected]>

testing/regress/cleanupReadme.md

@@ -0,0 +1,59 @@
+# Cleanup Parameter of Regression Suite run and query sub-command
+
+The cleanup parameter has been introduced to allow the user to automatically delete the workunits created by executing the Regression Suite on their local system.
+
+It is an optional argument of the run and query sub-command.
+
+A custom logging system also creates log files for each execution of the run and query sub-command that contains information about the workunit deletion. 
+
+### Command:
+> ./ecl-test run --cleanup [mode]
+
+> ./ecl-test query --cleanup [mode]
+
+Modes allowed are ‘workunits’, ‘passed’. Default is ‘none’. 
+
+- workunits - all passed and failed workunits are deleted.
+
+- passed - only the passed workunits of the queries executed are deleted.  
+
+- none - no workunits created during the current run command are deleted.  
+
+### Result:
+#### The sample terminal output for hthor target:
+
+>./ecl-test query ECL_query action1.ecl action2.ecl action4.ecl action5.ecl -t hthor --cleanup workunits
+
+[Action] Suite: hthor  
+[Action] Queries: 4  
+[Action]   1. Test: action1.ecl  
+[Pass]   1. Pass action1.ecl - W20240526-094322 (2 sec)  
+[Pass]   1. URL http://127.0.0.1:8010/?Widget=WUDetailsWidget&Wuid=W20240526-094322  
+[Action]   2. Test: action2.ecl  
+[Pass]   2. Pass action2.ecl - W20240526-094324 (2 sec)  
+[Pass]   2. URL http://127.0.0.1:8010/?Widget=WUDetailsWidget&Wuid=W20240526-094324  
+[Action]  3. Test: action4.ecl  
+[Pass]  3. Pass action4.ecl - W20240526-094325 (2 sec)  
+[Pass]  3. URL http://127.0.0.1:8010/?Widget=WUDetailsWidget&Wuid=W20240526-094325  
+[Action]   4. Test: action5.ecl  
+[Pass]   4. Pass action5.ecl - W20240526-094327 (2 sec)  
+[Pass]   4. URL http://127.0.0.1:8010/?Widget=WUDetailsWidget&Wuid=W20240526-094327  
+[Action]     
+    -------------------------------------------------  
+    Result:  
+    Passing: 4     
+    Failure: 0  
+    -------------------------------------------------  
+    Log: /root/HPCCSystems-regression/log/hthor.24-05-26-09-43-22.log  
+    -------------------------------------------------  
+    Elapsed time: 11 sec  (00:00:11)   
+    -------------------------------------------------  
+
+[Action] Automatic Cleanup Routine  
+
+[Pass] 1. Workunit Wuid=W20240526-094322 deleted successfully.  
+[Pass] 2. Workunit Wuid=W20240526-094324 deleted successfully.  
+[Failure] 3. Failed to delete Wuid=W20240526-094325. URL: http://127.0.0.1:8010/?Widget=WUDetailsWidget&Wuid=W20240526-094325  
+            Failed cannot open workunit  Wuid=W20240526-094325.. Response status code: 200  
+[Pass] 4. Workunit Wuid=W20240526-094327 deleted successfully.  
+Suite destructor.  

testing/regress/ecl-test

@@ -28,7 +28,9 @@ import inspect
 import signal
 import argparse
 
+from datetime import datetime
 from hpcc.regression.regress import Regression
+from hpcc.regression import cleanup
 from hpcc.util.util import checkClusters, checkHpccStatus
 from hpcc.util.util import setConfig, checkPqParam, getVersionNumbers, checkXParam, convertPath, isPositiveIntNum
 from hpcc.util.util import getEclRunArgs, isSudoer, getCodeInfo
@@ -273,9 +275,10 @@ class RegressMain:
         parser_run.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
                                 action='store_true')
         parser_run.add_argument('--handleEclccWarningFile', '-w', help="Create/overwrite/delete ECLCC warning file.",
-                                action='store_true')
-
-
+                                action='store_true')                                                   
+        parser_run.add_argument('--cleanup', choices=['workunits', 'passed', 'none'], help="Select the cleanup mode: none(default)/workunits/passed",
+                                nargs='?', type=str, metavar="workunits | passed | none", default='none')
+
         self.parser_query = subparsers.add_parser('query', help='Query help',  parents=[helperParser, commonParser, executionParser])
         self.parser_query.set_defaults(func='query')
         self.parser_query.add_argument('query', help="One or more ECL file(s). It can contain wildcards. (mandatory).",
@@ -285,9 +288,10 @@ class RegressMain:
         self.parser_query.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
                                 action='store_true')
         self.parser_query.add_argument('--handleEclccWarningFile', '-w', help="Create/overwrite/delete ECLCC warning file.",
-                                action='store_true')
-
-
+                                action='store_true')  
+        self.parser_query.add_argument('--cleanup', choices=['workunits', 'passed', 'none'], help="Select the cleanup mode: none(default)/workunits/passed",
+                                nargs='?', type=str, metavar="workunits | passed | none", default='none')                             
+
         try:
             self.args = parser.parse_args()
         except Error as e:
@@ -470,15 +474,26 @@ class RegressMain:
             else:
                 self.args.excludeFileSet = set()
 
-
             if self.args.func == 'list':
                 self.listClusters()
             elif self.args.func == 'query':
+                startTime = datetime.now()
                 self.query()
             elif self.args.func == 'setup':
                 self.setup()
             elif self.args.func == 'run':
+                startTime = datetime.now()
                 self.run()
+
+            #Process automatic deletion of workunits after test
+            if 'cleanup' in self.args:
+                cleanupMode = self.args.cleanup
+                if(cleanupMode == 'none'):
+                    logging.debug("Automatic cleanup has not been enabled.")       	    
+                else:
+                    logging.warning("Automatic Cleanup Routine\n")
+                    cleanup.doCleanup(self.config.logDir, cleanupMode, startTime)
+
         except Error as e:
             logging.critical(e)
             logging.critical(traceback.format_exc())
@@ -498,7 +513,6 @@ class RegressMain:
                 self.regress = None
         exit()
 
-
 def signalHandler(signum, frame):
     logging.critical("Signal %d received" %(signum))
     raise KeyboardInterrupt
@@ -508,4 +522,4 @@ if __name__ == "__main__":
     signal.signal(signal.SIGTERM, signalHandler)
 
     regressMain = RegressMain()
-    regressMain.main()
+    regressMain.main()

testing/regress/hpcc/regression/cleanup.py

@@ -0,0 +1,101 @@
+import logging
+import os
+import requests
+import time
+import glob
+from datetime import datetime,  timedelta
+
+# Configure logger
+logger = logging.getLogger('cleanup') 
+
+# Custom logger formatter for serial numbering of log records
+class SerialNumberFormatter(logging.Formatter):
+    def __init__(self):
+        super().__init__()
+        self.serialNumber = 0
+
+    def format(self, record):
+        self.serialNumber += 1
+        record.msg = str(self.serialNumber) + ". " + record.msg
+        return super().format(record)
+
+# Builds and configures the cleanup logger
+def buildCleanupLogger(logDir, cleanupLogger):
+    cleanupLogger.setLevel(logging.INFO)
+    curTime = time.strftime('%y-%m-%d-%H-%M-%S')
+    logName = "cleanup" + "." + curTime + ".log"
+    logPath = os.path.join(logDir, logName)
+    cleanupHandler = logging.FileHandler(logPath)
+    cleanupHandler.setFormatter(SerialNumberFormatter())
+    cleanupLogger.addHandler(cleanupHandler)
+    return cleanupLogger
+
+# Extracts regress log files corresponding to current run of RTE
+def getRegressLogs(mode, logDirPath, startTime):
+    requiredStrings = ['thor', 'roxie', 'hthor', 'roxie-workunit']
+    exclusionStrings = ['exclusion']  
+
+    logDirPath = logDirPath + "/*.%02d-%02d-%02d-*.log"
+    logDirPathStartDay = logDirPath % (startTime.year-2000,  startTime.month,  startTime.day)
+    fileNames = glob.glob(logDirPathStartDay)
+
+    nextDay = startTime + timedelta(days = 1)
+    logDirPathNextDay = logDirPath % (nextDay.year-2000,  nextDay.month,  nextDay.day)
+    fileNames += glob.glob(logDirPathNextDay)
+
+    for fileName in fileNames:
+        if any(string in fileName for string in exclusionStrings):
+            continue
+
+        if any(string in fileName for string in requiredStrings):
+            fileTime = datetime.fromtimestamp(os.path.getmtime(fileName))
+            if fileTime >= startTime:
+                getWorkunitDetails(fileName, mode)
+
+# Extracts workunit details from log file based on the cleanup mode
+def getWorkunitDetails(logFilePath, mode):
+    with open(logFilePath, 'r') as file:
+        lineList = file.readlines()
+        def extractUrl(line):
+            index = line.find("URL")
+            if index != -1:
+                return line[index + 3:-1]
+            else:
+                return None
+        if mode == "workunits":
+            for line in lineList:
+                url = extractUrl(line) 
+                if url:
+                    deleteWorkunit(url)
+        elif mode == "passed":
+            for i, line in enumerate(lineList):
+                if "pass" in line.lower():
+                    nextLine = lineList[i + 1]
+                    url = extractUrl(nextLine)
+                    if url:
+                        deleteWorkunit(url)
+
+# Deletes workunits
+def deleteWorkunit(url):
+    deletionUrl = url.replace("?Widget=WUDetailsWidget&Wuid", "WsWorkunits/WUDelete.json?Wuids")
+    startIndex = url.find('Wuid=')
+    wuid = url[startIndex:]
+    try:
+        response = requests.post(deletionUrl)
+        jsonResponse = response.json()
+        if response.status_code == 200:
+            if jsonResponse.get("WUDeleteResponse") == {}:
+                logger.info("Workunit %s deleted successfully.", wuid)
+            elif "ActionResults" in jsonResponse["WUDeleteResponse"]:
+                errorMessage = jsonResponse["WUDeleteResponse"]["ActionResults"]["WUActionResult"][0]["Result"]
+                logger.error("Failed to delete workunit %s.\n   URL:%s\n   %s Response status code: %d", wuid, url, errorMessage, response.status_code)
+        else:
+            logger.error("Failed to delete workunit %s.\n   URL:%s\n   Response status code: %d", wuid, url, response.status_code)
+    except requests.exceptions.RequestException as e:
+        logger.error("Error occurred while deleting workunit %s: %s.\n   URL: %s", wuid, str(e), url)
+
+# Main function to instantiate custom logger and initiate workunit detail extraction
+def doCleanup(logDir, cleanupMode, startTime):  
+    logDirPath = os.path.expanduser(logDir)
+    buildCleanupLogger(logDirPath, logger)
+    getRegressLogs(cleanupMode, logDirPath, startTime)