From 09fb4dd5e46bee7765f42980e1f7798b8889c09d Mon Sep 17 00:00:00 2001
From: Soren Rasmussen <s.c.rasmussen@gmail.com>
Date: Fri, 26 Jul 2024 12:55:26 -0600
Subject: [PATCH] Print summary of failed cases and exit it with an error if a
 failed cases exists Continue docker run so report of all failed tests gets
 produced.

---
 .github/workflows/ci_test_docker.yml |  2 +-
 scm/src/run_scm.py                   | 54 +++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml
index 730a5381c..b1dd5e747 100644
--- a/.github/workflows/ci_test_docker.yml
+++ b/.github/workflows/ci_test_docker.yml
@@ -28,4 +28,4 @@ jobs:
         run: |
           mkdir $HOME/output
           chmod a+rw $HOME/output
-          docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d --stop_on_error
+          docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d
diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py
index 40c658843..c0febaa48 100755
--- a/scm/src/run_scm.py
+++ b/scm/src/run_scm.py
@@ -3,6 +3,7 @@
 import argparse
 import f90nml
 import logging
+import numpy as np
 import os
 import re
 import shutil
@@ -156,17 +157,18 @@ def execute(cmd, ignore_error = False):
                          stderr = subprocess.PIPE, shell = True)
     (stdout, stderr) = p.communicate()
     status = p.returncode
+    message = 'Execution of "{0}" returned with exit code {1}\n'.format(cmd, status)
+    message += '    stdout: "{0}"\n'.format(stdout.decode(encoding='ascii', errors='ignore').rstrip('\n'))
+    message += '    stderr: "{0}"'.format(stderr.decode(encoding='ascii', errors='ignore').rstrip('\n'))
+
     if status == 0:
-        message = 'Execution of "{0}" returned with exit code {1}\n'.format(cmd, status)
-        message += '    stdout: "{0}"\n'.format(stdout.decode(encoding='ascii', errors='ignore').rstrip('\n'))
-        message += '    stderr: "{0}"'.format(stderr.decode(encoding='ascii', errors='ignore').rstrip('\n'))
         logging.debug(message)
     elif not ignore_error:
-        message = 'Execution of command "{0}" failed, exit code {1}\n'.format(cmd, status)
-        message += '    stdout: "{0}"\n'.format(stdout.decode(encoding='ascii', errors='ignore').rstrip('\n'))
-        message += '    stderr: "{0}"'.format(stderr.decode(encoding='ascii', errors='ignore').rstrip('\n'))
         logging.critical(message)
         raise Exception('Execution of command "{0}" failed, exit code {1}\n'.format(cmd, status))
+    else:
+        print("SHOULD PRINT ERROR MESSAGE: status and ignore error ==", status, ignore_error)
+        logging.error(message)
     return (status, stdout.decode(encoding='ascii', errors='ignore').rstrip('\n'), stderr.decode(encoding='ascii', errors='ignore').rstrip('\n'))
 
 def parse_arguments():
@@ -765,6 +767,30 @@ def copy_outdir(exp_dir):
         shutil.rmtree(home_output_dir)
     shutil.copytree(exp_dir, home_output_dir)
 
+def print_error_report(error_logs, total_count):
+    case_l = len(max(error_logs[:,0], key=len))
+    suite_l = len(max(error_logs[:,1], key=len))
+    namelist_l = len(max(error_logs[:,2], key=len))
+    status_l = len(max(error_logs[:,3], key=len))
+    # error_log contains header, subtracting 1 from error
+    error_count = error_logs.shape[0] - 1
+    passing_count = total_count - error_count
+    header_printed = False
+    column_width = (case_l + suite_l + namelist_l + status_l + 13)
+
+    # print formatted asummary
+    print("Failure Summary:")
+    print("-" * column_width)
+    for error_log in error_logs:
+        case_s, suite, namelist, status = error_log
+        print(f"| {case_s:<{case_l}} | {suite:<{suite_l}} | {namelist:<{namelist_l}} | {status:<{status_l}} |")
+        if not header_printed:
+            print("-" * column_width)
+            header_printed = True
+    print("-" * column_width)
+    print(f"[{error_count}/{total_count}] failed cases, [{passing_count}/{total_count}] passing cases")
+
+
 def main():
     (file, case, sdf, namelist, tracers, use_gdb, runtime, runtime_mult, docker, \
      verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out,       \
@@ -822,6 +848,8 @@ def main():
         if (tracers  != None): run_list[0]["tracers"]  = tracers
 
     # Loop through all input "run dictionaires"
+    error_logs = [["Failed Case", "Suite", "Namelist", "Status"]]
+    failed_case = False
     irun = 0
     for run in run_list:
 
@@ -894,13 +922,23 @@ def main():
             logging.info('Process "(case={0}, suite={1}, namelist={2}" completed successfully'. \
                          format(run["case"], run["suite"], active_suite.namelist))
         else:
-            logging.warning('Process "(case={0}, suite={1}, namelist={2}" exited with code {3}'. \
-                            format( run["case"], run["suite"], active_suite.namelist, status))
+            failed_case = True
+            error_str = 'Process "(case={0}, suite={1}, namelist={2}" exited with code {3}'. \
+                        format( run["case"], run["suite"], active_suite.namelist, status)
+            logging.warning(error_str)
+            error_logs = np.append(error_logs,
+                                   [[run["case"], run["suite"], active_suite.namelist, status]],
+                                   axis=0)
         #
         if time_elapsed:
             logging.info('    Elapsed time: {0}s'.format(time_elapsed))
         if docker:
             copy_outdir(exp_dir)
 
+    if (failed_case):
+        print_error_report(error_logs, len(run_list))
+        sys.exit(1)
+
+
 if __name__ == '__main__':
     main()