9pm.py: add new AsciiDoc test report

9pm.py

@@ -10,6 +10,7 @@
 import shutil
 import re
 import atexit
+import hashlib
 from datetime import datetime
 
 TEST_CNT=0
@@ -101,6 +102,13 @@ def execute(args, test, output_log):
 
     return skip_suite, test_skip, err
 
+def calculate_sha1sum(path):
+    sha1 = hashlib.sha1()
+    with open(path, "rb") as file:
+        for chunk in iter(lambda: file.read(4096), b""):
+            sha1.update(chunk)
+    return sha1.hexdigest()
+
 def run_onfail(cmdline, test):
     args = []
     if 'options' in test:
@@ -187,6 +195,16 @@ def parse_yaml(path):
             return -1
     return data
 
+def get_test_spec_path(case_path, test_spec):
+    case_dirname = os.path.dirname(case_path)
+    case_basename = os.path.basename(case_path)
+    case_name = os.path.splitext(case_basename)[0]
+
+    test_spec = test_spec.replace('<case>', case_name)
+
+    return  os.path.join(case_dirname, test_spec)
+
+
 def parse_suite(suite_path, parent_suite_path, options, name=None):
     suite = {}
     suite['suite'] = []
@@ -252,6 +270,14 @@ def parse_suite(suite_path, parent_suite_path, options, name=None):
                 case['mask'] = entry['mask']
 
             case['case'] = os.path.join(suite_dirname, entry['case'])
+
+            if 'settings' in suite:
+                if 'test-spec' in suite['settings']:
+                    test_spec_path = get_test_spec_path(case['case'], suite['settings']['test-spec'])
+                    if os.path.exists(test_spec_path):
+                        case['test-spec'] = test_spec_path
+                        case['test-spec-sha'] = calculate_sha1sum(test_spec_path)
+
             if not os.path.isfile(case['case']):
                 print("error, test case not found {}" . format(case['case']))
                 print("(referenced from {})" . format(suite_path))
@@ -267,43 +293,115 @@ def parse_suite(suite_path, parent_suite_path, options, name=None):
             sys.exit(1)
     return suite
 
-def get_github_emoji(result):
-    if result == "pass":
-        return ":white_check_mark:"
-    if result == "fail":
-        return ":red_circle:"
-    if result == "skip":
-        return ":large_orange_diamond:"
-    if result == "masked-fail":
-        return ":o:"
-    if result == "masked-skip":
-        return ":small_orange_diamond:"
-
-    return result
-
-def write_result_md_tree(md, gh, data, base):
+def write_report_result_tree(file, includes, data, depth):
     for test in data['suite']:
-        with open(md, 'a') as file:
-            file.write("{}- {} : {}\n".format(base, test['result'].upper(), test['name']))
+        indent = '  ' * depth
+        stars = '*' + '*' * depth
 
-        with open(gh, 'a') as file:
-            mark = get_github_emoji(test['result'])
-            file.write("{}- {} : {}\n".format(base, mark, test['name']))
+        string = f"{indent}"
+        string += f"{stars}"
+        string += f" [.{test['result']}]#{test['result'].upper()}#"
+        if 'outfile' in test:
+            string += f" <<output-{test['name']},{test['name']}>>"
+        else:
+            string += f" {test['name']}"
+
+        # Append (Spec) if there's a test specification
+        if 'test-spec' in test:
+            if test['test-spec-sha'] not in includes:
+                includes.append(test['test-spec-sha'])
+
+            include_dir = os.path.join(LOGDIR, "report-incl")
+            os.makedirs(include_dir, exist_ok=True)
+            # We ignore potential overwrites
+            shutil.copy(test['test-spec'], os.path.join(include_dir, test['test-spec-sha']))
+
+            string += f" <<incl-{test['test-spec-sha']},(Spec)>>"
+
+        file.write(f"{string}\n")
 
         if 'suite' in test:
-            write_result_md_tree(md, gh, test, base + "  ")
+            write_report_result_tree(file, includes, test, depth + 1)
 
-def write_result_files(data):
-    md = os.path.join(LOGDIR, 'result.md')
-    gh = os.path.join(LOGDIR, 'result-gh.md')
+def write_report_includes(file, includes, data, depth):
+    for sha1sum in includes:
+        # Note: not having incl- and breaks asciidoctor
+        file.write(f"\n[[incl-{sha1sum}]]\n")
+        file.write("include::{}[]\n" . format(os.path.join("report-incl", sha1sum)))
 
-    with open(md, 'a') as file:
-        file.write("# Test Result\n")
+def write_report_output(file, data, depth):
+    for test in data['suite']:
 
-    with open(gh, 'a') as file:
+        if 'outfile' in test:
+            if 'test-spec-sha' in test:
+                file.write(f"\n=== <<incl-{test['test-spec-sha']},{test['name']}>>\n")
+            else:
+                file.write(f"\n=== {test['name']}\n")
+
+            file.write(f"\n[[output-{test['name']}]]\n")
+            file.write(f"----\n")
+            file.write(f"include::{test['outfile']}[]\n")
+            file.write(f"----\n")
+
+        if 'suite' in test:
+            write_report_output(file, test, depth + 1)
+
+def write_report(data):
+    with open(os.path.join(LOGDIR, 'report.adoc'), 'a') as file:
+        current_date = datetime.now().strftime("%Y-%m-%d")
+        file.write("= 9pm Test Report\n")
+        file.write("Author: 9pm Test Framework\n")
+        file.write(f"Date: {current_date}\n")
+        file.write(":toc: left\n")
+        file.write(":toc-title: INDEX\n")
+        file.write(":sectnums:\n")
+        file.write(":pdf-page-size: A4\n")
+
+        file.write("\n== Test Result\n\n")
+
+        includes = []
+        write_report_result_tree(file, includes, data, 0)
+
+        file.write("\n<<<\n")
+        file.write("\n== Test Output\n")
+        write_report_output(file, data, 0)
+
+        file.write("\n<<<\n")
+        file.write("\n== Test Specification\n")
+        write_report_includes(file, includes, data, 0)
+
+
+def write_github_result_tree(file, data, depth):
+    icon_map = {
+        "pass": ":white_check_mark:",
+        "fail": ":red_circle:",
+        "skip": ":large_orange_diamond:",
+        "masked-fail": ":o:",
+        "masked-skip": ":small_orange_diamond:",
+    }
+    for test in data['suite']:
+        mark = icon_map.get(test['result'], "")
+        file.write("{}- {} : {}\n".format('  ' * depth, mark, test['name']))
+
+        if 'suite' in test:
+            write_github_result_tree(file, test, depth + 1)
+
+def write_github_result(data):
+    with open(os.path.join(LOGDIR, 'result-gh.md'), 'a') as file:
         file.write("# Test Result\n")
+        write_github_result_tree(file, data, 0)
+
+def write_md_result_tree(file, data, depth):
+    for test in data['suite']:
+        file.write("{}- {} : {}\n".format('  ' * depth, test['result'].upper(), test['name']))
 
-    write_result_md_tree(md, gh, data, "")
+        if 'suite' in test:
+            write_md_result_tree(file, test, depth + 1)
+
+def write_md_result(data):
+    with open(os.path.join(LOGDIR, 'result.md'), 'a') as file:
+        file.write("# Test Result\n")
+        write_md_result_tree(file, data, 0)
 
 def print_result_tree(data, base):
     i = 1
@@ -563,7 +661,9 @@ def main():
         cprint(pcolor.green, "\no Execution")
 
     print_result_tree(cmdl, "")
-    write_result_files(cmdl)
+    write_md_result(cmdl)
+    write_github_result(cmdl)
+    write_report(cmdl)
 
     db.close()
     sys.exit(err)

examples/README.md

@@ -9,7 +9,7 @@ understand in the context of 9pm.
 # Execution
 Start the all.yaml test suite with
 
-`./9pm.py examples/all.yaml examples/bash.sh`
+`./9pm.py examples/all.yaml examples/cases/bash.sh`
 
 The end result tree should look something like:
 

examples/all.yaml

@@ -1,6 +1,10 @@
 ---
+- settings:
+    test-spec: <case>.adoc
+
 - case: "cases/bash.sh"
 - case: "cases/python.py"
+- case: "cases/perl/test.pl"
 
 - case: "cases/bash.sh"
   name: "bash-with-no-options"
@@ -14,4 +18,8 @@
 - suite: "suites/nested.yaml"
   name: "nested-suite"
   opts:
-    - "opt-from-all"
+    - "opt-from-all"
+
+- case: "cases/on-fail/bash.sh"
+  name: "bash-run-onfail-cleanup"
+  onfail: "cleanup.sh"

examples/cases/bash.adoc

@@ -0,0 +1,19 @@
+=== Bash Example
+==== Description
+An example of how to create a 9pm test case by printing TAP from Python.
+
+This test specification has the same name as the testcase but with the .adoc extension.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam sed auctor ipsum. Sed pharetra vestibulum leo, ac vulputate odio tristique eget. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Sed dictum, est ut elementum bibendum, velit odio feugiat ante, et posuere metus erat a nulla. Curabitur lacinia vel lacus in feugiat. Integer pulvinar vestibulum felis et vehicula. Sed facilisis turpis eget felis cursus feugiat. Quisque sodales mauris id sapien viverra, in suscipit arcu aliquet. Vestibulum porttitor aliquam arcu mattis efficitur. In hac habitasse platea dictumst. Vivamus aliquet pretium dolor, ut imperdiet ligula tristique vel. Interdum et malesuada fames ac ante ipsum primis in faucibus. 
+
+==== Topology
+Localhost only.
+
+==== Test sequence
+. Lorem ipsum dolor sit amet
+. Consectetur adipiscing elit
+. Sed do eiusmod tempor incididunt
+. Ut labore et dolore magna aliqua
+. Excepteur sint occaecat cupidatat non proident
+
+<<<

examples/cases/on-fail/bash.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Output TAP (Test Anything Protocol) Plan
+echo "1..1"
+
+echo "not ok 1 - Fail to trigger on-fail script named in suite"

examples/cases/on-fail/cleanup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Hi from cleanup.sh"

examples/cases/perl/Readme.adoc

@@ -0,0 +1,19 @@
+=== Perl Example
+==== Description
+An example of how to create a 9pm test case by printing TAP from Perl.
+
+This test specification is named Readme.adoc and is located in the same directory as test.pl.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam sed auctor ipsum. Sed pharetra vestibulum leo, ac vulputate odio tristique eget. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Sed dictum, est ut elementum bibendum, velit odio feugiat ante, et posuere metus erat a nulla. Curabitur lacinia vel lacus in feugiat. Integer pulvinar vestibulum felis et vehicula. Sed facilisis turpis eget felis cursus feugiat. Quisque sodales mauris id sapien viverra, in suscipit arcu aliquet. Vestibulum porttitor aliquam arcu mattis efficitur. In hac habitasse platea dictumst. Vivamus aliquet pretium dolor, ut imperdiet ligula tristique vel. Interdum et malesuada fames ac ante ipsum primis in faucibus. 
+
+==== Topology
+Localhost only.
+
+==== Test sequence
+. Lorem ipsum dolor sit amet
+. Consectetur adipiscing elit
+. Sed do eiusmod tempor incididunt
+. Ut labore et dolore magna aliqua
+. Excepteur sint occaecat cupidatat non proident
+
+<<<

examples/cases/perl/test.pl

@@ -0,0 +1,5 @@
+#!/usr/bin/perl
+
+print "1..1\n";
+
+print "ok 1 - A good result from Perl\n";

examples/cases/python.adoc

@@ -0,0 +1,19 @@
+=== Python Example
+==== Description
+An example of how to create a 9pm test case by printing TAP from Python.
+
+This test specification has the same name as the testcase but with the .adoc extension.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam sed auctor ipsum. Sed pharetra vestibulum leo, ac vulputate odio tristique eget. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Sed dictum, est ut elementum bibendum, velit odio feugiat ante, et posuere metus erat a nulla. Curabitur lacinia vel lacus in feugiat. Integer pulvinar vestibulum felis et vehicula. Sed facilisis turpis eget felis cursus feugiat. Quisque sodales mauris id sapien viverra, in suscipit arcu aliquet. Vestibulum porttitor aliquam arcu mattis efficitur. In hac habitasse platea dictumst. Vivamus aliquet pretium dolor, ut imperdiet ligula tristique vel. Interdum et malesuada fames ac ante ipsum primis in faucibus. 
+
+==== Topology
+Localhost only.
+
+==== Test sequence
+. Lorem ipsum dolor sit amet
+. Consectetur adipiscing elit
+. Sed do eiusmod tempor incididunt
+. Ut labore et dolore magna aliqua
+. Excepteur sint occaecat cupidatat non proident
+
+<<<

examples/suites/nested.yaml

@@ -1,10 +1,13 @@
 ---
+- settings:
+    test-spec: Readme.adoc
+
 - case: "../cases/bash.sh"
   name: "bash-in-nested"
   opts:
     - "opt-from-nested"
 
-- case: "../cases/python.py"
-  name: "python-in-nested"
+- case: "../cases/perl/test.pl"
+  name: "perl-in-nested"
   opts:
     - "opt-from-nested"

report/theme.yml

@@ -0,0 +1,54 @@
+extends: default
+font:
+  catalog:
+    merge: true
+    SourceCodePro:
+      normal: SourceCodePro-Regular.ttf
+      bold: SourceCodePro-Bold.ttf
+      italic: SourceCodePro-Italic.ttf
+      bold-italic: SourceCodePro-BoldItalic.ttf
+
+base:
+  font-family: SourceCodePro
+  font-size: 10
+  line-height-length: 1.3
+  font-color: #1f2937
+
+code:
+  background-color: #ffffff
+  border-color: #ffffff
+
+heading:
+  font-family: SourceCodePro
+  font-style: bold
+
+role:
+  underline:
+    text-decoration: underline
+  pass:
+    font-color: #14532d
+    font-style: bold
+  fail:
+    font-color: #b91c1c
+    font-style: bold
+  skip:
+    font-color: #eab308
+    font-style: bold
+
+link:
+    font-color: #1f2937
+    text-decoration: underline
+
+heading-h1:
+  font-size: 24
+
+heading-h2:
+  font-size: 18
+
+heading-h3:
+  font-size: 12
+  margin-top: 30
+  text-decoration: underline
+
+heading-h4:
+  font-size: 11