Collect tests feature

rere.py

@@ -25,28 +25,36 @@
 from difflib import unified_diff
 from typing import List, BinaryIO, Tuple, Optional
 
-def read_blob_field(f: BinaryIO, name: bytes) -> bytes:
+NAME_ENCODING = 'utf-8'
+TEXT_ENCODING = 'utf-8'
+IO_ENCODING = 'utf-8'
+
+def bi_name(name : str) -> bytes:
+    assert ' ' not in name
+    return name.encode(NAME_ENCODING)
+
+def read_blob_field(f: BinaryIO, name: str) -> bytes:
     line = f.readline()
-    field = b':b ' + name + b' '
+    field = b':b ' + bi_name(name) + b' '
     assert line.startswith(field), field
     assert line.endswith(b'\n')
     size = int(line[len(field):-1])
     blob = f.read(size)
     assert f.read(1) == b'\n'
     return blob
 
-def read_int_field(f: BinaryIO, name: bytes) -> int:
+def read_int_field(f: BinaryIO, name: str) -> int:
     line = f.readline()
-    field = b':i ' + name + b' '
+    field = b':i ' + bi_name(name) + b' '
     assert line.startswith(field)
     assert line.endswith(b'\n')
     return int(line[len(field):-1])
 
-def write_int_field(f: BinaryIO, name: bytes, value: int):
-    f.write(b':i %s %d\n' % (name, value))
+def write_int_field(f: BinaryIO, name: str, value: int):
+    f.write(b':i %s %d\n' % (bi_name(name), value))
 
-def write_blob_field(f: BinaryIO, name: bytes, blob: bytes):
-    f.write(b':b %s %d\n' % (name, len(blob)))
+def write_blob_field(f: BinaryIO, name: str, blob: bytes):
+    f.write(b':b %s %d\n' % (bi_name(name), len(blob)))
     f.write(blob)
     f.write(b'\n')
 
@@ -66,22 +74,22 @@ def load_list(file_path: str) -> list[str]:
 
 def dump_snapshots(file_path: str, snapshots: list[dict]):
     with open(file_path, "wb") as f:
-        write_int_field(f, b"count", len(snapshots))
+        write_int_field(f, "count", len(snapshots))
         for snapshot in snapshots:
-            write_blob_field(f, b"shell", bytes(snapshot['shell'], 'utf-8'))
-            write_int_field(f, b"returncode", snapshot['returncode'])
-            write_blob_field(f, b"stdout", snapshot['stdout'])
-            write_blob_field(f, b"stderr", snapshot['stderr'])
+            write_blob_field(f, "shell", bytes(snapshot['shell'], TEXT_ENCODING))
+            write_int_field(f, "returncode", snapshot['returncode'])
+            write_blob_field(f, "stdout", snapshot['stdout'])
+            write_blob_field(f, "stderr", snapshot['stderr'])
 
 def load_snapshots(file_path: str) -> list[dict]:
     snapshots = []
     with open(file_path, "rb") as f:
-        count = read_int_field(f, b"count")
+        count = read_int_field(f, "count")
         for _ in range(count):
-            shell = read_blob_field(f, b"shell")
-            returncode = read_int_field(f, b"returncode")
-            stdout = read_blob_field(f, b"stdout")
-            stderr = read_blob_field(f, b"stderr")
+            shell = read_blob_field(f, "shell")
+            returncode = read_int_field(f, "returncode")
+            stdout = read_blob_field(f, "stdout")
+            stderr = read_blob_field(f, "stderr")
             snapshot = {
                 "shell": shell,
                 "returncode": returncode,
@@ -119,22 +127,26 @@ def load_snapshots(file_path: str) -> list[dict]:
         shells = load_list(test_list_path)
         snapshots = load_snapshots(f'{test_list_path}.bi')
 
-        if len(shells) != len(snapshots):
-            print(f"UNEXPECTED: Amount of shell commands in f{test_list_path}")
-            print(f"    EXPECTED: {len(snapshots)}")
-            print(f"    ACTUAL:   {len(shells)}")
-            print(f"NOTE: You may want to do `{program_name} record {test_list_path}` to update {test_list_path}.bi")
-            exit(1)
+        skipped_shells = []
+        failed_shells = []
 
-        for (shell, snapshot) in zip(shells, snapshots):
+        snap_dict = {}
+        for snapshot in snapshots:
+            snapshot_shell = snapshot['shell'].decode(TEXT_ENCODING)
+            snap_dict[snapshot_shell] = snapshot
+
+        for shell in shells:
             print(f"REPLAYING: {shell}")
-            snapshot_shell = snapshot['shell'].decode('utf-8')
-            if shell != snapshot_shell:
-                print(f"UNEXPECTED: shell command")
-                print(f"    EXPECTED: {snapshot_shell}")
-                print(f"    ACTUAL:   {shell}")
+            if shell not in snap_dict:
+                print(f"NOT FOUND: shell command")
+                print(f"    COMMAND:   {shell}")
                 print(f"NOTE: You may want to do `{program_name} record {test_list_path}` to update {test_list_path}.bi")
-                exit(1)
+                print(f"NOTE: Skiping this shell")
+                skipped_shells.append(shell)
+                continue
+
+            snapshot = snap_dict[shell]
+
             process = subprocess.run(['sh', '-c', shell], capture_output = True);
             failed = False
             if process.returncode != snapshot['returncode']:
@@ -144,22 +156,34 @@ def load_snapshots(file_path: str) -> list[dict]:
                 failed = True
             if process.stdout != snapshot['stdout']:
                 # TODO: support binary outputs
-                a = snapshot['stdout'].decode('utf-8').splitlines(keepends=True)
-                b = process.stdout.decode('utf-8').splitlines(keepends=True)
+                a = snapshot['stdout'].decode(IO_ENCODING).splitlines(keepends=True)
+                b = process.stdout.decode(IO_ENCODING).splitlines(keepends=True)
                 print(f"UNEXPECTED: stdout")
                 for line in unified_diff(a, b, fromfile="expected", tofile="actual"):
                     print(line, end='')
                 failed = True
             if process.stderr != snapshot['stderr']:
-                a = snapshot['stderr'].decode('utf-8').splitlines(keepends=True)
-                b = process.stderr.decode('utf-8').splitlines(keepends=True)
+                a = snapshot['stderr'].decode(IO_ENCODING).splitlines(keepends=True)
+                b = process.stderr.decode(IO_ENCODING).splitlines(keepends=True)
                 print(f"UNEXPECTED: stderr")
                 for line in unified_diff(a, b, fromfile="expected", tofile="actual"):
                     print(line, end='')
                 failed = True
             if failed:
-                exit(1)
-        print('OK')
+                failed_shells.append(shell)
+
+        print(f'REPORT: {len(skipped_shells)} Skipped:')
+        for skipped in skipped_shells:
+            print(f'    SKIPPED: {skipped}')
+
+        print(f'REPORT: {len(failed_shells)} failed:')
+        for failed in failed_shells:
+            print(f'    FAILED: {failed}')
+
+        if len(skipped_shells) == 0 and len(failed_shells) == 0:
+            print('OK')
+        else:
+            exit(1)
     else:
         print(f'ERROR: unknown subcommand {subcommand}');
         exit(1);

tests/meta.list

@@ -0,0 +1,3 @@
+./rere.py replay test.list
+./rere.py replay tests/skip-fail.list
+./rere.py replay tests/meta.list

tests/meta.list.bi

@@ -0,0 +1,51 @@
+:i count 3
+:b shell 26
+./rere.py replay test.list
+:i returncode 0
+:b stdout 124
+REPLAYING: echo 'Hello, World'
+REPLAYING: echo 'Foo, bar'
+REPLAYING: echo 'Ur, mom'
+REPORT: 0 Skipped:
+REPORT: 0 failed:
+OK
+
+:b stderr 0
+
+:b shell 37
+./rere.py replay tests/skip-fail.list
+:i returncode 1
+:b stdout 460
+REPLAYING: echo 'Hello, World'
+UNEXPECTED: stdout
+--- expected
++++ actual
+@@ -1 +1 @@
+-Hello, world
++Hello, World
+REPLAYING: echo 'Foo, bar, baz'
+NOT FOUND: shell command
+    COMMAND:   echo 'Foo, bar, baz'
+NOTE: You may want to do `./rere.py record tests/skip-fail.list` to update tests/skip-fail.list.bi
+NOTE: Skiping this shell
+REPLAYING: echo 'Ur, mom'
+REPORT: 1 Skipped:
+    SKIPPED: echo 'Foo, bar, baz'
+REPORT: 1 failed:
+    FAILED: echo 'Hello, World'
+
+:b stderr 0
+
+:b shell 32
+./rere.py replay tests/meta.list
+:i returncode 0
+:b stdout 171
+REPLAYING: ./rere.py replay test.list
+REPLAYING: ./rere.py replay tests/skip-fail.list
+REPLAYING: ./rere.py replay tests/meta.list
+REPORT: 0 Skipped:
+REPORT: 0 failed:
+OK
+
+:b stderr 0
+

tests/skip-fail.list

@@ -0,0 +1,3 @@
+echo 'Hello, World'
+echo 'Foo, bar, baz'
+echo 'Ur, mom'

tests/skip-fail.list.bi

@@ -0,0 +1,25 @@
+:i count 3
+:b shell 19
+echo 'Hello, World'
+:i returncode 0
+:b stdout 13
+Hello, world
+
+:b stderr 0
+
+:b shell 15
+echo 'Foo, bar'
+:i returncode 0
+:b stdout 9
+Foo, bar
+
+:b stderr 0
+
+:b shell 14
+echo 'Ur, mom'
+:i returncode 0
+:b stdout 8
+Ur, mom
+
+:b stderr 0
+