encapp: add quality option to encapp.py

With "--qaulity" the vmaf, ssim and psnr will be calculated automatically
while running.
TODO: provide more options. Currently everything will be default.
VMAF model can still be set with an environment variable.

Signed-off-by: JohanBlome <[email protected]>

scripts/encapp.py

@@ -19,13 +19,16 @@
 import datetime
 from google.protobuf import text_format
 import google.protobuf.descriptor_pool as descriptor_pool
+import multiprocessing
 
 import encapp_tool
 import encapp_tool.app_utils
 import encapp_tool.adb_cmds
 import encapp_tool.ffutils
+import encapp_quality
 import copy
 import random
+import pandas as pd
 
 SCRIPT_ROOT_DIR = os.path.abspath(
     os.path.join(encapp_tool.app_utils.SCRIPT_DIR, os.pardir)
@@ -42,6 +45,8 @@
 
 DEBUG = False
 
+QUALITY_PROCESSES = []
+
 FUNC_CHOICES = {
     "help": "show help options",
     "install": "install apks",
@@ -226,6 +231,7 @@ def valid_path(text):
 def run_encapp_test(protobuf_txt_filepath, serial, device_workdir, debug):
     if debug > 0:
         print(f"running test: {protobuf_txt_filepath}")
+    # TODO: add special exec command here.
     if encapp_tool.adb_cmds.USE_IDB:
         # remove log file first
         ret, _, stderr = encapp_tool.adb_cmds.run_cmd(
@@ -757,6 +763,24 @@ def run_codec_tests_file(
                 if not results[0]:
                     success = False
                 result_files += results[1]
+                # Run quality
+                if success and options.quality:
+                    global QUALITY_PROCESSES
+                    output = f"{local_workdir}/quality{basename}.csv"
+                    proc = multiprocessing.Process(
+                        target=encapp_quality.calculate_quality,
+                        args=(
+                            results[1],
+                            options.mediastore,
+                            output,
+                            True,
+                            options.debug,
+                        ),
+                    )
+                    QUALITY_PROCESSES.append([proc, output])
+                    proc.start()
+                    if debug:
+                        print("\n*** Quality proc is started!!!\n***\n")
 
             return success, result_files
 
@@ -1997,6 +2021,11 @@ def __call__(self, parser, options, values, option_string=None):
         default=None,
         help="Root directory for sources. If not set the input.filepath wll be absolute or relative from the current",
     )
+    parser.add_argument(
+        "--quality",
+        action="store_true",
+        help="Run the quality calculations as a part of the session. Same as running the encapp_quality tool after the encoding",
+    )
 
 
 def get_options(argv):
@@ -2339,6 +2368,31 @@ def main(argv):
             remove_encapp_gen_files(serial, options.device_workdir, options.debug)
         result_ok, result_json = codec_test(options, model, serial, options.debug)
 
+        global QUALITY_PROCESSES
+
+        if options.quality:
+            csvdata = []
+            counter = 1
+            total = len(QUALITY_PROCESSES)
+            while len(QUALITY_PROCESSES) > 0:
+                print(f"Wait for quality measurement, {counter}/{total}\r", end="")
+                quality = QUALITY_PROCESSES.pop()
+                proc = quality[0]
+                proc.join()
+                csvdata.append(quality[1])
+
+            output_csv = []
+            data = None
+            for file in csvdata:
+                data_ = pd.read_csv(file)
+                if data is None:
+                    data = data_
+                else:
+                    data = pd.merge(data, data_, how="outer")
+                os.remove(file)
+
+            data.to_csv(f"{options.local_workdir}/quality.csv")
+
         if not options.ignore_results and not options.dry_run:
             verify_app_version(result_json)
         if not result_ok:

scripts/encapp_quality.py

@@ -353,7 +353,6 @@ def run_quality(test_file, options, debug):
     found in options.media_path directory or overriden
     """
     global VMAF_MODEL
-    print(f"Run quality, {test_file}")
     if not os.path.exists(test_file):
         print("File not found: " + test_file)
         return None
@@ -602,10 +601,8 @@ def run_quality(test_file, options, debug):
         if options.get("crop_input", None):
             crop = f"crop={options.get('crop_input')},"
         filter_cmd = f"[0:v]fps={input_framerate}[d0];[d0]{crop}scale={input_width}:{input_height}[{diststream}];[{diststream}][1:v]"
-        print(f"\nFPS -> {input_framerate}")
         # Do calculations
         if recalc or not os.path.exists(vmaf_file):
-
             # important: vmaf must be called with videos in the right order
             # <distorted_video> <reference_video>
             # https://jina-liu.medium.com/a-practical-guide-for-vmaf-481b4d420d9c
@@ -618,7 +615,8 @@ def run_quality(test_file, options, debug):
             # Allow for an environment variable
             model = ""
             if os.environ.get("VMAF_MODEL_PATH", None):
-                print("Environment VMAF_PATH override model")
+                if debug:
+                    print("Environment VMAF_PATH override model")
                 model = "path=" + os.environ.get("VMAF_MODEL_PATH")
             else:
                 model = f"'version={VMAF_MODEL}'"
@@ -939,6 +937,52 @@ def get_options(argv):
     return options
 
 
+def calculate_quality(tests, source_path, output, quiet, debug):
+    """
+    Calculates vmaf, psnr and ssim for all test paths using the source_path to
+    look for references. All parameters will be default.
+    """
+    # Everything will be default.
+    # global VMAF_MODEL
+    # VMAF_MODEL = options.vmaf_model
+
+    # run all the tests
+    current = 1
+    total = len(tests)
+    if not quiet:
+        print(f"Total number of tests: {total}")
+    df = None
+    start = time.time()
+    options = {"header": True, "media_path": source_path, "quiet": quiet}
+    for test in tests:
+        try:
+            quality_dict = run_quality(test, options, False)
+        except Exception as ex:
+            print(f"{test} failed: {ex}")
+            continue
+        now = time.time()
+        run_for = now - start
+        time_per_test = float(run_for) / float(current)
+        time_left = round(time_per_test * (total - current))
+        time_left_m = int(time_left / 60)
+        time_left_s = int(time_left) % 60
+        if not quiet:
+            print(
+                f"Running {current}/{total}, Running for: {round(run_for)} sec, estimated time left {time_left_m}:{time_left_s:02} m:s"
+            )
+        current += 1
+        if quality_dict is None:
+            continue
+        if df is None:
+            df = pd.DataFrame(columns=quality_dict.keys())
+        df.loc[df.size] = quality_dict.values()
+    # write data to csv file
+    mode = "w"
+    if debug:
+        print(f"Write to {output}")
+    df.to_csv(output, mode=mode, index=False, header=True)
+
+
 def main(argv):
     """Calculate video quality properties (vmaf/ssim/psnr) and write
     a csv with relevant data