Misc porting changes

python/vmaf/core/executor.py

@@ -118,7 +118,7 @@ def _slugify(v):
 
         normalized_str = '_'.join(map(lambda k: '{k}_{v}'.format(k=k, v=_slugify(d[k])), sorted(d.keys())))
 
-        if len(normalized_str) > 196:  # upper limit of filename is 256 but leave some space for prefix/suffix
+        if len(normalized_str) > 140:  # upper limit of filename is 256 but leave some space for prefix/suffix
             normalized_str = hashlib.sha1(normalized_str.encode("utf-8")).hexdigest()
 
         return normalized_str
@@ -182,7 +182,7 @@ def _run(asset_lock):
                 lock.release()
                 return result
 
-            self.results = parallel_map(_run, list_args, processes=processes)
+            self.results = parallel_map(_run, list_args, processes=processes, sleep_sec=0.1)
         else:
             self.results = list(map(self._run_on_asset, self.assets))
 
@@ -590,14 +590,24 @@ def _open_workfile(cls, asset, path, workfile_path, yuv_type, workfile_yuv_type,
         # if fifo mode, mkfifo
         if fifo_mode:
             os.mkfifo(workfile_path)
+
+        if ref_or_dis == 'ref':
+            start_end_frame = asset.ref_start_end_frame
+        elif ref_or_dis == 'dis':
+            start_end_frame = asset.dis_start_end_frame
+        else:
+            assert False
+
         if yuv_type != 'notyuv':
             # in this case, for sure has width_height
             assert width_height is not None
             width, height = width_height
             src_fmt_cmd = cls._get_yuv_src_fmt_cmd(yuv_type, height, width)
         else:
             src_fmt_cmd = cls._get_notyuv_src_fmt_cmd(path)
-        vframes_cmd, select_cmd = cls._get_vframes_cmd(asset, ref_or_dis)
+
+        vframes_cmd, select_cmd = cls._get_vframes_cmd(start_end_frame)
+
         crop_cmd = cls._get_filter_cmd(asset, 'crop', ref_or_dis)
         pad_cmd = cls._get_filter_cmd(asset, 'pad', ref_or_dis)
         quality_width, quality_height = quality_width_height
@@ -709,14 +719,7 @@ def _get_filter_cmd(asset, key, target):
             if asset.get_filter_cmd(key, target) is not None else ""
 
     @staticmethod
-    def _get_vframes_cmd(asset, ref_or_dis):
-        if ref_or_dis == 'ref':
-            start_end_frame = asset.ref_start_end_frame
-        elif ref_or_dis == 'dis':
-            start_end_frame = asset.dis_start_end_frame
-        else:
-            raise AssertionError('Unknown ref_or_dis: {}'.format(ref_or_dis))
-
+    def _get_vframes_cmd(start_end_frame: Optional[list]):
         if start_end_frame is None:
             return "", ""
         else:

python/vmaf/core/feature_assembler.py

@@ -92,7 +92,13 @@ def _create_feature_result_dicts(self):
                     try:
                         result_dicts[result_index][scores_key] = result[scores_key]
                     except KeyError:
-                        scores_key_alt = BasicResult.scores_key_wildcard_match(result.result_dict, scores_key)
+                        # Determine scores keys from other features. These need to be discarded when wildcard-querying
+                        # for all scores keys pertinent to a particular atom feature.
+                        other_scores_keys = [self._get_scores_key(fextractor_type, other_atom_feature)
+                                             for other_atom_feature in self._get_atom_features(fextractor_type)
+                                             if other_atom_feature is not atom_feature]
+                        scores_key_alt = BasicResult.scores_key_wildcard_match(result.result_dict, scores_key,
+                                                                               excluded_scores_keys=other_scores_keys)
                         result_dicts[result_index][scores_key] = result[scores_key_alt]
         return result_dicts
 

python/vmaf/core/result.py

@@ -140,8 +140,10 @@ def _get_aggregate_score_str(self):
         return str_aggregate
 
     @staticmethod
-    def scores_key_wildcard_match(result_dict, scores_key):
+    def scores_key_wildcard_match(result_dict, scores_key, excluded_scores_keys=None):
         """
+        Find all matching score keys from a result dictionary. When a list of keys is provided via excluded_scores_keys,
+        these particular keys will be ommited from the query result.
         >>> BasicResult.scores_key_wildcard_match({'VMAF_integer_feature_vif_scale0_egl_1_scores': [0.983708]}, 'VMAF_integer_feature_vif_scale0_scores')
         'VMAF_integer_feature_vif_scale0_egl_1_scores'
         >>> BasicResult.scores_key_wildcard_match({'VMAF_integer_feature_vif_scale0_egl_1_scores': [0.983708]}, 'VMAF_integer_feature_vif_scale1_scores')
@@ -208,10 +210,24 @@ def scores_key_wildcard_match(result_dict, scores_key):
                  'VMAF_feature_vif_scale0_egl_1_scores': [0.983738]}
         >>> BasicResult.scores_key_wildcard_match(e, 'VMAF_feature_adm_num_scores')
         'VMAF_feature_adm_num_egl_1_scores'
+        >>> f = {'VMAF_integer_feature_vif_scale0_egl_1_scores': [111.207703], \
+                 'VMAF_integer_feature_vif_scale0_eg_1_scores': [109.423508]}
+        >>> BasicResult.scores_key_wildcard_match(f, 'VMAF_integer_feature_vif_scale0_scores')
+        'VMAF_integer_feature_vif_scale0_eg_1_scores'
+        >>> BasicResult.scores_key_wildcard_match(f, 'VMAF_integer_feature_vif_scale0_scores', excluded_scores_keys=[])
+        'VMAF_integer_feature_vif_scale0_eg_1_scores'
+        >>> BasicResult.scores_key_wildcard_match(f, 'VMAF_integer_feature_vif_scale0_scores', excluded_scores_keys=['VMAF_integer_feature_vif_scale0_eg_1_scores'])
+        'VMAF_integer_feature_vif_scale0_egl_1_scores'
+        >>> BasicResult.scores_key_wildcard_match(f, 'VMAF_integer_feature_vif_scale0_scores', excluded_scores_keys=['VMAF_integer_feature_vif_scale0_eg_1_scores', 'VMAF_integer_feature_vif_scale0_egl_1_scores'])
+        Traceback (most recent call last):
+        ...
+        KeyError: 'no key matches VMAF_integer_feature_vif_scale0_scores'
         """
 
         # first look for exact match
-        result_keys_sorted = sorted(result_dict.keys())
+        result_keys_sorted = [key for key in sorted(result_dict.keys()) if (excluded_scores_keys is not None and
+                                                                            key not in excluded_scores_keys) or
+                              excluded_scores_keys is None]
         for result_key in result_keys_sorted:
             if result_key == scores_key:
                 return result_key

python/vmaf/tools/misc.py

@@ -307,7 +307,7 @@ def index_and_value_of_min(l):
     return min(enumerate(l), key=lambda x: x[1])
 
 
-def parallel_map(func, list_args, processes=None):
+def parallel_map(func, list_args, processes=None, sleep_sec=0.01):
     """
     Build my own parallelized map function since multiprocessing's Process(),
     or Pool.map() cannot meet my both needs:
@@ -359,7 +359,7 @@ def func_wrapper(idx_args):
         if len(waiting_procs) == 0 and len(active_procs) == 0:
             break
 
-        sleep(0.01) # check every x sec
+        sleep(sleep_sec) # check every x sec
 
     # finally, collect results
     rets = list(map(lambda idx: return_dict[idx], range(len(list_args))))