Fixed the parser for DReyeVR actor data

simple_test.py

@@ -0,0 +1,55 @@
+from typing import Dict, Optional, List
+from src.parser import parse_file
+from src.utils import get_good_idxs, fill_gaps, smooth_arr, compute_YP, convert_to_df, flatten_dict
+import numpy as np
+import argparse
+from src.visualizer import (
+    set_results_dir,
+)
+
+
+def main(filename: str, results_dir: str, vlines: Optional[List[float]] = None):
+    set_results_dir(results_dir)
+
+    """parse the file"""
+    data: Dict[str, np.ndarray or dict] = parse_file(filename)
+    print(data.keys())
+    # Convert to pandas dataframe
+    import pandas as pd
+    actors_data = {}
+    actors_data["TimeElapsed"] = data["TimeElapsed"]
+    actors_data["Actors"] = {}
+    actors_data["Actors"]["Location"] = data["Actors"]["Location"]
+
+
+
+    data_frame = convert_to_df(data)
+    actors_df = convert_to_df(actors_data)
+
+    # Display the frame
+    from IPython.display import display
+    from tabulate import tabulate
+    print(data_frame.keys())
+    print(tabulate(actors_df, headers = 'keys', tablefmt = 'fancy_grid'))
+
+
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser(description="DReyeVR recording parser")
+    argparser.add_argument(
+        "-f",
+        "--file",
+        metavar="P",
+        type=str,
+        help="path of the (human readable) recording file",
+    )
+    argparser.add_argument(
+        "-o",
+        "--out",
+        default="results",
+        type=str,
+        help="path of the results folder",
+    )
+    args = argparser.parse_args()
+
+    main(args.file, args.out)

src/parser.py

@@ -109,7 +109,7 @@ def parse_custom_actor(
 def parse_actor_location_rotation(
     data: Dict[str, Any],
     data_line: str,
-    title: Optional[str] = "",
+    actors_key: str,
     t: Optional[int] = 0,
 ):
     Id: int = int(data_line[: data_line.find("Location")])
@@ -118,12 +118,14 @@ def parse_actor_location_rotation(
     open_1: int = close_0 + 2 + len("Rotation")  # always " Rotation "
     loc_data: tuple = eval(data_line[open_0:close_0])
     rot_data: tuple = eval(data_line[open_1:])
-    if Id not in data[title]:
-        data[title][Id] = {"Time": [], "Location": [], "Rotation": []}
-    data[title][Id]["Time"].append(t)
-    data[title][Id]["Location"].append(loc_data)
-    data[title][Id]["Rotation"].append(rot_data)
-
+    if len(data[actors_key]["Id"]) <= t:
+        data[actors_key]["Id"].append([])
+        data[actors_key]["Location"].append({})
+        data[actors_key]["Rotation"].append({})
+    data[actors_key]["Id"][t].append(Id)
+    data[actors_key]["Location"][t][Id] = loc_data
+    data[actors_key]["Rotation"][t][Id] = rot_data
+
 
 def validate(data: Dict[str, Any], L: Optional[int] = None) -> None:
     # verify the data structure is reasonable
@@ -239,6 +241,9 @@ def parse_file(
 
     actors_key: str = "Actors"
     data[actors_key] = {}
+    data[actors_key]["Id"] = []
+    data[actors_key]["Location"] = []
+    data[actors_key]["Rotation"] = []
 
     with open(path, "r") as f:
         start_t: float = time.time()
@@ -273,13 +278,12 @@ def parse_file(
                 data_line: str = line.strip(Carla_Actor).strip("\n")
                 if "Location:" not in data_line or "Rotation" not in data_line:
                     continue  # don't care about state, light, animation, etc.
-                if "TimestampCarla" in data:
-                    t = data["TimestampCarla"][_no_title_key][-1]  # get carla time
-                else:
-                    t = 0
-                parse_actor_location_rotation(data, data_line, title="Actors", t=t)
-                if debug:
-                    validate(data)
+
+                t = len(data["TimeElapsed"]) - 1 # get carla time
+                #print('!', t)
+
+                parse_actor_location_rotation(data, data_line, actors_key, t=t)
+
 
             # print status
             if i % 500 == 0:
@@ -316,4 +320,4 @@ def cache_data(data: Dict[str, Any], filename: str) -> None:
     filename = f"{os.path.join(cache_dir, actual_name)}.pkl"
     with open(filename, "wb") as filehandler:
         pickle.dump(data, filehandler)
-    print(f"cached data to {filename}")
+    print(f"cached data to {filename}")

src/utils.py

@@ -110,7 +110,9 @@ def cleanup_data_line(data_line: str) -> str:
 def convert_to_list(data: Dict[str, np.ndarray or dict]) -> Dict[str, list or dict]:
     list_data = {}
     for k in data.keys():
-        if isinstance(data[k], dict):
+        if isinstance(data[k], list):
+            list_data[k] = data[k]
+        elif isinstance(data[k], dict):
             list_data[k] = convert_to_list(data[k])
         else:
             assert isinstance(data[k], np.ndarray)
@@ -252,8 +254,11 @@ def convert_to_df(data: Dict[str, Any]) -> pd.DataFrame:
     data = convert_to_list(data)
     data = flatten_dict(data)
     lens = [len(x) for x in data.values()]
-    assert min(lens) == max(lens)  # all lengths are equal!
-    # NOTE: pandas can't haneld high dimensional np arrays, so we just use lists
+    # all lengths are equal!
+    if not min(lens) == max(lens):
+        import ipdb; ipdb.set_trace()
+
+    # NOTE: pandas can't handle high dimensional np arrays, so we just use lists
     df = pd.DataFrame.from_dict(data)
     print(f"created DReyeVR df in {time.time() - start_t:.3f}s")
     return df