stats: dump JSON stats into JSON Lines format

pantsbuild · Mar 29, 2024 · d853ca0 · d853ca0
1 parent 979578c
commit d853ca0
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 24 deletions.
diff --git a/src/python/pants/goal/stats_aggregator.py b/src/python/pants/goal/stats_aggregator.py
@@ -66,11 +66,11 @@ class StatsOutputFormat(Enum):
     """Output format for reporting Pants stats.
 
     text: Report stats in plain text.
-    json: Report stats in JSON.
+    jsonlines: Report stats in JSON Lines text format.
     """
 
     text = "text"
-    json = "json"
+    jsonlines = "jsonlines"
 
 
 class StatsAggregatorSubsystem(Subsystem):
@@ -127,27 +127,17 @@ def _log_or_write_to_file_plain(output_file: Optional[str], lines: list[str]) ->
 
 
 def _log_or_write_to_file_json(output_file: Optional[str], stats_object: StatsObject) -> None:
-    """Send JSON object to the stdout or write to the file."""
+    """Send JSON Lines single line object to the stdout or write to the file."""
     if not stats_object:
         return
 
     if not output_file:
-        logger.info(stats_object)
+        logger.info(json.dumps(stats_object))
         return
 
-    existing_stats = None
-    if Path(output_file).exists():
-        try:
-            with open(output_file) as fh:
-                existing_stats = json.load(fh)
-            if isinstance(existing_stats.get("stats"), list):
-                existing_stats["stats"].append(stats_object)
-        except Exception:
-            pass
-
-    with safe_open(output_file, "w") as fh:
-        stats = existing_stats if existing_stats else {"stats": [stats_object]}
-        json.dump(stats, fh, indent=4)
+    jsonline = json.dumps(stats_object) + "\n"
+    with safe_open(output_file, "a") as fh:
+        fh.write(jsonline)
     logger.info(f"Wrote Pants stats to {output_file}")
 
 
@@ -355,7 +345,7 @@ def __call__(
 
         if StatsOutputFormat.text == self.format:
             self._output_stats_in_plain_text(context)
-        elif StatsOutputFormat.json == self.format:
+        elif StatsOutputFormat.jsonlines == self.format:
             self._output_stats_in_json(context)
 
 

diff --git a/src/python/pants/goal/stats_aggregator_integration_test.py b/src/python/pants/goal/stats_aggregator_integration_test.py
@@ -87,8 +87,8 @@ def test_writing_to_output_file_json() -> None:
             "--plugins=hdrhistogram",
             "--stats-log",
             "--stats-memory-summary",
-            "--stats-format=json",
-            "--stats-output-file=stats.json",
+            "--stats-format=jsonlines",
+            "--stats-output-file=stats.jsonl",
             "roots",
         ]
         run_pants(argv1).assert_success()
@@ -97,14 +97,16 @@ def test_writing_to_output_file_json() -> None:
             "--plugins=hdrhistogram",
             "--stats-log",
             "--stats-memory-summary",
-            "--stats-format=json",
-            "--stats-output-file=stats.json",
+            "--stats-format=jsonlines",
+            "--stats-output-file=stats.jsonl",
             "list",
             "::",
         ]
         run_pants(argv2).assert_success()
-        with open("stats.json") as fh:
-            stats = json.load(fh)["stats"]
+        stats = []
+        with open("stats.jsonl") as fh:
+            for line in fh.readlines():
+                stats.append(json.loads(line))
 
         assert len(stats) == 2