From 9953a9e620b5324ac95c1e40a8f163fa084918e3 Mon Sep 17 00:00:00 2001
From: Julien Perrochet <julien.perrochet@orbitalize.com>
Date: Wed, 15 Nov 2023 10:36:18 +0100
Subject: [PATCH] [uss_qualifier] debug aggregate queries stats for #343

---
 monitoring/monitorlib/fetch/evaluation.py     |  3 +-
 .../monitorlib/fetch/evaluation_test.py       | 41 +++++++++++++++++++
 .../astm/netrid/common/aggregate_checks.py    | 13 ++++--
 3 files changed, 52 insertions(+), 5 deletions(-)
 create mode 100644 monitoring/monitorlib/fetch/evaluation_test.py

diff --git a/monitoring/monitorlib/fetch/evaluation.py b/monitoring/monitorlib/fetch/evaluation.py
index 64f2b4f612..6b11fedabc 100644
--- a/monitoring/monitorlib/fetch/evaluation.py
+++ b/monitoring/monitorlib/fetch/evaluation.py
@@ -28,7 +28,8 @@ def get_init_subsequent_queries_durations(
     init_durations: List[float] = list()  # list of initial queries duration
     subsequent_durations: List[float] = list()  # list of subsequent queries duration
 
-    for queries in queries_by_url.values():
+    for url, queries_iter in queries_by_url.items():
+        queries = queries_iter.copy()
         queries.sort(key=attrgetter("request.initiated_at"))  # sort queries by time
 
         for idx, query in enumerate(queries):
diff --git a/monitoring/monitorlib/fetch/evaluation_test.py b/monitoring/monitorlib/fetch/evaluation_test.py
new file mode 100644
index 0000000000..e58e9ae5f1
--- /dev/null
+++ b/monitoring/monitorlib/fetch/evaluation_test.py
@@ -0,0 +1,41 @@
+from datetime import datetime
+
+from implicitdict import StringBasedDateTime
+
+from monitoring.monitorlib.fetch import (
+    Query,
+    RequestDescription,
+    ResponseDescription,
+    evaluation,
+)
+
+
+def test_operator_id_non_ascii():
+    assert True == False
+
+
+def _fq(ts: int, elapsed: float):
+    """returns a fake query with the specified epoch timestamp as 'initiated_at'"""
+    return Query(
+        request=RequestDescription(
+            method=None,
+            url=None,
+            initiated_at=StringBasedDateTime(datetime.fromtimestamp(ts)),
+        ),
+        response=ResponseDescription(elapsed_s=elapsed, reported=None),
+    )
+
+
+def test_get_init_subsequent_queries_durations():
+    dummy_queries = [_fq(10, 2), _fq(12, 1), _fq(13, 0.9), _fq(14, 0.8)]
+    query_dict = {"some-url": dummy_queries}
+    (init_d, subseq_d) = evaluation.get_init_subsequent_queries_durations(5, query_dict)
+
+    assert init_d == [2]
+    assert subseq_d == [1, 0.9, 0.8]
+
+    query_dict = {"some-url": dummy_queries, "another-url": dummy_queries}
+
+    (init_d, subseq_d) = evaluation.get_init_subsequent_queries_durations(5, query_dict)
+    assert init_d == [2, 2]
+    assert subseq_d == [1, 0.9, 0.8, 1, 0.9, 0.8]
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py
index 4e4f15fe3d..754ed328ba 100644
--- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py
+++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py
@@ -79,10 +79,10 @@ def _init_queries(self, context: ExecutionContext):
 
         for query in self._queries:
             for base_url, participant in self._participants_by_base_url.items():
-                if query.request.url.startswith(base_url):
-                    self._queries_by_participant[participant].append(query)
-                    # TODO opportunity here to set the participant_id on the query if it's not already there
-                    #  maybe do so after most/all queries have been tagged at the call site where possible
+                if query.request.url.startswith(
+                    base_url
+                ) and not query.has_field_with_value("participant_id"):
+                    query.participant_id = participant
                     break
 
             # Only consider queries with the participant/server explicitly identified
@@ -372,3 +372,8 @@ def _dp_display_data_times_step(self):
                 f"{participant}/display_data",
                 f"percentiles on {len(relevant_queries)} relevant queries ({len(relevant_queries_by_url)} different URLs, {len(init_durations)} initial queries, {len(subsequent_durations)} subsequent queries): init 95th: {init_95th}; init 99th: {init_99th}; subsequent 95th: {subsequent_95th}; subsequent 99th: {subsequent_99th}",
             )
+
+            self.record_note(
+                f"{participant}/display_data details",
+                f"Initial durations: {init_durations} subsequent durations: {subsequent_durations}",
+            )