[uss_qualifier] Make report evaluation scenarios normal scenarios

monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py

@@ -1,11 +1,12 @@
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class CRDBAccess(GenericTestScenario):
     def __init__(self):
         super().__init__()
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
         # TODO: Implement
         self.end_test_scenario()

monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py

@@ -8,11 +8,7 @@
 from monitoring.uss_qualifier.configurations.configuration import ParticipantID
 from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource
 from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance
-from monitoring.uss_qualifier.scenarios.interuss.evaluation_scenario import (
-    ReportEvaluationScenario,
-)
 
-from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource
 from monitoring.uss_qualifier.resources.netrid import (
     NetRIDServiceProviders,
     NetRIDObserversResource,
@@ -24,8 +20,11 @@
 
 from loguru import logger
 
+from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
+
 
-class AggregateChecks(ReportEvaluationScenario):
+class AggregateChecks(GenericTestScenario):
     _rid_version: RIDVersion
     _service_providers: List[NetRIDServiceProvider]
     _observers: List[RIDSystemObserver]
@@ -38,13 +37,11 @@ class AggregateChecks(ReportEvaluationScenario):
 
     def __init__(
         self,
-        report_resource: TestSuiteReportResource,
         service_providers: NetRIDServiceProviders,
         observers: NetRIDObserversResource,
         dss_instances: DSSInstancesResource,
     ):
-        super().__init__(report_resource)
-        self._queries = self.report.queries()
+        super().__init__()
         self._service_providers = service_providers.service_providers
         self._observers = observers.observers
         self._dss_instances = dss_instances.dss_instances
@@ -71,6 +68,9 @@ def __init__(
             if dss.local_debug:
                 self._debug_mode_usses.add(dss.participant_id)
 
+    def _init_queries(self, context: ExecutionContext):
+        self._queries = list(context.sibling_queries())
+
         # collect and classify queries by participant
         self._queries_by_participant = {
             participant: list()
@@ -93,7 +93,9 @@ def __init__(
                 participant_queries.append(query)
                 self._queries_by_participant[query.participant_id] = participant_queries
 
-    def run(self):
+    def run(self, context: ExecutionContext):
+        self._init_queries(context)
+
         self.begin_test_scenario()
 
         self.record_note("participants", str(self._participants_by_base_url))

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py

@@ -12,6 +12,7 @@
 from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss import utils
 from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class ISAExpiry(GenericTestScenario):
@@ -43,7 +44,7 @@ def __init__(
         self._isa_end_time = self._isa.shifted_time_end(now)
         self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint]
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
 
         self._setup_case()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py

@@ -14,6 +14,7 @@
 from monitoring.uss_qualifier.resources import VerticesResource
 from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class ISASimple(GenericTestScenario):
@@ -47,7 +48,7 @@ def __init__(
             v.as_s2sphere() for v in problematically_big_area.specification.vertices
         ]
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
 
         self._setup_case()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py

@@ -4,6 +4,8 @@
 
 import arrow
 import s2sphere
+
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 from uas_standards.astm.f3411 import v19, v22a
 
 from monitoring.monitorlib.fetch import query_and_describe
@@ -69,7 +71,7 @@ def __init__(
         else:
             ValueError(f"Unsupported RID version '{self._dss.rid_version}'")
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
 
         self._setup_case()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py

@@ -22,6 +22,7 @@
 
 from typing import Dict
 
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 _24H_MIN_TOLERANCE_S = 23 * 3600 + 59 * 60  # 23 hours and 59 minutes
 _24H_MAX_TOLERANCE_S = 24 * 3600 + 1  # 24 hours sharp, plus a second
@@ -55,7 +56,7 @@ def __init__(
         self._sub_id = id_generator.id_factory.make_id(self.SUB_TYPE)
         self._isa = isa.specification
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
 
         self._setup_case()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py

@@ -18,6 +18,7 @@
 )
 from monitoring.uss_qualifier.scenarios.astm.netrid.dss_wrapper import DSSWrapper
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 VERTICES: List[s2sphere.LatLng] = [
     s2sphere.LatLng.from_degrees(lng=130.6205, lat=-23.6558),
@@ -88,7 +89,7 @@ def _get_entities_by_prefix(self, prefix: str) -> Dict[str, TestEntity]:
                 all_entities[entity.uuid] = entity
         return all_entities
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
 
         self.begin_test_case("Prerequisites")

monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py

@@ -31,6 +31,7 @@
     VirtualObserver,
 )
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class Misbehavior(GenericTestScenario):
@@ -69,7 +70,7 @@ def _rid_version(self) -> RIDVersion:
             "Misbehavior test scenario subclass must specify _rid_version"
         )
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
         self.begin_test_case("Unauthenticated requests")
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py

@@ -28,6 +28,7 @@
     VirtualObserver,
 )
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class NominalBehavior(GenericTestScenario):
@@ -61,7 +62,7 @@ def _rid_version(self) -> RIDVersion:
             "NominalBehavior test scenario subclass must specify _rid_version"
         )
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
         self.begin_test_case("Nominal flight")
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py

@@ -1,11 +1,12 @@
 from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 
 class OperatorInteractions(GenericTestScenario):
     def __init__(self):
         super().__init__()
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
         # TODO: Implement
         self.end_test_scenario()

monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.md

@@ -6,9 +6,6 @@ performances of the queries made during their execution.
 
 ## Resources
 
-### report_resource
-The report to evaluate. This resource is automatically injected by the test framework.
-
 ### service_providers
 The service providers to evaluate in the report.
 

monitoring/uss_qualifier/scenarios/astm/netrid/v19/aggregate_checks.py

@@ -1,6 +1,5 @@
 from monitoring.monitorlib.rid import RIDVersion
 from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource
-from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource
 from monitoring.uss_qualifier.resources.netrid import (
     NetRIDServiceProviders,
     NetRIDObserversResource,
@@ -14,10 +13,9 @@
 class AggregateChecks(TestScenario, CommonAggregateChecks):
     def __init__(
         self,
-        report_resource: TestSuiteReportResource,
         service_providers: NetRIDServiceProviders,
         observers: NetRIDObserversResource,
         dss_instances: DSSInstancesResource,
     ):
-        super().__init__(report_resource, service_providers, observers, dss_instances)
+        super().__init__(service_providers, observers, dss_instances)
         self._rid_version = RIDVersion.f3411_19

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.md

@@ -6,9 +6,6 @@ performances of the queries made during their execution.
 
 ## Resources
 
-### report_resource
-The report to evaluate. This resource is automatically injected by the test framework.
-
 ### service_providers
 The service providers to evaluate in the report.
 

monitoring/uss_qualifier/scenarios/astm/netrid/v22a/aggregate_checks.py

@@ -1,6 +1,5 @@
 from monitoring.monitorlib.rid import RIDVersion
 from monitoring.uss_qualifier.resources.astm.f3411 import DSSInstancesResource
-from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource
 from monitoring.uss_qualifier.resources.netrid import (
     NetRIDServiceProviders,
     NetRIDObserversResource,
@@ -14,10 +13,9 @@
 class AggregateChecks(TestScenario, CommonAggregateChecks):
     def __init__(
         self,
-        report_resource: TestSuiteReportResource,
         service_providers: NetRIDServiceProviders,
         observers: NetRIDObserversResource,
         dss_instances: DSSInstancesResource,
     ):
-        super().__init__(report_resource, service_providers, observers, dss_instances)
+        super().__init__(service_providers, observers, dss_instances)
         self._rid_version = RIDVersion.f3411_22a

monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.md

@@ -6,9 +6,6 @@ performances of the queries made during their execution.
 
 ## Resources
 
-### report_resource
-The report to evaluate. This resource is automatically injected by the test framework.
-
 ### flight_planners
 The flight planners subject to evaluation.
 

monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py

@@ -5,31 +5,27 @@
 from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.configurations.configuration import ParticipantID
 from monitoring.uss_qualifier.resources.flight_planning import FlightPlannersResource
-from monitoring.uss_qualifier.scenarios.interuss.evaluation_scenario import (
-    ReportEvaluationScenario,
-)
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 
 from uas_standards.astm.f3548.v21 import constants
 
-from monitoring.uss_qualifier.resources.interuss.report import TestSuiteReportResource
-
-
 from monitoring.uss_qualifier.scenarios.scenario import TestScenario
 
 
-class AggregateChecks(TestScenario, ReportEvaluationScenario):
+class AggregateChecks(TestScenario):
 
     _queries: List[fetch.Query]
     _attributed_queries: Dict[ParticipantID, Dict[QueryType, List[fetch.Query]]] = {}
 
     def __init__(
         self,
-        report_resource: TestSuiteReportResource,
         flight_planners: FlightPlannersResource,
     ):
-        super().__init__(report_resource)
+        super().__init__()
         self.flight_planners = flight_planners
-        self._queries = self.report.queries()
+
+    def _init_queries(self, context: ExecutionContext):
+        self._queries = list(context.sibling_queries())
 
         # collect and classify queries by participant, only participants part of flight_planners are considered
         self._attributed_queries = {
@@ -54,7 +50,8 @@ def __init__(
                     query
                 )
 
-    def run(self):
+    def run(self, context: ExecutionContext):
+        self._init_queries(context)
         self.begin_test_scenario()
 
         self.record_note("all_queries", f"{len(self._queries)}")

monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py

@@ -3,6 +3,7 @@
 from typing import List
 from urllib.parse import urlparse
 
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 from uas_standards.astm.f3548.v21.api import Volume4D, Volume3D, Polygon, LatLngPoint
 
 from monitoring.uss_qualifier.common_data_definitions import Severity
@@ -39,7 +40,7 @@ def __init__(
             if not dss.is_same_as(primary_dss_instance.dss)
         ]
 
-    def run(self):
+    def run(self, context: ExecutionContext):
 
         self.begin_test_scenario()
 

monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py

@@ -2,6 +2,7 @@
 
 from monitoring.monitorlib.geotemporal import Volume4DCollection
 from monitoring.uss_qualifier.common_data_definitions import Severity
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 from uas_standards.astm.f3548.v21.api import OperationalIntentState
 from uas_standards.astm.f3548.v21.constants import OiMaxPlanHorizonDays
 
@@ -141,7 +142,7 @@ def __init__(
                 f"`{self.me()}` TestScenario requirements for flight_intents not met: {e}"
             )
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
         self.record_note(
             "Tested USS",

monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py

@@ -1,6 +1,8 @@
 from typing import Optional
 
 import arrow
+
+from monitoring.uss_qualifier.suites.suite import ExecutionContext
 from uas_standards.astm.f3548.v21.api import (
     OperationalIntentReference,
 )
@@ -213,7 +215,7 @@ def __init__(
                 f"`{self.me()}` TestScenario requirements for flight_intents not met: {e}"
             )
 
-    def run(self):
+    def run(self, context: ExecutionContext):
         self.begin_test_scenario()
 
         self.record_note(