[uss_qualifier] Change USS_QUALIFIER_STOP_FAST to execution setting (#…

…318)

Change USS_QUALIFIER_STOP_FAST to execution setting

.github/workflows/CI.md

@@ -38,7 +38,7 @@ This check runs unit tests for mock_uss.
 
 ### uss_qualifier tests (`make test` in monitoring/uss_qualifier)
 
-Note: `export USS_QUALIFIER_STOP_FAST=true` is recommended as defining that environment variable will cause testing to stop on the first failure, and this is usually the desired behavior when debugging.
+Note: setting configuration `stop_fast` to `true` is recommended as setting that flag in configuration will cause testing to stop on the first failure, and this is usually the desired behavior when debugging.
 
 This action:
 

.github/workflows/ci.yml

@@ -45,8 +45,7 @@ jobs:
     with:
       name: uss_qualifier
       script: |
-        export CONFIG_NAME="" \
-          USS_QUALIFIER_STOP_FAST=true
+        export CONFIG_NAME=""
 
         cd monitoring/uss_qualifier
         make test

monitoring/uss_qualifier/configurations/configuration.py

@@ -126,6 +126,9 @@ class ExecutionConfiguration(ImplicitDict):
     skip_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None
     """If specified, do not execute test actions if they are selected by ANY of these conditions."""
 
+    stop_fast: Optional[bool] = False
+    """If true, escalate the Severity of any failed check to Critical in order to end the test run early."""
+
 
 class TestConfiguration(ImplicitDict):
     action: TestSuiteActionDeclaration

monitoring/uss_qualifier/configurations/dev/dss_probing.yaml

@@ -25,6 +25,8 @@ v1:
           id_generator: id_generator
           service_area: kentland_service_area
           problematically_big_area: kentland_problematically_big_area
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/dss_probing
     raw_report: {}

monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml

@@ -89,6 +89,12 @@ v1:
       - v1.test_run.resources.resource_declarations.flight_planners
       - v1.test_run.resources.resource_declarations.dss
 
+    # How to execute a test run using this configuration
+    execution:
+      # Since we expect no failed checks and want to stop execution immediately if there are any failed checks, we set
+      # this parameter to true.
+      stop_fast: true
+
   # This block defines artifacts related to the test run.  Note that all paths are
   # relative to where uss_qualifier is executed from, and are located inside the
   # Docker container executing uss_qualifier.

monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml

@@ -16,6 +16,8 @@ v1:
         resources:
           table: example_flight_check_table
           planner: uss1_flight_planner
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/general_flight_auth
     raw_report: {}

monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.yaml

@@ -61,5 +61,7 @@ v1:
                   kml_flights_data: kml_flights_data
                   kml_storage_config: kml_storage_config
               on_failure: Continue
+    execution:
+      stop_fast: true
   validation:
     "$ref": "./library/validation.yaml#/normal_test"

monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml

@@ -12,6 +12,8 @@ v1:
         suite_type: suites.uspace.geo_awareness_cis
         resources:
           source_document: source_document
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/geoawareness_cis
     raw_report: {}

monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml

@@ -9,6 +9,8 @@ v1:
         scenario_type: scenarios.interuss.geospatial_map.GeospatialFeatureComprehension
         resources:
           table: example_feature_check_table
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/geospatial_comprehension
     raw_report: {}

monitoring/uss_qualifier/configurations/dev/message_signing.yaml

@@ -48,6 +48,8 @@ v1:
           conflicting_flights: che_conflicting_flights
           priority_preemption_flights: che_conflicting_flights
           dss: scd_dss
+    execution:
+      stop_fast: true
 
   artifacts:
     output_path: output/message_signing

monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml

@@ -30,6 +30,8 @@ v1:
           id_generator: id_generator
           service_area: kentland_service_area
           problematically_big_area: au_problematically_big_area
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/netrid_v19
     raw_report: {}

monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml

@@ -30,6 +30,8 @@ v1:
           id_generator: id_generator
           service_area: kentland_service_area
           problematically_big_area: au_problematically_big_area
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/netrid_v22a
     raw_report: {}

monitoring/uss_qualifier/configurations/dev/uspace.yaml

@@ -82,6 +82,8 @@ v1:
                 id_generator: id_generator
                 service_area: service_area
                 problematically_big_area: problematically_big_area
+    execution:
+      stop_fast: true
   artifacts:
     output_path: output/uspace
     raw_report: {}

monitoring/uss_qualifier/local_testing.md

@@ -32,7 +32,7 @@ To tear down the complete set of local mocks (or ensure it is completely torn do
 
 Once the local interoperability ecosystem infrastructure and complete set of local mocks are deployed, execute uss_qualifier with [`./monitoring/uss_qualifier/run_locally.sh [CONFIG_NAME]`](run_locally.sh).  If no `CONFIG_NAME` is specified, all configurations used for the [continuous integration](../../.github/workflows) are executed; this involves multiple invocations of uss_qualifier.  Otherwise, a specific [configuration name](configurations/README.md#specifying) can be specified; e.g., [`configurations.dev.uspace`](configurations/dev/uspace.yaml).
 
-To cause uss_qualifier to stop at the first test failure (this is often useful when attempting to debug a test that should run fully successfully), set the USS_QUALIFIER_STOP_FAST environment variable with `export USS_QUALIFIER_STOP_FAST=true`.  To disable this behavior, `unset USS_QUALIFIER_STOP_FAST`.
+To cause uss_qualifier to stop at the first test failure (this is often useful when attempting to debug a test that should run fully successfully), set the `stop_fast` flag in the `execution` portion of the configuration.
 
 ## Presubmit testing
 

monitoring/uss_qualifier/run_locally.sh

@@ -66,7 +66,6 @@ docker run ${docker_args} --name uss_qualifier \
   -u "$(id -u):$(id -g)" \
   -e PYTHONBUFFERED=1 \
   -e AUTH_SPEC=${AUTH_SPEC} \
-  -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \
   -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
   -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
   -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \

monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py

@@ -7,6 +7,6 @@ def __init__(self):
         super().__init__()
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         # TODO: Implement
         self.end_test_scenario()

monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py

@@ -96,7 +96,7 @@ def _init_queries(self, context: ExecutionContext):
     def run(self, context: ExecutionContext):
         self._init_queries(context)
 
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.record_note("participants", str(self._participants_by_base_url))
         self.record_note("nb_queries", str(len(self._queries)))

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py

@@ -45,7 +45,7 @@ def __init__(
         self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint]
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self._setup_case()
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py

@@ -49,7 +49,7 @@ def __init__(
         ]
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self._setup_case()
         self._create_and_check_isa_case()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py

@@ -44,7 +44,7 @@ def __init__(
         self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint]
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self._setup_case()
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py

@@ -72,7 +72,7 @@ def __init__(
             ValueError(f"Unsupported RID version '{self._dss.rid_version}'")
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self._setup_case()
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_simple.py

@@ -95,7 +95,7 @@ def __init__(
         ]
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         loguru.logger.info("setup")
         self._setup_case()

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py

@@ -48,7 +48,7 @@ def __init__(
         self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint]
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self._setup_case()
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py

@@ -90,7 +90,7 @@ def _get_entities_by_prefix(self, prefix: str) -> Dict[str, TestEntity]:
         return all_entities
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Prerequisites")
 

monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py

@@ -71,7 +71,7 @@ def _rid_version(self) -> RIDVersion:
         )
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         self.begin_test_case("Unauthenticated requests")
 
         self.begin_test_step("Injection")

monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py

@@ -63,7 +63,7 @@ def _rid_version(self) -> RIDVersion:
         )
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         self.begin_test_case("Nominal flight")
 
         self.begin_test_step("Injection")

monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py

@@ -7,6 +7,6 @@ def __init__(self):
         super().__init__()
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         # TODO: Implement
         self.end_test_scenario()

monitoring/uss_qualifier/scenarios/astm/netrid/store_flight_data.py

@@ -26,7 +26,7 @@ def __init__(
         self._storage_config = storage_configuration
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         self.record_note(
             "Flight count",
             f"{len(self._flights_data.flight_collection.flights)} flights",

monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py

@@ -52,7 +52,7 @@ def _init_queries(self, context: ExecutionContext):
 
     def run(self, context: ExecutionContext):
         self._init_queries(context)
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.record_note("all_queries", f"{len(self._queries)}")
         for participant, queries_by_type in self._attributed_queries.items():

monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py

@@ -47,6 +47,6 @@ def __init__(
         self.dss = dss.dss
 
     def run(self):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         pass
         self.end_test_scenario()

monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py

@@ -42,7 +42,7 @@ def __init__(
 
     def run(self, context: ExecutionContext):
 
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Prerequisites")
 

monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py

@@ -143,7 +143,7 @@ def __init__(
             )
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         self.record_note(
             "Tested USS",
             f"{self.tested_uss.config.participant_id}",

monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py

@@ -216,7 +216,7 @@ def __init__(
             )
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.record_note(
             "Tested USS",

monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py

@@ -195,7 +195,7 @@ def __init__(
             )
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.record_note(
             "Tested USS",

monitoring/uss_qualifier/scenarios/dev/noop.py

@@ -12,7 +12,7 @@ def __init__(self, noop_config: NoOpResource):
         self.sleep_secs = noop_config.sleep_secs
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         self.begin_test_case("Sleep")
         self.begin_test_step("Sleep")
 

monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py

@@ -25,7 +25,7 @@ def __init__(self, source_document: SourceDocument):
         self.source_document = source_document
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.record_note(
             "Document",

monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py

@@ -15,7 +15,7 @@ def __init__(self, mock_uss: MockUSSResource):
         self._mock_uss = mock_uss.mock_uss
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Finalize message signing")
 

monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py

@@ -15,7 +15,7 @@ def __init__(self, mock_uss: MockUSSResource):
         self._mock_uss = mock_uss.mock_uss
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Start message signing")
 

monitoring/uss_qualifier/scenarios/flight_planning/record_planners.py

@@ -11,7 +11,7 @@ def __init__(self, flight_planners: FlightPlannersResource):
         self._flight_planners = flight_planners
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.record_note(
             "Available flight planners",

monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py

@@ -60,7 +60,7 @@ def __init__(
         self.participant_id = planner.participant_id
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Flight planning")
         self._plan_flights()

monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py

@@ -36,7 +36,7 @@ def __init__(
         self.table = table.table
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Map query")
         self._map_query()

monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py

@@ -31,7 +31,7 @@ def __init__(
         self.to_unconfigure = []
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         self.begin_test_case("Set locality")
 

monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py

@@ -22,7 +22,7 @@ class MockUSSLocalityConfiguration(object):
 
 class UnconfigureLocality(TestScenario):
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
 
         if not unconfigure_stack:
             raise ValueError(

monitoring/uss_qualifier/scenarios/interuss/unit_test.py

@@ -14,7 +14,7 @@ def __init__(self, step_under_test: Callable[["UnitTestScenario"], None]):
         super().__init__()
 
     def run(self, context: ExecutionContext):
-        self.begin_test_scenario()
+        self.begin_test_scenario(context)
         self.begin_test_case("Case under test")
         self.begin_test_step("Step under test")
         self.step_under_test(self)