From 7840f3dbaa5cc6df816abf1ada3077d03194fd72 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 6 Nov 2023 15:39:00 -0800 Subject: [PATCH] [uss_qualifier] Change USS_QUALIFIER_STOP_FAST to execution setting (#318) Change USS_QUALIFIER_STOP_FAST to execution setting --- .github/workflows/CI.md | 2 +- .github/workflows/ci.yml | 3 +-- .../configurations/configuration.py | 3 +++ .../configurations/dev/dss_probing.yaml | 2 ++ .../dev/f3548_self_contained.yaml | 6 +++++ .../dev/general_flight_auth.yaml | 2 ++ .../dev/generate_rid_test_data.yaml | 2 ++ .../configurations/dev/geoawareness_cis.yaml | 2 ++ .../dev/geospatial_comprehension.yaml | 2 ++ .../configurations/dev/message_signing.yaml | 2 ++ .../configurations/dev/netrid_v19.yaml | 2 ++ .../configurations/dev/netrid_v22a.yaml | 2 ++ .../configurations/dev/uspace.yaml | 2 ++ monitoring/uss_qualifier/local_testing.md | 2 +- monitoring/uss_qualifier/run_locally.sh | 1 - .../scenarios/astm/dss/crdb_access.py | 2 +- .../astm/netrid/common/aggregate_checks.py | 2 +- .../astm/netrid/common/dss/isa_expiry.py | 2 +- .../astm/netrid/common/dss/isa_simple.py | 2 +- .../dss/isa_subscription_interactions.py | 2 +- .../astm/netrid/common/dss/isa_validation.py | 2 +- .../netrid/common/dss/subscription_simple.py | 2 +- .../common/dss/subscription_validation.py | 2 +- .../netrid/common/dss_interoperability.py | 2 +- .../astm/netrid/common/misbehavior.py | 2 +- .../astm/netrid/common/nominal_behavior.py | 2 +- .../netrid/common/operator_interactions.py | 2 +- .../astm/netrid/store_flight_data.py | 2 +- .../scenarios/astm/utm/aggregate_checks.py | 2 +- .../get_op_data_validation.py | 2 +- .../astm/utm/dss_interoperability.py | 2 +- .../flight_intent_validation.py | 2 +- .../conflict_equal_priority_not_permitted.py | 2 +- .../conflict_higher_priority.py | 2 +- .../uss_qualifier/scenarios/dev/noop.py | 2 +- .../eurocae/ed269/source_data_model.py | 2 +- .../faa/uft/message_signing_finalize.py | 2 +- .../faa/uft/message_signing_start.py | 2 +- .../flight_planning/record_planners.py | 2 +- .../general_flight_authorization.py | 2 +- .../geospatial_feature_comprehension.py | 2 +- .../interuss/mock_uss/configure_locality.py | 2 +- .../interuss/mock_uss/unconfigure_locality.py | 2 +- .../scenarios/interuss/unit_test.py | 2 +- .../uss_qualifier/scenarios/scenario.py | 24 +++++++++++++------ .../uspace/flight_auth/validation.py | 2 +- .../versioning/get_system_versions.py | 2 +- monitoring/uss_qualifier/suites/suite.py | 10 ++++++++ .../configuration/ExecutionConfiguration.json | 7 ++++++ 49 files changed, 95 insertions(+), 43 deletions(-) diff --git a/.github/workflows/CI.md b/.github/workflows/CI.md index a4c5d74106..45b9860ad9 100644 --- a/.github/workflows/CI.md +++ b/.github/workflows/CI.md @@ -38,7 +38,7 @@ This check runs unit tests for mock_uss. ### uss_qualifier tests (`make test` in monitoring/uss_qualifier) -Note: `export USS_QUALIFIER_STOP_FAST=true` is recommended as defining that environment variable will cause testing to stop on the first failure, and this is usually the desired behavior when debugging. +Note: setting configuration `stop_fast` to `true` is recommended as setting that flag in configuration will cause testing to stop on the first failure, and this is usually the desired behavior when debugging. This action: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33681f30c3..997c03dd84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,7 @@ jobs: with: name: uss_qualifier script: | - export CONFIG_NAME="" \ - USS_QUALIFIER_STOP_FAST=true + export CONFIG_NAME="" cd monitoring/uss_qualifier make test diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index 94db32259b..a76745a100 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -126,6 +126,9 @@ class ExecutionConfiguration(ImplicitDict): skip_action_when: Optional[List[TestSuiteActionSelectionCondition]] = None """If specified, do not execute test actions if they are selected by ANY of these conditions.""" + stop_fast: Optional[bool] = False + """If true, escalate the Severity of any failed check to Critical in order to end the test run early.""" + class TestConfiguration(ImplicitDict): action: TestSuiteActionDeclaration diff --git a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml index a6c1c6394b..9295ea56c1 100644 --- a/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/dss_probing.yaml @@ -25,6 +25,8 @@ v1: id_generator: id_generator service_area: kentland_service_area problematically_big_area: kentland_problematically_big_area + execution: + stop_fast: true artifacts: output_path: output/dss_probing raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml index db4fddfd0b..05bc2d1534 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml @@ -89,6 +89,12 @@ v1: - v1.test_run.resources.resource_declarations.flight_planners - v1.test_run.resources.resource_declarations.dss + # How to execute a test run using this configuration + execution: + # Since we expect no failed checks and want to stop execution immediately if there are any failed checks, we set + # this parameter to true. + stop_fast: true + # This block defines artifacts related to the test run. Note that all paths are # relative to where uss_qualifier is executed from, and are located inside the # Docker container executing uss_qualifier. diff --git a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml index e1d39e5280..fffa591122 100644 --- a/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml +++ b/monitoring/uss_qualifier/configurations/dev/general_flight_auth.yaml @@ -16,6 +16,8 @@ v1: resources: table: example_flight_check_table planner: uss1_flight_planner + execution: + stop_fast: true artifacts: output_path: output/general_flight_auth raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.yaml b/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.yaml index 6ec98c40da..7fd2612cc1 100644 --- a/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.yaml +++ b/monitoring/uss_qualifier/configurations/dev/generate_rid_test_data.yaml @@ -61,5 +61,7 @@ v1: kml_flights_data: kml_flights_data kml_storage_config: kml_storage_config on_failure: Continue + execution: + stop_fast: true validation: "$ref": "./library/validation.yaml#/normal_test" diff --git a/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml b/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml index 373b631fea..fe18def589 100644 --- a/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml +++ b/monitoring/uss_qualifier/configurations/dev/geoawareness_cis.yaml @@ -12,6 +12,8 @@ v1: suite_type: suites.uspace.geo_awareness_cis resources: source_document: source_document + execution: + stop_fast: true artifacts: output_path: output/geoawareness_cis raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml b/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml index d48799efb5..00753667cd 100644 --- a/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml +++ b/monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml @@ -9,6 +9,8 @@ v1: scenario_type: scenarios.interuss.geospatial_map.GeospatialFeatureComprehension resources: table: example_feature_check_table + execution: + stop_fast: true artifacts: output_path: output/geospatial_comprehension raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml index dde36e218e..c2b2657d0d 100644 --- a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml @@ -48,6 +48,8 @@ v1: conflicting_flights: che_conflicting_flights priority_preemption_flights: che_conflicting_flights dss: scd_dss + execution: + stop_fast: true artifacts: output_path: output/message_signing diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml index 3a63bd3cb3..9b5bca2072 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml @@ -30,6 +30,8 @@ v1: id_generator: id_generator service_area: kentland_service_area problematically_big_area: au_problematically_big_area + execution: + stop_fast: true artifacts: output_path: output/netrid_v19 raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml index 5441555c5a..7a9904ca17 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml @@ -30,6 +30,8 @@ v1: id_generator: id_generator service_area: kentland_service_area problematically_big_area: au_problematically_big_area + execution: + stop_fast: true artifacts: output_path: output/netrid_v22a raw_report: {} diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index 8aa10486c9..72581df5ac 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -82,6 +82,8 @@ v1: id_generator: id_generator service_area: service_area problematically_big_area: problematically_big_area + execution: + stop_fast: true artifacts: output_path: output/uspace raw_report: {} diff --git a/monitoring/uss_qualifier/local_testing.md b/monitoring/uss_qualifier/local_testing.md index b854dd4801..c1f90d6a13 100644 --- a/monitoring/uss_qualifier/local_testing.md +++ b/monitoring/uss_qualifier/local_testing.md @@ -32,7 +32,7 @@ To tear down the complete set of local mocks (or ensure it is completely torn do Once the local interoperability ecosystem infrastructure and complete set of local mocks are deployed, execute uss_qualifier with [`./monitoring/uss_qualifier/run_locally.sh [CONFIG_NAME]`](run_locally.sh). If no `CONFIG_NAME` is specified, all configurations used for the [continuous integration](../../.github/workflows) are executed; this involves multiple invocations of uss_qualifier. Otherwise, a specific [configuration name](configurations/README.md#specifying) can be specified; e.g., [`configurations.dev.uspace`](configurations/dev/uspace.yaml). -To cause uss_qualifier to stop at the first test failure (this is often useful when attempting to debug a test that should run fully successfully), set the USS_QUALIFIER_STOP_FAST environment variable with `export USS_QUALIFIER_STOP_FAST=true`. To disable this behavior, `unset USS_QUALIFIER_STOP_FAST`. +To cause uss_qualifier to stop at the first test failure (this is often useful when attempting to debug a test that should run fully successfully), set the `stop_fast` flag in the `execution` portion of the configuration. ## Presubmit testing diff --git a/monitoring/uss_qualifier/run_locally.sh b/monitoring/uss_qualifier/run_locally.sh index da4423dc2a..d6efcc3721 100755 --- a/monitoring/uss_qualifier/run_locally.sh +++ b/monitoring/uss_qualifier/run_locally.sh @@ -66,7 +66,6 @@ docker run ${docker_args} --name uss_qualifier \ -u "$(id -u):$(id -g)" \ -e PYTHONBUFFERED=1 \ -e AUTH_SPEC=${AUTH_SPEC} \ - -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \ -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \ -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \ -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \ diff --git a/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py b/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py index 2ddcb48f79..c21045e0ad 100644 --- a/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py +++ b/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py @@ -7,6 +7,6 @@ def __init__(self): super().__init__() def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) # TODO: Implement self.end_test_scenario() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py index b5c13acf21..4e4f15fe3d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/aggregate_checks.py @@ -96,7 +96,7 @@ def _init_queries(self, context: ExecutionContext): def run(self, context: ExecutionContext): self._init_queries(context) - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note("participants", str(self._participants_by_base_url)) self.record_note("nb_queries", str(len(self._queries))) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py index df1c209d04..8925025bf0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_expiry.py @@ -45,7 +45,7 @@ def __init__( self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self._setup_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index 3a1e03be18..9e16e1033b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -49,7 +49,7 @@ def __init__( ] def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self._setup_case() self._create_and_check_isa_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py index 78c47fc46c..1b104c23d5 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_subscription_interactions.py @@ -44,7 +44,7 @@ def __init__( self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self._setup_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py index 210c97deec..5cf8136fb6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validation.py @@ -72,7 +72,7 @@ def __init__( ValueError(f"Unsupported RID version '{self._dss.rid_version}'") def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self._setup_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_simple.py index 778b7f5c84..55a7bd3025 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_simple.py @@ -95,7 +95,7 @@ def __init__( ] def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) loguru.logger.info("setup") self._setup_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py index 3fbd4d31f9..06ed097505 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py @@ -48,7 +48,7 @@ def __init__( self._isa_area = [vertex.as_s2sphere() for vertex in self._isa.footprint] def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self._setup_case() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py index 9b2afc830f..376ca44586 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py @@ -90,7 +90,7 @@ def _get_entities_by_prefix(self, prefix: str) -> Dict[str, TestEntity]: return all_entities def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Prerequisites") diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py index fac17a0464..00ce4e9196 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/misbehavior.py @@ -71,7 +71,7 @@ def _rid_version(self) -> RIDVersion: ) def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Unauthenticated requests") self.begin_test_step("Injection") diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index e20e3487ef..92f95c81e1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -63,7 +63,7 @@ def _rid_version(self) -> RIDVersion: ) def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Nominal flight") self.begin_test_step("Injection") diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py index a29155e655..c2da3d2d69 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py @@ -7,6 +7,6 @@ def __init__(self): super().__init__() def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) # TODO: Implement self.end_test_scenario() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/store_flight_data.py b/monitoring/uss_qualifier/scenarios/astm/netrid/store_flight_data.py index 9a6fc6e5d5..04c05fa695 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/store_flight_data.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/store_flight_data.py @@ -26,7 +26,7 @@ def __init__( self._storage_config = storage_configuration def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note( "Flight count", f"{len(self._flights_data.flight_collection.flights)} flights", diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py index dca6567f50..bd4dc50afd 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/aggregate_checks.py @@ -52,7 +52,7 @@ def _init_queries(self, context: ExecutionContext): def run(self, context: ExecutionContext): self._init_queries(context) - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note("all_queries", f"{len(self._queries)}") for participant, queries_by_type in self._attributed_queries.items(): diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py index 063b1677d1..79997a1342 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py @@ -47,6 +47,6 @@ def __init__( self.dss = dss.dss def run(self): - self.begin_test_scenario() + self.begin_test_scenario(context) pass self.end_test_scenario() diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py index 06aa4002e4..ce59ff94fe 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss_interoperability.py @@ -42,7 +42,7 @@ def __init__( def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Prerequisites") diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py index c4541480df..d571b7966c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py @@ -143,7 +143,7 @@ def __init__( ) def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note( "Tested USS", f"{self.tested_uss.config.participant_id}", diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py index 75f99a550a..599f85b88e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py @@ -216,7 +216,7 @@ def __init__( ) def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note( "Tested USS", diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py index 5dc63a9219..b27fa4465f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py @@ -195,7 +195,7 @@ def __init__( ) def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note( "Tested USS", diff --git a/monitoring/uss_qualifier/scenarios/dev/noop.py b/monitoring/uss_qualifier/scenarios/dev/noop.py index e938233313..7272905cc0 100644 --- a/monitoring/uss_qualifier/scenarios/dev/noop.py +++ b/monitoring/uss_qualifier/scenarios/dev/noop.py @@ -12,7 +12,7 @@ def __init__(self, noop_config: NoOpResource): self.sleep_secs = noop_config.sleep_secs def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Sleep") self.begin_test_step("Sleep") diff --git a/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py b/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py index 3baa318dee..4ddc29da52 100644 --- a/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py +++ b/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py @@ -25,7 +25,7 @@ def __init__(self, source_document: SourceDocument): self.source_document = source_document def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note( "Document", diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py index 5e63bf8aaa..e1f1d464de 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py @@ -15,7 +15,7 @@ def __init__(self, mock_uss: MockUSSResource): self._mock_uss = mock_uss.mock_uss def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Finalize message signing") diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py index 7848eb70da..004cc322b1 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py @@ -15,7 +15,7 @@ def __init__(self, mock_uss: MockUSSResource): self._mock_uss = mock_uss.mock_uss def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Start message signing") diff --git a/monitoring/uss_qualifier/scenarios/flight_planning/record_planners.py b/monitoring/uss_qualifier/scenarios/flight_planning/record_planners.py index 9578aa8eaa..f4e4860b9e 100644 --- a/monitoring/uss_qualifier/scenarios/flight_planning/record_planners.py +++ b/monitoring/uss_qualifier/scenarios/flight_planning/record_planners.py @@ -11,7 +11,7 @@ def __init__(self, flight_planners: FlightPlannersResource): self._flight_planners = flight_planners def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note( "Available flight planners", diff --git a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py index 68a4595265..f3279905ce 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py +++ b/monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py @@ -60,7 +60,7 @@ def __init__( self.participant_id = planner.participant_id def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Flight planning") self._plan_flights() diff --git a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py index fe45c724ea..1b9df5b774 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py +++ b/monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py @@ -36,7 +36,7 @@ def __init__( self.table = table.table def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Map query") self._map_query() diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py index cb52f10ea6..09d7008035 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py @@ -31,7 +31,7 @@ def __init__( self.to_unconfigure = [] def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Set locality") diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py index 636b86de5a..604bb4798d 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py @@ -22,7 +22,7 @@ class MockUSSLocalityConfiguration(object): class UnconfigureLocality(TestScenario): def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) if not unconfigure_stack: raise ValueError( diff --git a/monitoring/uss_qualifier/scenarios/interuss/unit_test.py b/monitoring/uss_qualifier/scenarios/interuss/unit_test.py index 5f49b60c7e..9d1590a961 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/unit_test.py +++ b/monitoring/uss_qualifier/scenarios/interuss/unit_test.py @@ -14,7 +14,7 @@ def __init__(self, step_under_test: Callable[["UnitTestScenario"], None]): super().__init__() def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Case under test") self.begin_test_step("Step under test") self.step_under_test(self) diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index c903e0fc0c..be95a1a1fb 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -1,4 +1,3 @@ -import os from abc import ABC, abstractmethod from datetime import datetime from enum import Enum @@ -37,9 +36,6 @@ from monitoring.uss_qualifier.resources.definitions import ResourceTypeName, ResourceID from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation -_STOP_FAST_FLAG = "USS_QUALIFIER_STOP_FAST" -STOP_FAST = os.environ.get(_STOP_FAST_FLAG, "").strip().lower() == "true" - class ScenarioCannotContinueError(Exception): def __init__(self, msg): @@ -66,6 +62,7 @@ class PendingCheck(object): _phase: ScenarioPhase _documentation: TestCheckDocumentation _step_report: TestStepReport + _stop_fast: bool _on_failed_check: Optional[Callable[[FailedCheck], None]] _participants: List[ParticipantID] _outcome_recorded: bool = False @@ -76,12 +73,14 @@ def __init__( documentation: TestCheckDocumentation, participants: List[ParticipantID], step_report: TestStepReport, + stop_fast: bool, on_failed_check: Optional[Callable[[FailedCheck], None]], ): self._phase = phase self._documentation = documentation self._participants = participants self._step_report = step_report + self._stop_fast = stop_fast self._on_failed_check = on_failed_check def __enter__(self): @@ -108,11 +107,11 @@ def record_failed( requirements = self._documentation.applicable_requirements if ( - STOP_FAST + self._stop_fast and severity != Severity.Critical and self._phase != ScenarioPhase.CleaningUp ): - note = f"Severity {severity} upgraded to Critical because {_STOP_FAST_FLAG} environment variable indicates true" + note = f"Severity {severity} upgraded to Critical because `stop_fast` flag set true in configuration" logger.info(note) details += "\n" + note severity = Severity.Critical @@ -193,6 +192,9 @@ class GenericTestScenario(ABC): _step_report: Optional[TestStepReport] = None _allow_undocumented_checks = False # When this variable is set to True, it allows undocumented checks to be executed by the scenario. This is primarly intended to simplify internal unit testing. + _context = None + """Execution context; set at begin_test_scenario.""" + def __init__(self): self.documentation = get_documentation(self.__class__) self._phase = ScenarioPhase.NotStarted @@ -284,7 +286,14 @@ def record_note(self, key: str, message: str) -> None: ) logger.info(f"Note: {key} -> {message}") - def begin_test_scenario(self) -> None: + def begin_test_scenario(self, context) -> None: + """Indicate that test scenario execution is beginning. + + Args: + context: Execution context with type monitoring.uss_qualifier.suites.suite.ExecutionContext. Type hint is + not annotated because doing so would create a circular reference. + """ + self._context = context self._expect_phase(ScenarioPhase.NotStarted) self._make_scenario_report() self._phase = ScenarioPhase.ReadyForTestCase @@ -400,6 +409,7 @@ def check( documentation=check_documentation, participants=[] if participants is None else participants, step_report=self._step_report, + stop_fast=self._context.stop_fast, on_failed_check=self.on_failed_check, ) diff --git a/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py b/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py index d1b4451533..440d1442ca 100644 --- a/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py +++ b/monitoring/uss_qualifier/scenarios/uspace/flight_auth/validation.py @@ -69,7 +69,7 @@ def __init__( ) def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.record_note("Planner", self.ussp.participant_id) diff --git a/monitoring/uss_qualifier/scenarios/versioning/get_system_versions.py b/monitoring/uss_qualifier/scenarios/versioning/get_system_versions.py index e5837da030..b5186ae64e 100644 --- a/monitoring/uss_qualifier/scenarios/versioning/get_system_versions.py +++ b/monitoring/uss_qualifier/scenarios/versioning/get_system_versions.py @@ -19,7 +19,7 @@ def __init__( self._system_identity = system_identity.system_identity def run(self, context: ExecutionContext): - self.begin_test_scenario() + self.begin_test_scenario(context) self.begin_test_case("Get versions") self.begin_test_step("Get versions") diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index e6dcca1705..f5e27578bb 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -396,6 +396,16 @@ def sibling_queries(self) -> Iterator[Query]: for q in child.report.queries(): yield q + @property + def stop_fast(self) -> bool: + if ( + self.config is not None + and "stop_fast" in self.config + and self.config.stop_fast is not None + ): + return self.config.stop_fast + return False + def _compute_n_of( self, target: TestSuiteAction, condition: TestSuiteActionSelectionCondition ) -> int: diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json index b94efac0a6..103698bbbd 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json @@ -26,6 +26,13 @@ "array", "null" ] + }, + "stop_fast": { + "description": "If true, escalate the Severity of any failed check to Critical in order to end the test run early.", + "type": [ + "boolean", + "null" + ] } }, "type": "object"