diff --git a/monitoring/uss_qualifier/action_generators/README.md b/monitoring/uss_qualifier/action_generators/README.md new file mode 100644 index 0000000000..ffe5fcda69 --- /dev/null +++ b/monitoring/uss_qualifier/action_generators/README.md @@ -0,0 +1,5 @@ +# Action generators + +The bulk of uss_qualifier's automated testing logic is contained in [test scenarios](../scenarios/README.md). A [test suite](../suites/README.md) is essentially a static "playlist" of test actions to perform (test scenarios, action generators, and other test suites), all of which ultimately resolve to test scenarios. An action generator is essentially a dynamic "playlist" of test actions -- it can generate test actions that vary according to provided resource values, situations, or other conditions only necessarily known at runtime. + +For documentation purposes, all action generators must statically declare the test actions they may take. However, whether each (or any) of these actions will actually be taken at runtime cannot be statically determined in general. diff --git a/monitoring/uss_qualifier/scenarios/astm/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/dss/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py b/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py new file mode 100644 index 0000000000..46d51cb813 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/dss/crdb_access.py @@ -0,0 +1,9 @@ +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + + +class CRDBAccess(GenericTestScenario): + def __init__(self): + super().__init__() + + def run(self): + pass # TODO: Implement diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py new file mode 100644 index 0000000000..15d9ed03f7 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/operator_interactions.py @@ -0,0 +1,9 @@ +from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario + + +class OperatorInteractions(GenericTestScenario): + def __init__(self): + super().__init__() + + def run(self): + pass # TODO: Implement diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py index ab08466a88..c6ecd1745b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/__init__.py @@ -2,3 +2,4 @@ from .nominal_behavior import NominalBehavior from .misbehavior import Misbehavior from .aggregate_checks import AggregateChecks +from .operator_interactions import OperatorInteractions diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py index 266bb9c958..b34b4d2c7c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/__init__.py @@ -1 +1,2 @@ from .subscription_validation import SubscriptionValidation +from .crdb_access import CRDBAccess diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/crdb_access.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/crdb_access.md new file mode 100644 index 0000000000..cc50418996 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/crdb_access.md @@ -0,0 +1,37 @@ +# ASTM NetRID DSS: Direct CRDB access test scenario + +## Overview + +Attempt to directly access the CockroachDB (CRDB) nodes intercommunicating to form the DSS Airspace Representation for the DSS instances under test, for the purpose of determining compliance to certain DSS interoperability requirements. + +Note that none of this scenario is implemented yet. + +## Resources + +### crdb_nodes + +Set of CockroachDB nodes constituting the DSS instances under test. + +TODO: Create this resource + +## Verify security interoperability test case + +### Attempt unauthorized access test step + +In this test step, uss_qualifier attempts to connect to each CRDB node in insecure mode. + +#### CRDB node in insecure mode check + +If connection to a CRDB node in insecure mode succeeds, the USS will have failed to authenticate clients per **[astm.f3411.v19.DSS0110](../../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement this step and check + +### Verify encryption test step + +This step verifies the use of TLS for every CRDB node specified. + +#### TLS in use check + +If a CRDB node does not have TLS in use, the test will have failed to verify the encryption requirement **[astm.f3411.v19.DSS0120](../../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement this step and check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/crdb_access.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/crdb_access.py new file mode 100644 index 0000000000..fc872bd5e8 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/crdb_access.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.dss.crdb_access import ( + CRDBAccess as CommonCRDBAccess, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class CRDBAccess(TestScenario, CommonCRDBAccess): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md new file mode 100644 index 0000000000..6c83858c44 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md @@ -0,0 +1,95 @@ +# ASTM NetRID: Operator interactions test scenario + +## Overview + +Set up situations in which operator interactions (notifications) are required, and verify that those notifications are observed for the USS under test. + +Note that none of this scenario is implemented yet. + +## Resources + +### service_provider + +A singular `NetRIDServiceProvider` to be tested via the injection of RID flight data. + +TODO: Create this resource + +### operator_notifications + +Means by which to ask "What user/operator notifications have been observed for user/operator X (from the USS under test) over time period Z". + +TODO: Create this resource + +### flights_data + +A [`FlightDataResource`](../../../../resources/netrid/flight_data.py) containing 1 flight. This flight must: +* (Phase 1): Start out nominal +* (Phase 2): Then contain a pause (lack of telemetry) sufficient to trigger NET0040 +* (Phase 3): Then resume nominal telemetry +* (Phase 4): Then contain insufficient data to trigger NET0030 + +### orchestrated_dss + +A DSS instance that is equipped to fail on command, and will be used by the USS under test. + +TODO: Create this resource + +## Failed ISA test case + +### Verify no ISAs test step + +uss_qualifier checks the DSS to ensure that the Service Provider under test does not have any ISAs in the system. If ISAs are present, the Service Provider is instructed to clear the area of active flights, after which uss_qualifier reverifies the absence of ISAs. + +### Disable DSS test step + +uss_qualifier commands the orchestrated DSS to fail when interacting with normal clients. + +### Enumerate pre-existing operator notifications test step + +uss_qualifier retrieves the current (pre-existing) set of operator notifications. + +### Inject flight test step + +uss_qualifier attempts to inject a flight into the Service Provider under test, knowing that the Service Provider will not be able to create an ISA. + +#### Flight failed check + +Since the DSS is known to fail when attempting to create an ISA, if the Service Provider successfully creates the flight, they will have not met **[astm.f3411.v19.NET0610](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement + +### Enumerate operator notifications test step + +uss_qualifier retrieves the current (after failed flight) set of operator notifications. + +#### Operator notified of discoverability failure check + +The "after" set of operator notifications should contain at least one more entry than the "before" set of operator notifications. If there was no new operator notification, the Service Provider will not have met **[astm.f3411.v19.NET0620](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement + +## In-flight notifications test case + +### Inject flight test step + +uss_qualifier injects the flight into the Service Provider under test with the intention of observing the whole flight. + +### Enumerate pre-existing operator notifications test step + +uss_qualifier retrieves the current (pre-existing) set of operator notifications. + +### Poll Service Provider test step + +Throughout the duration of the flight, uss_qualifier periodically polls for operator notifications. + +#### Insufficient telemetry operator notification check + +If the Service Provider under test does not provide an operator notification in Phase 2 (regarding the telemetry feed stopping), it will not have complied with **[astm.f3411.v19.NET0040](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement + +#### Missing data operator notification check + +If the Service Provider under test does not provide an operator notification in Phase 4 (regarding missing data fields in reported telemetry), it will not have complied with **[astm.f3411.v19.NET0030](../../../../requirements/astm/f3411/v19.md)**. + +TODO: Implement diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.py new file mode 100644 index 0000000000..c914c7c691 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.operator_interactions import ( + OperatorInteractions as CommonOperatorInteractions, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class OperatorInteractions(TestScenario, CommonOperatorInteractions): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py index ab08466a88..c6ecd1745b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/__init__.py @@ -2,3 +2,4 @@ from .nominal_behavior import NominalBehavior from .misbehavior import Misbehavior from .aggregate_checks import AggregateChecks +from .operator_interactions import OperatorInteractions diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py index 3cd77afc8e..a3a46b4509 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/__init__.py @@ -1,2 +1,3 @@ from .isa_simple import ISASimple from .subscription_validation import SubscriptionValidation +from .crdb_access import CRDBAccess diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/crdb_access.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/crdb_access.md new file mode 100644 index 0000000000..c82efc893b --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/crdb_access.md @@ -0,0 +1,37 @@ +# ASTM NetRID DSS: Direct CRDB access test scenario + +## Overview + +Attempt to directly access the CockroachDB (CRDB) nodes intercommunicating to form the DSS Airspace Representation for the DSS instances under test, for the purpose of determining compliance to certain DSS interoperability requirements. + +Note that none of this scenario is implemented yet. + +## Resources + +### crdb_nodes + +Set of CockroachDB nodes constituting the DSS instances under test. + +TODO: Create this resource + +## Verify security interoperability test case + +### Attempt unauthorized access test step + +In this test step, uss_qualifier attempts to connect to each CRDB node in insecure mode. + +#### CRDB node in insecure mode check + +If connection to a CRDB node in insecure mode succeeds, the USS will have failed to authenticate clients per **[astm.f3411.v22a.DSS0110](../../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement this step and check + +### Verify encryption test step + +This step verifies the use of TLS for every CRDB node specified. + +#### TLS in use check + +If a CRDB node does not have TLS in use, the test will have failed to verify the encryption requirement **[astm.f3411.v22a.DSS0120](../../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement this step and check diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/crdb_access.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/crdb_access.py new file mode 100644 index 0000000000..fc872bd5e8 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/crdb_access.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.dss.crdb_access import ( + CRDBAccess as CommonCRDBAccess, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class CRDBAccess(TestScenario, CommonCRDBAccess): + pass diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md new file mode 100644 index 0000000000..503b4de63a --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md @@ -0,0 +1,95 @@ +# ASTM NetRID: Operator interactions test scenario + +## Overview + +Set up situations in which operator interactions (notifications) are required, and verify that those notifications are observed for the USS under test. + +Note that none of this scenario is implemented yet. + +## Resources + +### service_provider + +A singular `NetRIDServiceProvider` to be tested via the injection of RID flight data. + +TODO: Create this resource + +### operator_notifications + +Means by which to ask "What user/operator notifications have been observed for user/operator X (from the USS under test) over time period Z". + +TODO: Create this resource + +### flights_data + +A [`FlightDataResource`](../../../../resources/netrid/flight_data.py) containing 1 flight. This flight must: +* (Phase 1): Start out nominal +* (Phase 2): Then contain a pause (lack of telemetry) sufficient to trigger NET0040 +* (Phase 3): Then resume nominal telemetry +* (Phase 4): Then contain insufficient data to trigger NET0030 + +### orchestrated_dss + +A DSS instance that is equipped to fail on command, and will be used by the USS under test. + +TODO: Create this resource + +## Failed ISA test case + +### Verify no ISAs test step + +uss_qualifier checks the DSS to ensure that the Service Provider under test does not have any ISAs in the system. If ISAs are present, the Service Provider is instructed to clear the area of active flights, after which uss_qualifier reverifies the absence of ISAs. + +### Disable DSS test step + +uss_qualifier commands the orchestrated DSS to fail when interacting with normal clients. + +### Enumerate pre-existing operator notifications test step + +uss_qualifier retrieves the current (pre-existing) set of operator notifications. + +### Inject flight test step + +uss_qualifier attempts to inject a flight into the Service Provider under test, knowing that the Service Provider will not be able to create an ISA. + +#### Flight failed check + +Since the DSS is known to fail when attempting to create an ISA, if the Service Provider successfully creates the flight, they will have not met **[astm.f3411.v22a.NET0610](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement + +### Enumerate operator notifications test step + +uss_qualifier retrieves the current (after failed flight) set of operator notifications. + +#### Operator notified of discoverability failure check + +The "after" set of operator notifications should contain at least one more entry than the "before" set of operator notifications. If there was no new operator notification, the Service Provider will not have met **[astm.f3411.v22a.NET0620](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement + +## In-flight notifications test case + +### Inject flight test step + +uss_qualifier injects the flight into the Service Provider under test with the intention of observing the whole flight. + +### Enumerate pre-existing operator notifications test step + +uss_qualifier retrieves the current (pre-existing) set of operator notifications. + +### Poll Service Provider test step + +Throughout the duration of the flight, uss_qualifier periodically polls for operator notifications. + +#### Insufficient telemetry operator notification check + +If the Service Provider under test does not provide an operator notification in Phase 2 (regarding the telemetry feed stopping), it will not have complied with **[astm.f3411.v22a.NET0040](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement + +#### Missing data operator notification check + +If the Service Provider under test does not provide an operator notification in Phase 4 (regarding missing data fields in reported telemetry), it will not have complied with **[astm.f3411.v22a.NET0030](../../../../requirements/astm/f3411/v22a.md)**. + +TODO: Implement diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.py b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.py new file mode 100644 index 0000000000..c914c7c691 --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.py @@ -0,0 +1,8 @@ +from monitoring.uss_qualifier.scenarios.astm.netrid.common.operator_interactions import ( + OperatorInteractions as CommonOperatorInteractions, +) +from monitoring.uss_qualifier.scenarios.scenario import TestScenario + + +class OperatorInteractions(TestScenario, CommonOperatorInteractions): + pass diff --git a/monitoring/uss_qualifier/suites/README.md b/monitoring/uss_qualifier/suites/README.md index 984769f297..aa7554ac3c 100644 --- a/monitoring/uss_qualifier/suites/README.md +++ b/monitoring/uss_qualifier/suites/README.md @@ -1,9 +1,37 @@ # USS Qualifier test suites -A test suite is a set of tests that establish compliance to the thing they're named after. Example: Passing the "ASTM F3548-21" test suite should indicate the systems under test are compliant with ASTM F3548-21. +A test suite is a set of tests that establish compliance to the thing they're named after; it may be thought of as a "playlist" of test scenarios. Example: Passing the "ASTM F3548-21" test suite should indicate the systems under test are compliant with ASTM F3548-21. -A test suite is composed of a list of {test suite|test scenario}; each element on the list is executed sequentially. +A test suite is composed of a list of actions, each one being a [test scenario](../scenarios/README.md), [action generator](../action_generators/README.md), or another test suite. Each action on the list is executed sequentially. A test suite is defined with a YAML file following the [`TestSuiteDefinition` schema](definitions.py). +## Documentation + Test suite documentation is generated automatically; use `make format` from the repository root to regenerate it. + +### Actions + +This section of a test suite documentation page summarizes the actions that will be taken when the test suite is run. + +### Checked requirements + +This section of a test suite documentation page summarizes the requirements the suite may be capable of checking. A requirement is "checked" if there is [a declared test check](../scenarios/README.md#checks) with [documentation](../scenarios/README.md#test-checks) indicating that a failure of that check would imply a failure of the requirement. A particular test run may not perform all potential checks. For instance, if a test step is performed only in certain situations (e.g., a provided resource has certain characteristics) and skipped otherwise, then the checks associated with that test step will not be performed in that test run. All checks that are performed will be recorded in the report as passed or failed. + +The columns of the table in this section are as follows: + +#### Package + +All [requirements defined in this monitoring repository](../requirements/README.md) are defined exactly once in a particular "package" (InterUSS organizational structure to keep track of requirements) which represents the canonical source of that requirement. This column specifies and links to the package in which the requirement is defined in this monitoring repository. + +#### Requirement + +This column contains the name of the requirement in the [package](#package) namespace. Since periods are not allowed in requirement names, when InterUSS finds it helpful to refer to a particular portion of a named requirement, the name and portion(s) are usually delimited with commas. For instance, if a document specified REQ003 and that requirement had three distinct parts (a, b, and c), one requirement name might be `REQ003,a`. + +#### Status + +Test check documentation may include a `TODO:` note indicating that the check (and perhaps the containing test step) has not been fully implemented. If there is no such note for any of the checks relevant to this requirement, this column will indicate "Implemented". If all checks relevant to this requirement have this indication, this column will indicate "TODO". If some checks have this indication and others do not, this column will indicate "Implemented + TODO". A requirement is only listed in this table if the test suite may cause a check to happen that is associated with that requirement. + +#### Checked in + +This column includes a list of test scenarios containing checks associated with the corresponding requirement. Only test scenarios that may be run as part of the test suite are included. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md index 089ea93df4..f8db323ffc 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -2,26 +2,28 @@ # ASTM F3411-19 test suite [`suites.astm.netrid.f3411_19`](./f3411_19.yaml) -## Actions +## [Actions](../../README.md#actions) 1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) 1. Suite: [DSS instance probing for ASTM NetRID F3411-19](f3411_19/dss_probing.md) ([`suites.astm.netrid.f3411_19.dss_probing`](f3411_19/dss_probing.yaml)) -2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) -3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) -4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) -4. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) +2. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../scenarios/astm/netrid/v19/dss/crdb_access.md) ([`scenarios.astm.netrid.v19.dss.CRDBAccess`](../../../scenarios/astm/netrid/v19/dss/crdb_access.py)) +3. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) +4. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) +5. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v19/misbehavior.md) ([`scenarios.astm.netrid.v19.Misbehavior`](../../../scenarios/astm/netrid/v19/misbehavior.py)) +6. Scenario: [ASTM NetRID: Operator interactions](../../../scenarios/astm/netrid/v19/operator_interactions.md) ([`scenarios.astm.netrid.v19.OperatorInteractions`](../../../scenarios/astm/netrid/v19/operator_interactions.py)) +6. Scenario: [ASTM F3411-19 NetRID aggregate checks](../../../scenarios/astm/netrid/v19/aggregate_checks.md) ([`scenarios.astm.netrid.v19.AggregateChecks`](../../../scenarios/astm/netrid/v19/aggregate_checks.py)) -## Checked requirements +## [Checked requirements](../../README.md#checked-requirements) - - - - + + + + - + @@ -63,12 +65,12 @@ - + - + @@ -111,6 +113,16 @@ + + + + + + + + + + @@ -166,9 +178,19 @@ + + + + + + + + + + - + @@ -228,8 +250,13 @@ - - + + + + + + + @@ -249,7 +276,7 @@ - + diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml index 70e5e81486..b508bd8a3b 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml @@ -24,6 +24,10 @@ actions: dss_instances_source: dss_instances dss_instance_id: dss on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.dss.CRDBAccess + resources: {} + on_failure: Continue - test_scenario: scenario_type: scenarios.astm.netrid.v19.DSSInteroperability resources: @@ -47,6 +51,10 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v19.OperatorInteractions + resources: {} + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v19.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md index 2bf986cf77..a4a9cff7db 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19/dss_probing.md @@ -2,18 +2,18 @@ # DSS instance probing for ASTM NetRID F3411-19 test suite [`suites.astm.netrid.f3411_19.dss_probing`](./dss_probing.yaml) -## Actions +## [Actions](../../../README.md#actions) 1. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.md) ([`scenarios.astm.netrid.v19.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v19/dss/subscription_validation.py)) -## Checked requirements +## [Checked requirements](../../../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3411
.v19		astm
.f3411
.v19		 A2-6-1,1a Implemented ASTM F3411-19 NetRID DSS interoperability
A2-6-1,3cIn progressImplemented + TODO ASTM F3411-19 NetRID DSS interoperability
A2-6-1,3dIn progressImplemented + TODO ASTM F3411-19 NetRID DSS interoperability
Implemented ASTM F3411-19 NetRID DSS interoperability
DSS0110TODOASTM NetRID DSS: Direct CRDB access
DSS0120TODOASTM NetRID DSS: Direct CRDB access
DSS0130 ImplementedImplemented ASTM F3411-19 NetRID DSS interoperability
NET0030TODOASTM NetRID: Operator interactions
NET0040TODOASTM NetRID: Operator interactions
NET0240PlannedTODO ASTM F3411-19 NetRID aggregate checks
NET0610ImplementedASTM NetRID nominal behaviorImplemented + TODOASTM NetRID nominal behavior
ASTM NetRID: Operator interactions
NET0620TODOASTM NetRID: Operator interactions
NET0710
UpsertTestResultPlannedTODO ASTM NetRID nominal behavior
- - - - + + + + diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md index 6096cf8563..cd85fdb6dd 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -2,26 +2,28 @@ # ASTM F3411-22a test suite [`suites.astm.netrid.f3411_22a`](./f3411_22a.yaml) -## Actions +## [Actions](../../README.md#actions) 1. Action generator: [`action_generators.astm.f3411.ForEachDSS`](../../../action_generators/astm/f3411/for_each_dss.py) 1. Suite: [DSS instance probing for ASTM NetRID F3411-22a](f3411_22a/dss_probing.md) ([`suites.astm.netrid.f3411_22a.dss_probing`](f3411_22a/dss_probing.yaml)) -2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) -3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) -4. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v22a/misbehavior.md) ([`scenarios.astm.netrid.v22a.Misbehavior`](../../../scenarios/astm/netrid/v22a/misbehavior.py)) -4. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) +2. Scenario: [ASTM NetRID DSS: Direct CRDB access](../../../scenarios/astm/netrid/v22a/dss/crdb_access.md) ([`scenarios.astm.netrid.v22a.dss.CRDBAccess`](../../../scenarios/astm/netrid/v22a/dss/crdb_access.py)) +3. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) +4. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) +5. Scenario: [ASTM NetRID SP clients misbehavior handling](../../../scenarios/astm/netrid/v22a/misbehavior.md) ([`scenarios.astm.netrid.v22a.Misbehavior`](../../../scenarios/astm/netrid/v22a/misbehavior.py)) +6. Scenario: [ASTM NetRID: Operator interactions](../../../scenarios/astm/netrid/v22a/operator_interactions.md) ([`scenarios.astm.netrid.v22a.OperatorInteractions`](../../../scenarios/astm/netrid/v22a/operator_interactions.py)) +6. Scenario: [ASTM F3411-22a NetRID aggregate checks](../../../scenarios/astm/netrid/v22a/aggregate_checks.md) ([`scenarios.astm.netrid.v22a.AggregateChecks`](../../../scenarios/astm/netrid/v22a/aggregate_checks.py)) -## Checked requirements +## [Checked requirements](../../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3411
.v19
- - - - + + + + - + @@ -63,12 +65,12 @@ - + - + @@ -111,6 +113,16 @@ + + + + + + + + + + @@ -166,9 +178,19 @@ + + + + + + + + + + - + @@ -268,8 +290,13 @@ - - + + + + + + + @@ -294,7 +321,7 @@ - + diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml index 88d4f2011a..9039ee6ef6 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml @@ -26,6 +26,10 @@ actions: dss_instances_source: dss_instances dss_instance_id: dss on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v22a.dss.CRDBAccess + resources: {} + on_failure: Continue - test_scenario: scenario_type: scenarios.astm.netrid.v22a.DSSInteroperability resources: @@ -49,6 +53,10 @@ actions: evaluation_configuration: evaluation_configuration dss_pool: dss_instances on_failure: Continue + - test_scenario: + scenario_type: scenarios.astm.netrid.v22a.OperatorInteractions + resources: {} + on_failure: Continue report_evaluation_scenario: scenario_type: scenarios.astm.netrid.v22a.AggregateChecks resources: diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md index 8a60f30246..946c3bc491 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a/dss_probing.md @@ -2,19 +2,19 @@ # DSS instance probing for ASTM NetRID F3411-22a test suite [`suites.astm.netrid.f3411_22a.dss_probing`](./dss_probing.yaml) -## Actions +## [Actions](../../../README.md#actions) 1. Scenario: [ASTM NetRID DSS: Simple ISA](../../../../scenarios/astm/netrid/v22a/dss/isa_simple.md) ([`scenarios.astm.netrid.v22a.dss.ISASimple`](../../../../scenarios/astm/netrid/v22a/dss/isa_simple.py)) 2. Scenario: [ASTM NetRID DSS: Subscription Validation](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.md) ([`scenarios.astm.netrid.v22a.dss.SubscriptionValidation`](../../../../scenarios/astm/netrid/v22a/dss/subscription_validation.py)) -## Checked requirements +## [Checked requirements](../../../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3411
.v22a		astm
.f3411
.v22a		 A2-6-1,1a Implemented ASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cIn progressImplemented + TODO ASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dIn progressImplemented + TODO ASTM F3411-22a NetRID DSS interoperability
Implemented ASTM F3411-22a NetRID DSS interoperability
DSS0110TODOASTM NetRID DSS: Direct CRDB access
DSS0120TODOASTM NetRID DSS: Direct CRDB access
DSS0130 ImplementedImplemented ASTM F3411-22a NetRID DSS interoperability
NET0030TODOASTM NetRID: Operator interactions
NET0040TODOASTM NetRID: Operator interactions
NET0240PlannedTODO ASTM F3411-22a NetRID aggregate checks
NET0610ImplementedASTM NetRID nominal behaviorImplemented + TODOASTM NetRID nominal behavior
ASTM NetRID: Operator interactions
NET0620TODOASTM NetRID: Operator interactions
NET0710
UpsertTestResultPlannedTODO ASTM NetRID nominal behavior
- - - - + + + + diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index b6f70a1d08..c3f4c1c418 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -2,7 +2,7 @@ # ASTM F3548-21 test suite [`suites.astm.utm.f3548_21`](./f3548_21.yaml) -## Actions +## [Actions](../../README.md#actions) 1. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) @@ -11,14 +11,14 @@ 3. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) -## Checked requirements +## [Checked requirements](../../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3411
.v22a
- - - - + + + + diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py index 8ad4f3c6e6..72003d44b0 100644 --- a/monitoring/uss_qualifier/suites/documentation/documentation.py +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -27,7 +27,6 @@ FileReference, ) from monitoring.uss_qualifier.requirements.definitions import RequirementID -from monitoring.uss_qualifier.requirements.documentation import get_requirement from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName from monitoring.uss_qualifier.scenarios.documentation.definitions import ( TestScenarioDocumentation, @@ -38,12 +37,12 @@ get_documentation_by_name, ) from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name +from monitoring.uss_qualifier.suites import suite as suite_module from monitoring.uss_qualifier.suites.definitions import ( TestSuiteDefinition, ActionType, TestSuiteActionDeclaration, ) -from monitoring.uss_qualifier.suites.suite import TestSuiteAction @dataclass @@ -93,9 +92,13 @@ def make_test_suite_documentation( lines.append(f"{prefix}[`{get_package_name(suite_yaml_file)}`](./{local_path})") lines.append("") - lines.append("## Actions") - lines.append("") + suite_readme_abspath = os.path.join( + os.path.dirname(suite_module.__file__), "README.md" + ) base_path = os.path.dirname(suite_yaml_file) + suites_readme_path = os.path.relpath(suite_readme_abspath, start=base_path) + lines.append(f"## [Actions]({suites_readme_path}#actions)") + lines.append("") i = 0 for i, action in enumerate(suite_def.actions): lines.extend( @@ -130,7 +133,9 @@ def make_test_suite_documentation( ) lines.append("") - lines.append("## Checked requirements") + lines.append( + f"## [Checked requirements]({suites_readme_path}#checked-requirements)" + ) lines.append("") reqs = _collect_requirements_from_suite_def(suite_def) if not reqs: @@ -141,10 +146,14 @@ def make_test_suite_documentation( # Use an HTML table rather than Markdown table to enabled advanced features like spans lines.append("
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3548
.v21
") lines.append(" ") - lines.append(" ") - lines.append(" ") - lines.append(" ") - lines.append(" ") + lines.append(f' ') + lines.append( + f' ' + ) + lines.append(f' ') + lines.append( + f' ' + ) lines.append(" ") req_ids_by_package: Dict[str, List[RequirementID]] = {} @@ -176,9 +185,9 @@ def make_test_suite_documentation( if has_complete and not has_todo: status_text = "Implemented" elif has_todo and not has_complete: - status_text = "Planned" + status_text = "TODO" elif has_todo and has_complete: - status_text = "In progress" + status_text = "Implemented + TODO" else: status_text = "Not implemented" checked_in = list( diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md index 3cfd3ac839..7efaf4f0f9 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -2,20 +2,20 @@ # UFT message signing test suite [`suites.faa.uft.message_signing`](./message_signing.yaml) -## Actions +## [Actions](../../README.md#actions) 1. Scenario: [Start message signing](../../../scenarios/faa/uft/message_signing_start.md) ([`scenarios.faa.uft.StartMessageSigningReport`](../../../scenarios/faa/uft/message_signing_start.py)) 2. Suite: [ASTM F3548-21](../../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../../astm/utm/f3548_21.yaml)) 3. Scenario: [Finalize message signing](../../../scenarios/faa/uft/message_signing_finalize.md) ([`scenarios.faa.uft.FinalizeMessageSigningReport`](../../../scenarios/faa/uft/message_signing_finalize.py)) -## Checked requirements +## [Checked requirements](../../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
- - - - + + + + diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md index 22af2d494b..93884674fc 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.md +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -2,20 +2,20 @@ # U-space flight authorisation test suite [`suites.uspace.flight_auth`](./flight_auth.yaml) -## Actions +## [Actions](../README.md#actions) 1. Suite: [ASTM F3548-21](../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../astm/utm/f3548_21.yaml)) 2. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Flight authorisation validation](../../scenarios/uspace/flight_auth/validation.md) ([`scenarios.uspace.flight_auth.Validation`](../../scenarios/uspace/flight_auth/validation.py)) -## Checked requirements +## [Checked requirements](../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3548
.v21
- - - - + + + + diff --git a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md index 22d7140529..2c5098b044 100644 --- a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md +++ b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md @@ -2,10 +2,10 @@ # U-Space Common Information Service test suite [`suites.uspace.geo_awareness_cis`](./geo_awareness_cis.yaml) -## Actions +## [Actions](../README.md#actions) 1. Scenario: [EUROCAE ED-269 UAS geographical zone model](../../scenarios/eurocae/ed269/source_data_model.md) ([`scenarios.eurocae.ed269.source_data_model.SourceDataModelValidation`](../../scenarios/eurocae/ed269/source_data_model.py)) -## Checked requirements +## [Checked requirements](../README.md#checked-requirements) _This test suite documentation does not indicate that any requirements are checked._ diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md index ec7130edb8..f9bef3269c 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.md +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -2,21 +2,21 @@ # U-Space network identification test suite [`suites.uspace.network_identification`](./network_identification.yaml) -## Actions +## [Actions](../README.md#actions) 1. Suite: [ASTM F3411-22a](../astm/netrid/f3411_22a.md) ([`suites.astm.netrid.f3411_22a`](../astm/netrid/f3411_22a.yaml)) -## Checked requirements +## [Checked requirements](../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3548
.v21
- - - - + + + + - + @@ -58,12 +58,12 @@ - + - + @@ -106,6 +106,16 @@ + + + + + + + + + + @@ -161,9 +171,19 @@ + + + + + + + + + + - + @@ -263,8 +283,13 @@ - - + + + + + + + @@ -289,7 +314,7 @@ - + diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md index 1e24a9d65f..88434d430d 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.md +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -2,22 +2,22 @@ # U-space required services test suite [`suites.uspace.required_services`](./required_services.yaml) -## Actions +## [Actions](../README.md#actions) 1. Suite: [U-space flight authorisation](flight_auth.md) ([`suites.uspace.flight_auth`](flight_auth.yaml)) 2. Suite: [U-Space network identification](network_identification.md) ([`suites.uspace.network_identification`](network_identification.yaml)) -## Checked requirements +## [Checked requirements](../README.md#checked-requirements)
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3411
.v22a		astm
.f3411
.v22a		 A2-6-1,1a Implemented ASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cIn progressImplemented + TODO ASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dIn progressImplemented + TODO ASTM F3411-22a NetRID DSS interoperability
Implemented ASTM F3411-22a NetRID DSS interoperability
DSS0110TODOASTM NetRID DSS: Direct CRDB access
DSS0120TODOASTM NetRID DSS: Direct CRDB access
DSS0130 ImplementedImplemented ASTM F3411-22a NetRID DSS interoperability
NET0030TODOASTM NetRID: Operator interactions
NET0040TODOASTM NetRID: Operator interactions
NET0240PlannedTODO ASTM F3411-22a NetRID aggregate checks
NET0610ImplementedASTM NetRID nominal behaviorImplemented + TODOASTM NetRID nominal behavior
ASTM NetRID: Operator interactions
NET0620TODOASTM NetRID: Operator interactions
NET0710
UpsertTestResultPlannedTODO ASTM NetRID nominal behavior
- - - - + + + + - + @@ -59,12 +59,12 @@ - + - + @@ -107,6 +107,16 @@ + + + + + + + + + + @@ -162,9 +172,19 @@ + + + + + + + + + + - + @@ -264,8 +284,13 @@ - - + + + + + + + @@ -397,7 +422,7 @@ - +
PackageRequirementStatusChecked inPackageRequirementStatusChecked in
astm
.f3411
.v22a		astm
.f3411
.v22a		 A2-6-1,1a Implemented ASTM F3411-22a NetRID DSS interoperability
A2-6-1,3cIn progressImplemented + TODO ASTM F3411-22a NetRID DSS interoperability
A2-6-1,3dIn progressImplemented + TODO ASTM F3411-22a NetRID DSS interoperability
Implemented ASTM F3411-22a NetRID DSS interoperability
DSS0110TODOASTM NetRID DSS: Direct CRDB access
DSS0120TODOASTM NetRID DSS: Direct CRDB access
DSS0130 ImplementedImplemented ASTM F3411-22a NetRID DSS interoperability
NET0030TODOASTM NetRID: Operator interactions
NET0040TODOASTM NetRID: Operator interactions
NET0240PlannedTODO ASTM F3411-22a NetRID aggregate checks
NET0610ImplementedASTM NetRID nominal behaviorImplemented + TODOASTM NetRID nominal behavior
ASTM NetRID: Operator interactions
NET0620TODOASTM NetRID: Operator interactions
NET0710
UpsertTestResultPlannedTODO ASTM NetRID nominal behavior