diff --git a/Makefile b/Makefile index 5d66437610..df3ac45dc7 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,9 @@ format: json-schema cd monitoring && make format .PHONY: lint -lint: python-lint shell-lint json-schema-lint +lint: + cd monitoring && make lint + cd schemas && make lint .PHONY: check-hygiene check-hygiene: python-lint hygiene validate-uss-qualifier-docs shell-lint diff --git a/monitoring/uss_qualifier/Makefile b/monitoring/uss_qualifier/Makefile index 3105b705b5..7a84d1d8af 100644 --- a/monitoring/uss_qualifier/Makefile +++ b/monitoring/uss_qualifier/Makefile @@ -4,6 +4,7 @@ lint: validate-docs python-lint shell-lint .PHONY: validate-docs validate-docs: ./scripts/validate_test_definitions.sh + ./scripts/format_test_suite_docs.sh --lint .PHONY: python-lint python-lint: @@ -20,6 +21,7 @@ format: format-documentation .PHONY: format-documentation format-documentation: ./scripts/format_test_documentation.sh + ./scripts/format_test_suite_docs.sh .PHONY: unit_test unit_test: diff --git a/monitoring/uss_qualifier/fileio.py b/monitoring/uss_qualifier/fileio.py index 74c254418f..134d59742d 100644 --- a/monitoring/uss_qualifier/fileio.py +++ b/monitoring/uss_qualifier/fileio.py @@ -1,4 +1,3 @@ -import hashlib import json import os from typing import Tuple, Optional, Dict, List, Union @@ -27,6 +26,8 @@ * .kml (content) """ +_package_root = os.path.dirname(__file__) + def resolve_filename(data_file: FileReference) -> str: if data_file.startswith(FILE_PREFIX): @@ -37,7 +38,7 @@ def resolve_filename(data_file: FileReference) -> str: return data_file else: # Package-based name (without extension) - path_parts = [os.path.dirname(__file__)] + path_parts = [_package_root] path_parts += data_file.split(".") file_name = None @@ -52,6 +53,23 @@ def resolve_filename(data_file: FileReference) -> str: ) +def get_package_name(local_file_path: str) -> FileReference: + """Get the Python-package style name of the specified file path on the local system. + + Args: + local_file_path: Path to YAML or JSON file on the local system. + + Returns: Python-package style name; e.g., suites.astm.netrid.f3411_19 + """ + base, ext = os.path.splitext(local_file_path) + if ext.lower() not in {".yaml", ".json"}: + raise ValueError( + f"Package name does not exist for non-dictionary file {local_file_path}" + ) + rel_path = os.path.relpath(base, start=_package_root) + return ".".join(os.path.normpath(rel_path).split(os.path.sep)) + + def _load_content_from_file_name(file_name: str) -> str: if file_name.startswith(HTTP_PREFIX) or file_name.startswith(HTTPS_PREFIX): # http(s):// web file reference diff --git a/monitoring/uss_qualifier/scenarios/documentation/definitions.py b/monitoring/uss_qualifier/scenarios/documentation/definitions.py index 457f7643f2..1cde8ac14d 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/definitions.py +++ b/monitoring/uss_qualifier/scenarios/documentation/definitions.py @@ -31,6 +31,7 @@ def get_step_by_name(self, step_name: str) -> Optional[TestStepDocumentation]: class TestScenarioDocumentation(ImplicitDict): name: str url: Optional[str] = None + local_path: str resources: Optional[List[str]] cases: List[TestCaseDocumentation] cleanup: Optional[TestStepDocumentation] diff --git a/monitoring/uss_qualifier/scenarios/documentation/parsing.py b/monitoring/uss_qualifier/scenarios/documentation/parsing.py index 13e34a8c52..cdec8572ba 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/parsing.py +++ b/monitoring/uss_qualifier/scenarios/documentation/parsing.py @@ -296,6 +296,7 @@ def _parse_documentation(scenario: Type) -> TestScenarioDocumentation: "cases": test_cases, "resources": resources, "url": url, + "local_path": os.path.abspath(doc_filename), } if cleanup is not None: kwargs["cleanup"] = cleanup diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 174216a469..b3901e938e 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import Enum import inspect -from typing import Callable, Dict, List, Optional, TypeVar, Union, Set +from typing import Callable, Dict, List, Optional, TypeVar, Union, Set, Type import arrow from implicitdict import StringBasedDateTime @@ -159,6 +159,20 @@ def record_passed( self._step_report.passed_checks.append(passed_check) +def get_scenario_type_by_name(scenario_type_name: str) -> Type: + inspection.import_submodules(scenarios_module) + scenario_type = inspection.get_module_object_by_name( + parent_module=uss_qualifier_module, object_name=scenario_type_name + ) + if not issubclass(scenario_type, TestScenario): + raise NotImplementedError( + "Scenario type {} is not a subclass of the TestScenario base class".format( + scenario_type.__name__ + ) + ) + return scenario_type + + class GenericTestScenario(ABC): """Generic Test Scenario allowing mutualization of test scenario implementation. @@ -185,16 +199,7 @@ def make_test_scenario( declaration: TestScenarioDeclaration, resource_pool: Dict[ResourceID, ResourceTypeName], ) -> "TestScenario": - inspection.import_submodules(scenarios_module) - scenario_type = inspection.get_module_object_by_name( - parent_module=uss_qualifier_module, object_name=declaration.scenario_type - ) - if not issubclass(scenario_type, TestScenario): - raise NotImplementedError( - "Scenario type {} is not a subclass of the TestScenario base class".format( - scenario_type.__name__ - ) - ) + scenario_type = get_scenario_type_by_name(declaration.scenario_type) constructor_signature = inspect.signature(scenario_type.__init__) constructor_args = {} diff --git a/monitoring/uss_qualifier/scripts/format_test_suite_docs.sh b/monitoring/uss_qualifier/scripts/format_test_suite_docs.sh new file mode 100755 index 0000000000..7a397ca516 --- /dev/null +++ b/monitoring/uss_qualifier/scripts/format_test_suite_docs.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -eo pipefail +set -o xtrace + +# Find and change to repo root directory +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "${BASEDIR}/../../.." || exit 1 + +monitoring/build.sh || exit 1 + +# shellcheck disable=SC2086 +docker run --name test_suite_docs_formatter \ + --rm \ + -v "$(pwd):/app" \ + interuss/monitoring \ + uss_qualifier/scripts/in_container/format_test_suite_docs.sh "$@" diff --git a/monitoring/uss_qualifier/scripts/in_container/format_test_suite_docs.sh b/monitoring/uss_qualifier/scripts/in_container/format_test_suite_docs.sh new file mode 100755 index 0000000000..1061b6474d --- /dev/null +++ b/monitoring/uss_qualifier/scripts/in_container/format_test_suite_docs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# This script is intended to be called from within a Docker container running +# mock_uss via the interuss/monitoring image. In that context, this script is +# the entrypoint into the test definition validation tool. + +# Ensure uss_qualifier is the working directory +OS=$(uname) +if [[ $OS == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "${BASEDIR}/../.." || exit 1 + +# Run validation +python suites/documentation/format_documentation.py "$@" diff --git a/monitoring/uss_qualifier/suites/README.md b/monitoring/uss_qualifier/suites/README.md index 097d8a91e7..984769f297 100644 --- a/monitoring/uss_qualifier/suites/README.md +++ b/monitoring/uss_qualifier/suites/README.md @@ -5,3 +5,5 @@ A test suite is a set of tests that establish compliance to the thing they're na A test suite is composed of a list of {test suite|test scenario}; each element on the list is executed sequentially. A test suite is defined with a YAML file following the [`TestSuiteDefinition` schema](definitions.py). + +Test suite documentation is generated automatically; use `make format` from the repository root to regenerate it. diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md new file mode 100644 index 0000000000..b91c6ea824 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.md @@ -0,0 +1,9 @@ + +# ASTM F3411-19 test suite +[`suites.astm.netrid.f3411_19`](./f3411_19.yaml) + +## Actions + +1. Action generator: `action_generators.astm.f3411.ForEachDSS` +2. Scenario: [ASTM F3411-19 NetRID DSS interoperability](../../../scenarios/astm/netrid/v19/dss_interoperability.md) ([`scenarios.astm.netrid.v19.DSSInteroperability`](../../../scenarios/astm/netrid/v19/dss_interoperability.py)) +3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v19/nominal_behavior.md) ([`scenarios.astm.netrid.v19.NominalBehavior`](../../../scenarios/astm/netrid/v19/nominal_behavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md new file mode 100644 index 0000000000..8dd20db5b5 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.md @@ -0,0 +1,9 @@ + +# ASTM F3411-22a test suite +[`suites.astm.netrid.f3411_22a`](./f3411_22a.yaml) + +## Actions + +1. Action generator: `action_generators.astm.f3411.ForEachDSS` +2. Scenario: [ASTM F3411-22a NetRID DSS interoperability](../../../scenarios/astm/netrid/v22a/dss_interoperability.md) ([`scenarios.astm.netrid.v22a.DSSInteroperability`](../../../scenarios/astm/netrid/v22a/dss_interoperability.py)) +3. Scenario: [ASTM NetRID nominal behavior](../../../scenarios/astm/netrid/v22a/nominal_behavior.md) ([`scenarios.astm.netrid.v22a.NominalBehavior`](../../../scenarios/astm/netrid/v22a/nominal_behavior.py)) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md new file mode 100644 index 0000000000..62a6f5c093 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -0,0 +1,9 @@ + +# ASTM F3548-21 test suite +[`suites.astm.utm.f3548_21`](./f3548_21.yaml) + +## Actions + +1. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` +3. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` diff --git a/monitoring/uss_qualifier/suites/documentation/__init__.py b/monitoring/uss_qualifier/suites/documentation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/uss_qualifier/suites/documentation/documentation.py b/monitoring/uss_qualifier/suites/documentation/documentation.py new file mode 100644 index 0000000000..ca453656d1 --- /dev/null +++ b/monitoring/uss_qualifier/suites/documentation/documentation.py @@ -0,0 +1,90 @@ +import glob +import inspect +import os +from typing import Iterator, Optional + +from implicitdict import ImplicitDict + +from monitoring.uss_qualifier.fileio import ( + load_dict_with_references, + get_package_name, + resolve_filename, +) +from monitoring.uss_qualifier.scenarios.documentation.parsing import get_documentation +from monitoring.uss_qualifier.scenarios.scenario import get_scenario_type_by_name +from monitoring.uss_qualifier.suites.definitions import TestSuiteDefinition, ActionType + + +def find_test_suites(start_path: Optional[str] = None) -> Iterator[str]: + if start_path is None: + start_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + for yaml in glob.glob(os.path.join(start_path, "*.yaml")): + yield yaml + for subfolder in os.listdir(start_path): + full_path = os.path.join(start_path, subfolder) + if not os.path.isdir(full_path): + continue + for suite in find_test_suites(full_path): + yield suite + + +def make_test_suite_documentation(test_suite_yaml_file: str) -> str: + lines = [] + suite_def: TestSuiteDefinition = ImplicitDict.parse( + load_dict_with_references("file://" + test_suite_yaml_file), TestSuiteDefinition + ) + lines.append( + "" + ) + lines.append(f"# {suite_def.name} test suite") + local_path = os.path.split(test_suite_yaml_file)[-1] + lines.append(f"[`{get_package_name(test_suite_yaml_file)}`](./{local_path})") + lines.append("") + + lines.append("## Actions") + lines.append("") + base_path = os.path.dirname(test_suite_yaml_file) + for i, action in enumerate(suite_def.actions): + action_type = action.get_action_type() + if action_type == ActionType.TestScenario: + scenario_type = get_scenario_type_by_name( + action.test_scenario.scenario_type + ) + py_rel_path = os.path.relpath(inspect.getfile(scenario_type), base_path) + scenario_doc = get_documentation(scenario_type) + doc_rel_path = os.path.relpath(scenario_doc.local_path, start=base_path) + lines.append( + f"{i + 1}. Scenario: [{scenario_doc.name}]({doc_rel_path}) ([`{action.test_scenario.scenario_type}`]({py_rel_path}))" + ) + elif action_type == ActionType.TestSuite: + if "suite_type" in action.test_suite and action.test_suite.suite_type: + suite_def = ImplicitDict.parse( + load_dict_with_references(action.test_suite.suite_type), + TestSuiteDefinition, + ) + suite_path = resolve_filename(action.test_suite.suite_type) + suite_rel_path = os.path.relpath(suite_path, start=base_path) + doc_path = os.path.splitext(suite_path)[0] + ".md" + doc_rel_path = os.path.relpath(doc_path, start=base_path) + lines.append( + f"{i + 1}. Suite: [{suite_def.name}]({doc_rel_path}) ([`{action.test_suite.suite_type}`]({suite_rel_path}))" + ) + elif "suite_definition" in action.test_suite and action.suite_definition: + # TODO: Generate additional test suite documentation for in-suite suite definition + lines.append(f"{i + 1}. Suite: ") + else: + raise ValueError( + f"Test suite action {i + 1} missing suite type or definition in {test_suite_yaml_file}" + ) + elif action_type == ActionType.ActionGenerator: + # TODO: Add documentation for action generators + lines.append( + f"{i + 1}. Action generator: `{action.action_generator.generator_type}`" + ) + else: + raise NotImplementedError( + f"Unsupported test suite action type: {action_type}" + ) + + lines.append("") + return "\n".join(lines) diff --git a/monitoring/uss_qualifier/suites/documentation/format_documentation.py b/monitoring/uss_qualifier/suites/documentation/format_documentation.py new file mode 100644 index 0000000000..10b3ea2a5a --- /dev/null +++ b/monitoring/uss_qualifier/suites/documentation/format_documentation.py @@ -0,0 +1,53 @@ +import argparse +import os +import sys + +from monitoring.uss_qualifier.suites.documentation.documentation import ( + find_test_suites, + make_test_suite_documentation, +) + + +def main(lint: bool) -> int: + changes = False + for suite_yaml_file in find_test_suites(): + suite_doc_content = make_test_suite_documentation(suite_yaml_file) + suite_doc_file = os.path.splitext(suite_yaml_file)[0] + ".md" + if os.path.exists(suite_doc_file): + with open(suite_doc_file, "r") as f: + existing_content = f.read() + if existing_content == suite_doc_content: + # No changes needed + continue + changes = True + if lint: + print(f"Test suite documentation must be regenerated: {suite_doc_file}") + else: + with open(suite_doc_file, "w") as f: + f.write(suite_doc_content) + print(f"Wrote test suite documentation: {suite_doc_file}") + + if lint and changes: + return -1 + if not changes: + print("No test suite documentation changes needed.") + return os.EX_OK + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Automatically manage test suite documentation" + ) + + parser.add_argument( + "--lint", + action="store_true", + help="When specified, do not make any documentation changes and exit with error if changes are needed.", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + sys.exit(main(args.lint)) diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.md b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md new file mode 100644 index 0000000000..a5fe6c401d --- /dev/null +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.md @@ -0,0 +1,9 @@ + +# UFT message signing test suite +[`suites.faa.uft.message_signing`](./message_signing.yaml) + +## Actions + +1. Scenario: [Start message signing](../../../scenarios/faa/uft/message_signing_start.md) ([`scenarios.faa.uft.StartMessageSigningReport`](../../../scenarios/faa/uft/message_signing_start.py)) +2. Suite: [ASTM F3548-21](../../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../../astm/utm/f3548_21.yaml)) +3. Scenario: [Finalize message signing](../../../scenarios/faa/uft/message_signing_finalize.md) ([`scenarios.faa.uft.FinalizeMessageSigningReport`](../../../scenarios/faa/uft/message_signing_finalize.py)) diff --git a/monitoring/uss_qualifier/suites/interuss/generate_test_data.md b/monitoring/uss_qualifier/suites/interuss/generate_test_data.md new file mode 100644 index 0000000000..662ddf6765 --- /dev/null +++ b/monitoring/uss_qualifier/suites/interuss/generate_test_data.md @@ -0,0 +1,8 @@ + +# Generate RID test data test suite +[`suites.interuss.generate_test_data`](./generate_test_data.yaml) + +## Actions + +1. Scenario: [Store NetRID flight data](../../scenarios/astm/netrid/store_flight_data.md) ([`scenarios.astm.netrid.StoreFlightData`](../../scenarios/astm/netrid/store_flight_data.py)) +2. Scenario: [Store NetRID flight data](../../scenarios/astm/netrid/store_flight_data.md) ([`scenarios.astm.netrid.StoreFlightData`](../../scenarios/astm/netrid/store_flight_data.py)) diff --git a/monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md b/monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md new file mode 100644 index 0000000000..540abc5a0e --- /dev/null +++ b/monitoring/uss_qualifier/suites/interuss/generate_test_data_twice.md @@ -0,0 +1,7 @@ + +# Generate RID test data twice test suite +[`suites.interuss.generate_test_data_twice`](./generate_test_data_twice.yaml) + +## Actions + +1. Action generator: `action_generators.Repeat` diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.md b/monitoring/uss_qualifier/suites/uspace/flight_auth.md new file mode 100644 index 0000000000..25bdb21bce --- /dev/null +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.md @@ -0,0 +1,8 @@ + +# U-space flight authorisation test suite +[`suites.uspace.flight_auth`](./flight_auth.yaml) + +## Actions + +1. Suite: [ASTM F3548-21](../astm/utm/f3548_21.md) ([`suites.astm.utm.f3548_21`](../astm/utm/f3548_21.yaml)) +2. Action generator: `action_generators.flight_planning.FlightPlannerCombinations` diff --git a/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md new file mode 100644 index 0000000000..910b3700fb --- /dev/null +++ b/monitoring/uss_qualifier/suites/uspace/geo_awareness_cis.md @@ -0,0 +1,7 @@ + +# U-Space Common Information Service test suite +[`suites.uspace.geo_awareness_cis`](./geo_awareness_cis.yaml) + +## Actions + +1. Scenario: [EUROCAE ED-269 UAS geographical zone model](../../scenarios/eurocae/ed269/source_data_model.md) ([`scenarios.eurocae.ed269.source_data_model.SourceDataModelValidation`](../../scenarios/eurocae/ed269/source_data_model.py)) diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.md b/monitoring/uss_qualifier/suites/uspace/network_identification.md new file mode 100644 index 0000000000..1cfec80a32 --- /dev/null +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.md @@ -0,0 +1,7 @@ + +# U-Space network identification test suite +[`suites.uspace.network_identification`](./network_identification.yaml) + +## Actions + +1. Suite: [ASTM F3411-22a](../astm/netrid/f3411_22a.md) ([`suites.astm.netrid.f3411_22a`](../astm/netrid/f3411_22a.yaml)) diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.md b/monitoring/uss_qualifier/suites/uspace/required_services.md new file mode 100644 index 0000000000..6598ea51be --- /dev/null +++ b/monitoring/uss_qualifier/suites/uspace/required_services.md @@ -0,0 +1,8 @@ + +# U-space required services test suite +[`suites.uspace.required_services`](./required_services.yaml) + +## Actions + +1. Suite: [U-space flight authorisation](flight_auth.md) ([`suites.uspace.flight_auth`](flight_auth.yaml)) +2. Suite: [U-Space network identification](network_identification.md) ([`suites.uspace.network_identification`](network_identification.yaml))