From dc07bf19227c25bfa1fe63bbc345ae83155cd032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Wed, 18 Oct 2023 10:53:19 +0200 Subject: [PATCH] [uss_qualifier] Enable execution of multiple configurations in the same container --- monitoring/uss_qualifier/README.md | 2 + monitoring/uss_qualifier/main.py | 89 +++++++++++++++------ monitoring/uss_qualifier/run_locally.sh | 101 +++++++++++++----------- 3 files changed, 124 insertions(+), 68 deletions(-) diff --git a/monitoring/uss_qualifier/README.md b/monitoring/uss_qualifier/README.md index ccd611db01..d8dc6fb49d 100644 --- a/monitoring/uss_qualifier/README.md +++ b/monitoring/uss_qualifier/README.md @@ -10,6 +10,8 @@ The `uss_qualifier` tool is a synchronous executable built into the `interuss/mo The primary input accepted by uss_qualifier is the "configuration" specified with the `--config` option. This option should be a [reference to a configuration file](configurations/README.md) that the user has constructed or been provided to test the desired system for the desired characteristics. If testing a standard local system (DSS + dummy auth + mock USSs), the user can specify an alternate configuration reference as a single argument to `run_locally.sh` (the default configuration is `configurations.dev.local_test`). +Several comma-separated "configurations" may be passed via the `--config` option. If specified, the `--report` and `--config-output` options must have the same number of comma-separated values, which may be empty. + When building a custom configuration file, consider starting from [`configurations.dev.f3548_self_contained`](configurations/dev/f3548_self_contained.yaml), as it contains all information necessary to run the test without the usage of sometimes-configuring `$ref`s and `allOf`s. See [configurations documentation](configurations/README.md) for more information. ### Quick start diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py index 8d1ab536f7..99645fdbaf 100644 --- a/monitoring/uss_qualifier/main.py +++ b/monitoring/uss_qualifier/main.py @@ -45,20 +45,20 @@ def parseArgs() -> argparse.Namespace: parser.add_argument( "--config", - help="Configuration string according to monitoring/uss_qualifier/configurations/README.md", + help="Configuration string according to monitoring/uss_qualifier/configurations/README.md; Several comma-separated strings may be specified", required=True, ) parser.add_argument( "--report", default=None, - help="(Overrides setting in artifacts configuration) File name of the report to write (if test configuration provided) or read (if test configuration not provided)", + help="(Overrides setting in artifacts configuration) File name of the report to write (if test configuration provided) or read (if test configuration not provided); Several comma-separated file names matching the configurations may be specified", ) parser.add_argument( "--config-output", default=None, - help="If specified, write the configuration as parsed (potentially from multiple files) to the single file specified by this path", + help="If specified, write the configuration as parsed (potentially from multiple files) to the single file specified by this path; Several comma-separated file names matching the configurations may be specified", ) parser.add_argument( @@ -115,12 +115,16 @@ def execute_test_run( ) -def main() -> int: - args = parseArgs() - - config_src = load_dict_with_references(args.config) +def run_config( + config_name: str, + config_output: str, + report_path: str, + skip_validation: bool, + exit_before_execution: bool, +): + config_src = load_dict_with_references(config_name) - if not args.skip_validation: + if not skip_validation: logger.info("Validating configuration...") validation_errors = validate_config(config_src) if validation_errors: @@ -132,33 +136,33 @@ def main() -> int: whole_config = ImplicitDict.parse(config_src, USSQualifierConfiguration) - if args.config_output: - logger.info("Writing flattened configuration to {}", args.config_output) - if args.config_output.lower().endswith(".json"): - with open(args.config_output, "w") as f: + if config_output: + logger.info("Writing flattened configuration to {}", config_output) + if config_output.lower().endswith(".json"): + with open(config_output, "w") as f: json.dump(whole_config, f, indent=2, sort_keys=True) - elif args.config_output.lower().endswith(".yaml"): - with open(args.config_output, "w") as f: + elif config_output.lower().endswith(".yaml"): + with open(config_output, "w") as f: yaml.dump(json.loads(json.dumps(whole_config)), f, sort_keys=True) else: raise ValueError( "Unsupported extension for --config-output; only .json or .yaml file paths may be specified" ) - if args.exit_before_execution: + if exit_before_execution: logger.info("Exiting because --exit-before-execution specified.") - return os.EX_OK + return config: USSQualifierConfigurationV1 = whole_config.v1 - if args.report: + if report_path: if not config.artifacts: config.artifacts = ArtifactsConfiguration( - ReportConfiguration(report_path=args.report) + ReportConfiguration(report_path=report_path) ) elif not config.artifacts.report: - config.artifacts.report = ReportConfiguration(report_path=args.report) + config.artifacts.report = ReportConfiguration(report_path=report_path) else: - config.artifacts.report.report_path = args.report + config.artifacts.report.report_path = report_path do_not_save_report = False if config.test_run: @@ -215,15 +219,56 @@ def main() -> int: generate_sequence_view(report, config.artifacts.sequence_view) if "validation" in config and config.validation: - logger.info(f"Validating test run report for configuration '{args.config}'") + logger.info(f"Validating test run report for configuration '{config_name}'") if not validate_report(report, config.validation): logger.error( - f"Validation failed on test run report for configuration '{args.config}'" + f"Validation failed on test run report for configuration '{config_name}'" ) return -1 return os.EX_OK +def main() -> int: + args = parseArgs() + + config_names = str(args.config).split(",") + + if args.config_output: + config_outputs = str(args.config_output).split(",") + if len(config_outputs) != len(config_names): + raise ValueError( + f"Need matching number of config_output, expected {len(config_names)}, got {len(config_outputs)}" + ) + else: + config_outputs = ["" for _ in config_names] + + if args.report: + report_paths = str(args.report).split(",") + if len(report_paths) != len(config_names): + raise ValueError( + f"Need matching number of report, expected {len(config_names)}, got {len(report_paths)}" + ) + else: + report_paths = ["" for _ in config_names] + + for idx, config_name in enumerate(config_names): + logger.info( + f"========== Running uss_qualifier for configuration {config_name} ==========" + ) + run_config( + config_name, + config_outputs[idx], + report_paths[idx], + args.skip_validation, + args.exit_before_execution, + ) + logger.info( + f"========== Completed uss_qualifier for configuration {config_name} ==========" + ) + + return os.EX_OK + + if __name__ == "__main__": sys.exit(main()) diff --git a/monitoring/uss_qualifier/run_locally.sh b/monitoring/uss_qualifier/run_locally.sh index 571e6d8220..a03a8bec35 100755 --- a/monitoring/uss_qualifier/run_locally.sh +++ b/monitoring/uss_qualifier/run_locally.sh @@ -24,57 +24,66 @@ CONFIG_NAME="${1:-ALL}" OTHER_ARGS=${@:2} if [ "$CONFIG_NAME" == "ALL" ]; then - declare -a all_configurations=( \ - "configurations.dev.noop" \ - "configurations.dev.dss_probing" \ - "configurations.dev.geoawareness_cis" \ - "configurations.dev.generate_rid_test_data" \ - "configurations.dev.geospatial_comprehension" \ - "configurations.dev.general_flight_auth" \ - "configurations.dev.f3548" \ - "configurations.dev.f3548_self_contained" \ - "configurations.dev.netrid_v22a" \ - "configurations.dev.uspace" \ - ) - # TODO: Add configurations.dev.netrid_v19 - echo "Running configurations: ${all_configurations[*]}" - for configuration_name in "${all_configurations[@]}"; do - monitoring/uss_qualifier/run_locally.sh "$configuration_name" - done -else - CONFIG_FLAG="--config ${CONFIG_NAME}" + CONFIG_NAME="\ +configurations.dev.noop,\ +configurations.dev.dss_probing,\ +configurations.dev.geoawareness_cis,\ +configurations.dev.generate_rid_test_data,\ +configurations.dev.geospatial_comprehension,\ +configurations.dev.general_flight_auth,\ +configurations.dev.f3548,\ +configurations.dev.f3548_self_contained,\ +configurations.dev.netrid_v22a,\ +configurations.dev.uspace" +fi +# TODO: Add configurations.dev.netrid_v19 - AUTH_SPEC='DummyOAuth(http://oauth.authority.localutm:8085/token,uss_qualifier)' +echo "Running configuration(s): ${CONFIG_NAME}" - QUALIFIER_OPTIONS="$CONFIG_FLAG $OTHER_ARGS" +CONFIG_FLAG="--config ${CONFIG_NAME}" - OUTPUT_DIR="monitoring/uss_qualifier/output" - mkdir -p "$OUTPUT_DIR" +AUTH_SPEC='DummyOAuth(http://oauth.authority.localutm:8085/token,uss_qualifier)' - CACHE_DIR="monitoring/uss_qualifier/.templates_cache" - mkdir -p "$CACHE_DIR" +QUALIFIER_OPTIONS="$CONFIG_FLAG $OTHER_ARGS" - if [ "$CI" == "true" ]; then - docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions) - else - docker_args="-it" - fi +OUTPUT_DIR="monitoring/uss_qualifier/output" +mkdir -p "$OUTPUT_DIR" + +CACHE_DIR="monitoring/uss_qualifier/.templates_cache" +mkdir -p "$CACHE_DIR" - echo "========== Running uss_qualifier for configuration ${CONFIG_NAME} ==========" - # shellcheck disable=SC2086 - docker run ${docker_args} --name uss_qualifier \ - --rm \ - --network interop_ecosystem_network \ - -u "$(id -u):$(id -g)" \ - -e PYTHONBUFFERED=1 \ - -e AUTH_SPEC=${AUTH_SPEC} \ - -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \ - -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \ - -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \ - -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \ - -w /app/monitoring/uss_qualifier \ - interuss/monitoring \ - python main.py $QUALIFIER_OPTIONS - echo "========== Completed uss_qualifier for configuration ${CONFIG_NAME} ==========" +if [ "$CI" == "true" ]; then + docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions) +else + docker_args="-it" fi +start_time=$(date +%Y-%m-%dT%H:%M:%S) +# shellcheck disable=SC2086 +docker run ${docker_args} --name uss_qualifier \ + --rm \ + --network interop_ecosystem_network \ + -u "$(id -u):$(id -g)" \ + -e PYTHONBUFFERED=1 \ + -e AUTH_SPEC=${AUTH_SPEC} \ + -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \ + -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \ + -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \ + -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \ + -w /app/monitoring/uss_qualifier \ + interuss/monitoring \ + python main.py $QUALIFIER_OPTIONS + +# Set return code according to whether the test run was fully successful +reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time") +# shellcheck disable=SC2068 +for REPORT in ${reports_generated[@]}; do + successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT") + if echo "${successful}" | grep -iqF true; then + echo "Full success indicated by $REPORT" + else + echo "Could not establish that all uss_qualifier tests passed in $REPORT" + exit 1 + fi +done +