diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60f4554497..ec66e9fdb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,18 +60,6 @@ jobs: cd monitoring/uss_qualifier make test - uss_qualifier_F3411-19-test: - name: uss_qualifier F3411-19 tests - uses: ./.github/workflows/monitoring-test.yml - with: - name: uss_qualifier_F3411-19 - script: | - export CONFIG_NAME=configurations.dev.netrid_v19 \ - USS_QUALIFIER_STOP_FAST=true - - cd monitoring/uss_qualifier - make test - prober-test: name: prober tests uses: ./.github/workflows/monitoring-test.yml @@ -83,7 +71,7 @@ jobs: publish-gh-pages: name: Publish GitHub Pages - needs: [hygiene-tests, monitorlib-test, mock_uss-test, uss_qualifier-test, uss_qualifier_F3411-19-test, prober-test] + needs: [hygiene-tests, monitorlib-test, mock_uss-test, uss_qualifier-test, prober-test] if: ${{ always() && contains(join(needs.*.result, ','), 'success') }} runs-on: ubuntu-latest permissions: diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index d6ea2a32e6..2914bf1bf4 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -92,6 +92,15 @@ net_rid_sims: kml_source: kml_file: path: file://./test_data/usa/kentland/rid.kml + flight_start_delay: 5s + foca_flights_data: + $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json + resource_type: resources.netrid.FlightDataResource + specification: + kml_source: + kml_file: + path: file://./test_data/che/rid/foca.kml + flight_start_delay: 5s flight_auth: $ref: '#/f3548_che' diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml index 9bc131053b..58e5e85459 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v19.yaml @@ -21,5 +21,22 @@ v1: report_path: output/report_netrid_v19.json tested_roles: report_path: output/tested_roles_netrid_v19 + tested_requirements: + output_path: output/tested_requirements_f3411v19 + requirement_collections: + sp_dp_dss: + requirement_sets: + - astm.f3411.v19.service_provider#Tested by automated tests + - astm.f3411.v19.display_provider#Automated verification + - astm.f3411.v19.dss_provider + sp_dss: + requirement_sets: + - astm.f3411.v19.service_provider#Tested by automated tests + - astm.f3411.v19.dss_provider + participant_requirements: + uss1: sp_dp_dss + uss2: sp_dss + sequence_view: + output_path: output/sequence_netrid_v19 validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml index ed7c9ec0b1..0405c07c31 100644 --- a/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml +++ b/monitoring/uss_qualifier/configurations/dev/netrid_v22a.yaml @@ -33,9 +33,10 @@ v1: requirement_sets: - astm.f3411.v22a.service_provider#Mandatory requirements - astm.f3411.v22a.dss_provider - - astm.f3548.v21.scd#Automated verification participant_requirements: uss1: sp_dp_dss uss2: sp_dss + sequence_view: + output_path: output/sequence_netrid_v22a validation: $ref: ./library/validation.yaml#/normal_test diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index 8c723367ff..3406b8b704 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -23,7 +23,7 @@ v1: scd_dss: scd_dss scd_dss_instances: scd_dss_instances - flights_data: adjacent_circular_flights_data + flights_data: foca_flights_data service_providers: netrid_service_providers_v22a observers: netrid_observers_v22a evaluation_configuration: netrid_observation_evaluation_configuration diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index c4dfaa2abc..e56ae9dd93 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -1,4 +1,6 @@ from __future__ import annotations + +import math import os from dataclasses import dataclass from datetime import datetime @@ -15,9 +17,8 @@ from monitoring.uss_qualifier.configurations.configuration import ( ParticipantID, SequenceViewConfiguration, - TestConfiguration, ) -from monitoring.uss_qualifier.fileio import load_content, load_dict_with_references +from monitoring.uss_qualifier.fileio import load_dict_with_references from monitoring.uss_qualifier.reports import jinja_env from monitoring.uss_qualifier.reports.report import ( TestRunReport, @@ -37,6 +38,9 @@ from monitoring.uss_qualifier.suites.definitions import ActionType, TestSuiteDefinition +UNATTRIBUTED_PARTICIPANT = "unattributed" + + class NoteEvent(ImplicitDict): key: str message: str @@ -144,7 +148,9 @@ def rows(self) -> int: @dataclass class TestedParticipant(object): - has_failures: bool + has_failures: bool = False + has_successes: bool = False + has_queries: bool = False class TestedScenario(ImplicitDict): @@ -152,6 +158,7 @@ class TestedScenario(ImplicitDict): name: str url: str scenario_index: int + duration: str epochs: List[Epoch] participants: Dict[ParticipantID, TestedParticipant] @@ -276,20 +283,28 @@ def append_notes(new_notes): for passed_check in step.passed_checks: events.append(Event(passed_check=passed_check)) all_events.append(events[-1]) - for pid in passed_check.participants: - p = scenario_participants.get( - pid, TestedParticipant(has_failures=False) - ) + participants = ( + passed_check.participants + if passed_check.participants + else [UNATTRIBUTED_PARTICIPANT] + ) + for pid in participants: + p = scenario_participants.get(pid, TestedParticipant()) + p.has_successes = True scenario_participants[pid] = p if "queries" in step and step.queries: for query in step.queries: events.append(Event(query=query)) all_events.append(events[-1]) - if "server_id" in query and query.server_id: - p = scenario_participants.get( - query.server_id, TestedParticipant(has_failures=False) - ) - scenario_participants[query.server_id] = p + participant_id = ( + query.server_id + if "server_id" in query and query.server_id + else UNATTRIBUTED_PARTICIPANT + ) + p = scenario_participants.get(participant_id, TestedParticipant()) + p.has_queries = True + scenario_participants[participant_id] = p + for failed_check in step.failed_checks: query_events = [] if ( @@ -312,10 +327,13 @@ def append_notes(new_notes): Event(failed_check=failed_check, query_events=query_events) ) all_events.append(events[-1]) - for pid in failed_check.participants: - p = scenario_participants.get( - pid, TestedParticipant(has_failures=True) - ) + participants = ( + failed_check.participants + if failed_check.participants + else [UNATTRIBUTED_PARTICIPANT] + ) + for pid in participants: + p = scenario_participants.get(pid, TestedParticipant()) p.has_failures = True scenario_participants[pid] = p if "notes" in report and report.notes: @@ -380,12 +398,23 @@ def append_notes(new_notes): else: post_notes = {} if post_notes: + latest_step_time = max(v.timestamp.datetime for v in post_notes.values()) append_notes(post_notes) + if "end_time" in report and report.end_time: + latest_step_time = report.end_time.datetime + + dt_s = round((latest_step_time - report.start_time.datetime).total_seconds()) + dt_m = math.floor(dt_s / 60) + dt_s -= dt_m * 60 + padding = "0" if dt_s < 10 else "" + duration = f"{dt_m}:{padding}{dt_s}" + scenario = TestedScenario( type=report.scenario_type, name=report.name, url=report.documentation_url, + duration=duration, epochs=epochs, scenario_index=indexer.scenario_index, participants=scenario_participants, @@ -573,6 +602,9 @@ def _generate_scenario_pages( if node.node_type == ActionNodeType.Scenario: all_participants = list(node.scenario.participants) all_participants.sort() + if UNATTRIBUTED_PARTICIPANT in all_participants: + all_participants.remove(UNATTRIBUTED_PARTICIPANT) + all_participants.append(UNATTRIBUTED_PARTICIPANT) scenario_file = os.path.join( config.output_path, f"s{node.scenario.scenario_index}.html" ) @@ -584,6 +616,7 @@ def _generate_scenario_pages( all_participants=all_participants, EpochType=EpochType, EventType=EventType, + UNATTRIBUTED_PARTICIPANT=UNATTRIBUTED_PARTICIPANT, len=len, str=str, ) @@ -606,6 +639,9 @@ def generate_sequence_view( max_suite_cols = max(len(r.suite_cells) for r in overview_rows) all_participants = _enumerate_all_participants(node) all_participants.sort() + if UNATTRIBUTED_PARTICIPANT in all_participants: + all_participants.remove(UNATTRIBUTED_PARTICIPANT) + all_participants.append(UNATTRIBUTED_PARTICIPANT) overview_file = os.path.join(config.output_path, "index.html") template = jinja_env.get_template("sequence_view/overview.html") with open(overview_file, "w") as f: @@ -617,6 +653,7 @@ def generate_sequence_view( max_suite_cols=max_suite_cols, all_participants=all_participants, ActionNodeType=ActionNodeType, + UNATTRIBUTED_PARTICIPANT=UNATTRIBUTED_PARTICIPANT, len=len, ) ) diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html index 8f4b5010b3..4595384515 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/overview.html @@ -124,7 +124,7 @@

{{ "Baseline" if is_baseline else "Environment" }}

{% set current = namespace(address="v1.test_run.resources.resource_declarations") %} {% for resource_id, resource_dec in report.configuration.resources.resource_declarations.items() %} {% set current.address = current.address + "." + resource_id %} - {% if (current.address in report.configuration.non_baseline_inputs) != is_baseline %} + {% if ("non_baseline_inputs" in report.configuration and report.configuration.non_baseline_inputs and (current.address in report.configuration.non_baseline_inputs) != is_baseline) or (("non_baseline_inputs" not in report.configuration or not report.configuration.non_baseline_inputs) and is_baseline) %}
  • {{ resource_id }} ({{ resource_dec.resource_type }})