diff --git a/github_pages/static/index.md b/github_pages/static/index.md index 25a167a753..036b0aa6c1 100644 --- a/github_pages/static/index.md +++ b/github_pages/static/index.md @@ -9,5 +9,5 @@ These reports were generated during continuous integration for the most recent P ### [U-space developer](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/uspace.yaml) [test configuration](https://github.com/interuss/monitoring/tree/main/monitoring/uss_qualifier/configurations) * [Sequence view](./artifacts/uss_qualifier/reports/sequence_uspace) -* [Tested requirements](./artifacts/uss_qualifier/tested_requirements_uspace) -* [Demonstrated capabilities](./artifacts/uss_qualifier/capabilities_uspace.html) +* [Tested requirements](./artifacts/uss_qualifier/reports/tested_requirements_uspace) +* [Demonstrated capabilities](./artifacts/uss_qualifier/reports/capabilities_uspace.html) diff --git a/monitoring/monitorlib/html/templates/explorer.html b/monitoring/monitorlib/html/templates/explorer.html index af2e2d2bfa..bb3d03e84b 100644 --- a/monitoring/monitorlib/html/templates/explorer.html +++ b/monitoring/monitorlib/html/templates/explorer.html @@ -58,7 +58,7 @@ cursor: pointer; } .collapsed>.node_key { - color: red; + color: blue; } .node_key { font-weight: bold; diff --git a/monitoring/uss_qualifier/reports/sequence_view.py b/monitoring/uss_qualifier/reports/sequence_view.py index 40c19c774d..6c02e9c2b7 100644 --- a/monitoring/uss_qualifier/reports/sequence_view.py +++ b/monitoring/uss_qualifier/reports/sequence_view.py @@ -3,7 +3,8 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import List, Dict, Optional, Iterator +import html +from typing import List, Dict, Optional, Iterator, Union from implicitdict import ImplicitDict @@ -40,6 +41,7 @@ class Event(ImplicitDict): event_index: int = 0 passed_check: Optional[PassedCheck] = None failed_check: Optional[FailedCheck] = None + query_events: Optional[List[Union[Event, str]]] = None query: Optional[Query] = None note: Optional[NoteEvent] = None @@ -69,6 +71,15 @@ def timestamp(self) -> datetime: else: raise ValueError("Invalid Event type") + def get_query_links(self) -> str: + links = [] + for e in self.query_events: + if isinstance(e, str): + links.append(e) + else: + links.append(f'{e.event_index}') + return ", ".join(links) + class TestedStep(ImplicitDict): name: str @@ -181,20 +192,24 @@ def _compute_tested_scenario( report: TestScenarioReport, indexer: Indexer ) -> TestedScenario: epochs = [] + all_events = [] event_index = 1 def append_notes(new_notes): - nonlocal event_index + nonlocal event_index, all_events events = [] for k, v in new_notes.items(): events.append( Event( note=NoteEvent( - key=k, message=v.message, timestamp=v.timestamp.datetime + key=html.escape(k), + message=html.escape(v.message), + timestamp=v.timestamp.datetime, ), event_index=event_index, ) ) + all_events.append(events[-1]) event_index += 1 events.sort(key=lambda e: e.timestamp) epochs.append(Epoch(events=events)) @@ -239,27 +254,45 @@ def append_notes(new_notes): events = [] for passed_check in step.passed_checks: events.append(Event(passed_check=passed_check)) + all_events.append(events[-1]) for pid in passed_check.participants: p = scenario_participants.get( pid, TestedParticipant(has_failures=False) ) scenario_participants[pid] = p - for failed_check in step.failed_checks: - events.append(Event(failed_check=failed_check)) - for pid in failed_check.participants: - p = scenario_participants.get( - pid, TestedParticipant(has_failures=True) - ) - p.has_failures = True - scenario_participants[pid] = p if "queries" in step and step.queries: for query in step.queries: events.append(Event(query=query)) + all_events.append(events[-1]) if "server_id" in query and query.server_id: p = scenario_participants.get( query.server_id, TestedParticipant(has_failures=False) ) scenario_participants[query.server_id] = p + for failed_check in step.failed_checks: + query_events = [] + for query_timestamp in failed_check.query_report_timestamps: + found = False + for e in all_events: + if ( + e.type == EventType.Query + and e.query.request.initiated_at == query_timestamp + ): + query_events.append(e) + found = True + break + if not found: + query_events.append(query_timestamp) + events.append( + Event(failed_check=failed_check, query_events=query_events) + ) + all_events.append(events[-1]) + for pid in failed_check.participants: + p = scenario_participants.get( + pid, TestedParticipant(has_failures=True) + ) + p.has_failures = True + scenario_participants[pid] = p if "notes" in report and report.notes: for key, note in report.notes.items(): if step.start_time.datetime <= note.timestamp.datetime: @@ -270,12 +303,13 @@ def append_notes(new_notes): events.append( Event( note=NoteEvent( - key=key, - message=note.message, + key=html.escape(key), + message=html.escape(note.message), timestamp=note.timestamp.datetime, ) ) ) + all_events.append(events[-1]) # Sort this step's events by time events.sort(key=lambda e: e.timestamp) diff --git a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html index c2fafc2f73..044715e59b 100644 --- a/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html +++ b/monitoring/uss_qualifier/reports/templates/sequence_view/scenario.html @@ -60,6 +60,9 @@ .not_tested { background-color: rgb(192, 192, 192); } + .failed_check_summary { + font-style: italic; + } {{ explorer_header() }} @@ -129,6 +132,17 @@

{{ test_scenario.type }}

{% else %} {{ event.failed_check.name }} {% endif %} + {% if event.query_events %} + [{{ event.get_query_links() }}] + {% endif %} + {% if event.failed_check.summary %} +
+ {{ event.failed_check.summary }} + {% endif %} + {% if event.failed_check.details %} +
+ {{ event.failed_check.details.replace("\n", "
") }} + {% endif %} {% for participant_id in all_participants %} {% if participant_id in event.failed_check.participants %} @@ -155,7 +169,7 @@

{{ test_scenario.type }}

{% elif event.type == EventType.Note %} 📓 - {{ event.note.key }}: {{ event.note.value }} + {{ event.note.key }}: {{ event.note.message }} {% else %} ???Render error: unknown EventType '{{ event.type }}' diff --git a/monitoring/uss_qualifier/reports/tested_requirements.py b/monitoring/uss_qualifier/reports/tested_requirements.py index 6749d2f4ae..85d84438bd 100644 --- a/monitoring/uss_qualifier/reports/tested_requirements.py +++ b/monitoring/uss_qualifier/reports/tested_requirements.py @@ -1,5 +1,7 @@ import os from dataclasses import dataclass +import re +from functools import cmp_to_key from typing import List, Union, Dict, Set, Optional from implicitdict import ImplicitDict, StringBasedDateTime @@ -265,10 +267,95 @@ def print_datetime(t: Optional[StringBasedDateTime]) -> Optional[str]: ) +def _split_strings_numbers(s: str) -> List[Union[int, str]]: + digits = "0123456789" + current_number = "" + current_string = "" + parts = [] + for si in s: + if si in digits: + if current_string: + parts.append(current_string) + current_string = "" + current_number += si + else: + if current_number: + parts.append(int(current_number)) + current_number = "" + current_string += si + if current_number: + parts.append(int(current_number)) + elif current_string: + parts.append(current_string) + return parts + + +def _requirement_id_parts(req_id: str) -> List[str]: + """Split a requirement ID into sortable parts. + + Each ID is split into parts in multiple phases (example: astm.f3411.v22a.NET0260,Table1,1b): + * Split at periods (splits into package and plain ID) + * Example: ["astm", "f3411", "v22a", "NET0260,Table1,1b"] + * Split at commas (splits portions of plain ID by convention) + * Example: ["astm", "f3411", "v22a", "NET0260", "Table1", "1b"] + * Split at transitions between words and numbers (so numbers are their own parts and non-numbers are their own parts) + * Example: ["astm", "f", 3411, "v", 22, "a", "NET", 260, "Table", 1, 1, "b"] + + Args: + req_id: Requirement ID to split. + + Returns: Constituent parts of req_id. + """ + old_parts = req_id.split(".") + parts = [] + for p in old_parts: + parts.extend(p.split(",")) + old_parts = parts + parts = [] + for p in old_parts: + parts.extend(_split_strings_numbers(p)) + return parts + + +def _compare_requirement_ids(r1: TestedRequirement, r2: TestedRequirement) -> int: + """Compare requirement IDs for the purpose of sorting. + + The requirement IDs are split into parts and then the parts compared. If all parts are equal but one ID has more + parts, the ID with fewer parts is first. See _requirement_id_parts for how requirement IDs are split. + + Returns: + * -1 if r1 should be before r2 + * 0 if r1 is equal to r2 + * 1 if r1 should be after r2 + """ + parts1 = _requirement_id_parts(r1.id) + parts2 = _requirement_id_parts(r2.id) + i = 0 + while i < min(len(parts1), len(parts2)): + p1 = parts1[i] + p2 = parts2[i] + if p1 == p2: + i += 1 + continue + if isinstance(p1, int): + if isinstance(p2, int): + return -1 if p1 < p2 else 1 + else: + return -1 + else: + if isinstance(p2, int): + return 1 + else: + return -1 if p1 < p2 else 1 + if i == len(parts1) and i == len(parts2): + return 0 + return -1 if len(parts1) < len(parts2) else 1 + + def _sort_breakdown(breakdown: TestedBreakdown) -> None: breakdown.packages.sort(key=lambda p: p.id) for package in breakdown.packages: - package.requirements.sort(key=lambda r: r.id) + package.requirements.sort(key=cmp_to_key(_compare_requirement_ids)) for requirement in package.requirements: requirement.scenarios.sort(key=lambda s: s.name)