[uss_qualifier] Improve test run report validation (#298)

* Improve test run report validation * Fix Severity comparison logic 89aaa4b
interuss · Oct 31, 2023 · 9748316 · 9748316
1 parent dd3fb67
commit 9748316
Show file tree

Hide file tree

Showing 160 changed files with 37,767 additions and 37,723 deletions.
diff --git a/artifacts/uss_qualifier/reports/dss_probing/report.json b/artifacts/uss_qualifier/reports/dss_probing/report.json
diff --git a/artifacts/uss_qualifier/reports/dss_probing/requirements/status.json b/artifacts/uss_qualifier/reports/dss_probing/requirements/status.json
@@ -1,9 +1,9 @@
 {
   "test_run_information": {
-    "test_run_id": "89002a6304cf4d6aac99eb506ea6b57cb8741d4a9c83d8ae34c9d56d6d37cad6",
-    "start_time": "2023-10-31 18:25:50 UTC",
-    "end_time": "2023-10-31 18:26:59 UTC",
-    "baseline": "b3910e5a1c8804fc386ceefae76c384603959af8c1a77958b82a9fa33dcc2595",
+    "test_run_id": "a8e46b00104ecd448d9b3ccc9d1230d2225d6ae7dc58e8bbb2b9e3f2beacb12b",
+    "start_time": "2023-10-31 18:26:23 UTC",
+    "end_time": "2023-10-31 18:27:31 UTC",
+    "baseline": "1fac33ad177319f525d4309b880fd8fb609ba5d71c15561f9f6ae04733578839",
     "environment": "383670842b90e623e04660bab8132198849a61457e0d90caf9eebb0df6322efb"
   },
   "participant_verifications": {

diff --git a/artifacts/uss_qualifier/reports/dss_probing/requirements/uss1.html b/artifacts/uss_qualifier/reports/dss_probing/requirements/uss1.html
diff --git a/artifacts/uss_qualifier/reports/dss_probing/requirements/uss2.html b/artifacts/uss_qualifier/reports/dss_probing/requirements/uss2.html
diff --git a/artifacts/uss_qualifier/reports/dss_probing/sequence/index.html b/artifacts/uss_qualifier/reports/dss_probing/sequence/index.html
@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-  <title>TR-89002a6 sequence view</title>
+  <title>TR-a8e46b0 sequence view</title>
   <style>
     body {
       margin: 0;
@@ -107,31 +107,31 @@ <h1>Sequence view of uss_qualifier test run</h1>
       </tr>
       <tr>
         <td>Test run identifier</td>
-        <td>TR-89002a6</td>
+        <td>TR-a8e46b0</td>
       </tr>
       <tr>
         <td>Start time</td>
-        <td>2023-10-31 18:25:50 UTC</td>
+        <td>2023-10-31 18:26:23 UTC</td>
       </tr>
       <tr>
         <td>End time</td>
-        <td>2023-10-31 18:26:59 UTC</td>
+        <td>2023-10-31 18:27:31 UTC</td>
       </tr>
       <tr>
         <td>Test baseline identifier</td>
-        <td>TB-b3910e5</td>
+        <td>TB-1fac33a</td>
       </tr>
       <tr>
         <td>Environment identifier</td>
         <td>TE-3836708</td>
       </tr>
       <tr>
         <td>Codebase version</td>
-        <td>interuss/monitoring/v0.0.0-89c68fc</td>
+        <td>interuss/monitoring/v0.0.0-89aaa4b</td>
       </tr>
       <tr>
         <td>Commit hash</td>
-        <td>89c68fc0a1590b8442eeab9c8bddbe84f23c6974</td>
+        <td>89aaa4bba65b0bde664bed817e2968c855cc5658</td>
       </tr>
     </table>
   </div>

diff --git a/artifacts/uss_qualifier/reports/dss_probing/sequence/s1.html b/artifacts/uss_qualifier/reports/dss_probing/sequence/s1.html
diff --git a/artifacts/uss_qualifier/reports/dss_probing/sequence/s10.html b/artifacts/uss_qualifier/reports/dss_probing/sequence/s10.html
diff --git a/artifacts/uss_qualifier/reports/dss_probing/sequence/s11.html b/artifacts/uss_qualifier/reports/dss_probing/sequence/s11.html
@@ -91,7 +91,7 @@
 </head>
 <body>
 <div>
-    <h2><a href="https://github.com/interuss/monitoring/blob/89c68fc0a1590b8442eeab9c8bddbe84f23c6974/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a></h2>
+    <h2><a href="https://github.com/interuss/monitoring/blob/89aaa4bba65b0bde664bed817e2968c855cc5658/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md">ASTM NetRID DSS: ISA Expiry</a></h2>
   <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
   <table>
     <tr class="header_row">
@@ -105,10 +105,10 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
 
             <tr id="e1">
                 <td rowspan="2"><span class="sticky_cell_value">
-                    <a href="https://github.com/interuss/monitoring/blob/89c68fc0a1590b8442eeab9c8bddbe84f23c6974/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#setup-test-case">Setup</a>
+                    <a href="https://github.com/interuss/monitoring/blob/89aaa4bba65b0bde664bed817e2968c855cc5658/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#setup-test-case">Setup</a>
                 </span></td>
                 <td rowspan="2"><span class="sticky_cell_value">
-                    <a href="https://github.com/interuss/monitoring/blob/89c68fc0a1590b8442eeab9c8bddbe84f23c6974/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#ensure-clean-workspace-test-step">Ensure clean workspace</a>
+                    <a href="https://github.com/interuss/monitoring/blob/89aaa4bba65b0bde664bed817e2968c855cc5658/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#ensure-clean-workspace-test-step">Ensure clean workspace</a>
                 </span></td>
               <td><a href="#e1">1</a></td>
                 <td>&#x1F310;</td>
@@ -131,7 +131,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">initiated_at</div>:
-        <div class="node_value">    2023-10-31T18:26:08.355294Z
+        <div class="node_value">    2023-10-31T18:26:41.087662Z
 </div>
       </li>
       <li class="collapseable">
@@ -159,7 +159,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Authorization</div>:
-        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM1NywiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy5yZWFkLmlkZW50aWZpY2F0aW9uX3NlcnZpY2VfYXJlYXMiLCJzdWIiOiJ1c3NfcXVhbGlmaWVyIn0.REDACTED
+        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM5MCwiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy5yZWFkLmlkZW50aWZpY2F0aW9uX3NlcnZpY2VfYXJlYXMiLCJzdWIiOiJ1c3NfcXVhbGlmaWVyIn0.REDACTED
 </div>
       </li>
   </ul>
@@ -198,7 +198,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Date</div>:
-        <div class="node_value">    Tue, 31 Oct 2023 18:26:08 GMT
+        <div class="node_value">    Tue, 31 Oct 2023 18:26:41 GMT
 </div>
       </li>
       <li class="not_collapseable">
@@ -212,20 +212,20 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">elapsed_s</div>:
-        <div class="node_value">    0.001959
+        <div class="node_value">    0.001819
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">reported</div>:
-        <div class="node_value">    2023-10-31T18:26:08.357926Z
+        <div class="node_value">    2023-10-31T18:26:41.090248Z
 </div>
       </li>
       <li class="collapseable">
         <div class="node_key">json</div>:
         <div class="node_value">      <ul>
       <li class="not_collapseable">
         <div class="node_key">message</div>:
-        <div class="node_value">    ISA 00000171-e36d-40be-d38b-eca6ca350000 not found (E:0e5858e1-5ab2-4d12-8a11-6b3457c7042a)
+        <div class="node_value">    ISA 00000171-e36d-40be-d38b-eca6ca350000 not found (E:0bc697d7-3db4-4f11-a772-747e81574551)
 </div>
       </li>
   </ul>
@@ -263,10 +263,10 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
             </tr>
             <tr id="e3">
                 <td rowspan="6"><span class="sticky_cell_value">
-                    <a href="https://github.com/interuss/monitoring/blob/89c68fc0a1590b8442eeab9c8bddbe84f23c6974/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#isa-expiry-test-case">ISA Expiry</a>
+                    <a href="https://github.com/interuss/monitoring/blob/89aaa4bba65b0bde664bed817e2968c855cc5658/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#isa-expiry-test-case">ISA Expiry</a>
                 </span></td>
                 <td rowspan="6"><span class="sticky_cell_value">
-                    <a href="https://github.com/interuss/monitoring/blob/89c68fc0a1590b8442eeab9c8bddbe84f23c6974/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#isa-expiry-test-step">ISA Expiry</a>
+                    <a href="https://github.com/interuss/monitoring/blob/89aaa4bba65b0bde664bed817e2968c855cc5658/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md#isa-expiry-test-step">ISA Expiry</a>
                 </span></td>
               <td><a href="#e3">3</a></td>
                 <td>&#x1F310;</td>
@@ -289,7 +289,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">initiated_at</div>:
-        <div class="node_value">    2023-10-31T18:26:08.359314Z
+        <div class="node_value">    2023-10-31T18:26:41.091825Z
 </div>
       </li>
       <li class="collapseable">
@@ -327,7 +327,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Authorization</div>:
-        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM2OCwiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy53cml0ZS5pZGVudGlmaWNhdGlvbl9zZXJ2aWNlX2FyZWFzIiwic3ViIjoidXNzX3F1YWxpZmllciJ9.REDACTED
+        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDQwMCwiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy53cml0ZS5pZGVudGlmaWNhdGlvbl9zZXJ2aWNlX2FyZWFzIiwic3ViIjoidXNzX3F1YWxpZmllciJ9.REDACTED
 </div>
       </li>
   </ul>
@@ -441,12 +441,12 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">time_start</div>:
-        <div class="node_value">    2023-10-31T18:26:08.358494Z
+        <div class="node_value">    2023-10-31T18:26:41.090932Z
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">time_end</div>:
-        <div class="node_value">    2023-10-31T18:26:13.358494Z
+        <div class="node_value">    2023-10-31T18:26:46.090932Z
 </div>
       </li>
   </ul>
@@ -460,7 +460,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">request_id</div>:
-        <div class="node_value">    63b124ea-4239-4d34-ae9b-277948a5d8ef
+        <div class="node_value">    2bf1ba50-a98e-4a44-b99d-c8c94ee05188
 </div>
       </li>
   </ul>
@@ -494,7 +494,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Date</div>:
-        <div class="node_value">    Tue, 31 Oct 2023 18:26:08 GMT
+        <div class="node_value">    Tue, 31 Oct 2023 18:26:41 GMT
 </div>
       </li>
       <li class="not_collapseable">
@@ -508,12 +508,12 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">elapsed_s</div>:
-        <div class="node_value">    0.008765
+        <div class="node_value">    0.010387
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">reported</div>:
-        <div class="node_value">    2023-10-31T18:26:08.368882Z
+        <div class="node_value">    2023-10-31T18:26:41.103078Z
 </div>
       </li>
       <li class="collapseable">
@@ -541,17 +541,17 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">time_start</div>:
-        <div class="node_value">    2023-10-31T18:26:08.358494Z
+        <div class="node_value">    2023-10-31T18:26:41.090932Z
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">time_end</div>:
-        <div class="node_value">    2023-10-31T18:26:13.358494Z
+        <div class="node_value">    2023-10-31T18:26:46.090932Z
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">version</div>:
-        <div class="node_value">    1f4q4f1jea3s8
+        <div class="node_value">    1f4q4g02tfplo
 </div>
       </li>
       <li class="not_collapseable">
@@ -618,7 +618,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">initiated_at</div>:
-        <div class="node_value">    2023-10-31T18:26:13.377504Z
+        <div class="node_value">    2023-10-31T18:26:46.110419Z
 </div>
       </li>
       <li class="collapseable">
@@ -646,7 +646,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Authorization</div>:
-        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM1NywiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy5yZWFkLmlkZW50aWZpY2F0aW9uX3NlcnZpY2VfYXJlYXMiLCJzdWIiOiJ1c3NfcXVhbGlmaWVyIn0.REDACTED
+        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM5MCwiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy5yZWFkLmlkZW50aWZpY2F0aW9uX3NlcnZpY2VfYXJlYXMiLCJzdWIiOiJ1c3NfcXVhbGlmaWVyIn0.REDACTED
 </div>
       </li>
   </ul>
@@ -685,7 +685,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Date</div>:
-        <div class="node_value">    Tue, 31 Oct 2023 18:26:13 GMT
+        <div class="node_value">    Tue, 31 Oct 2023 18:26:46 GMT
 </div>
       </li>
       <li class="not_collapseable">
@@ -699,12 +699,12 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">elapsed_s</div>:
-        <div class="node_value">    0.00305
+        <div class="node_value">    0.003375
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">reported</div>:
-        <div class="node_value">    2023-10-31T18:26:13.381650Z
+        <div class="node_value">    2023-10-31T18:26:46.115142Z
 </div>
       </li>
       <li class="collapseable">
@@ -772,7 +772,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">initiated_at</div>:
-        <div class="node_value">    2023-10-31T18:26:13.383317Z
+        <div class="node_value">    2023-10-31T18:26:46.115947Z
 </div>
       </li>
       <li class="collapseable">
@@ -800,7 +800,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Authorization</div>:
-        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM1NywiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy5yZWFkLmlkZW50aWZpY2F0aW9uX3NlcnZpY2VfYXJlYXMiLCJzdWIiOiJ1c3NfcXVhbGlmaWVyIn0.REDACTED
+        <div class="node_value">    Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkc3MudXNzMi5sb2NhbHV0bSIsImV4cCI6MTY5ODc4MDM5MCwiaXNzIjoiZHVtbXkiLCJzY29wZSI6ImRzcy5yZWFkLmlkZW50aWZpY2F0aW9uX3NlcnZpY2VfYXJlYXMiLCJzdWIiOiJ1c3NfcXVhbGlmaWVyIn0.REDACTED
 </div>
       </li>
   </ul>
@@ -839,7 +839,7 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">Date</div>:
-        <div class="node_value">    Tue, 31 Oct 2023 18:26:13 GMT
+        <div class="node_value">    Tue, 31 Oct 2023 18:26:46 GMT
 </div>
       </li>
       <li class="not_collapseable">
@@ -853,12 +853,12 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">elapsed_s</div>:
-        <div class="node_value">    0.001981
+        <div class="node_value">    0.00227
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">reported</div>:
-        <div class="node_value">    2023-10-31T18:26:13.386132Z
+        <div class="node_value">    2023-10-31T18:26:46.119076Z
 </div>
       </li>
       <li class="collapseable">
@@ -879,17 +879,17 @@ <h3>scenarios.astm.netrid.v19.dss.ISAExpiry</h3>
       </li>
       <li class="not_collapseable">
         <div class="node_key">time_start</div>:
-        <div class="node_value">    2023-10-31T18:26:08.358494Z
+        <div class="node_value">    2023-10-31T18:26:41.090932Z
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">time_end</div>:
-        <div class="node_value">    2023-10-31T18:26:13.358494Z
+        <div class="node_value">    2023-10-31T18:26:46.090932Z
 </div>
       </li>
       <li class="not_collapseable">
         <div class="node_key">version</div>:
-        <div class="node_value">    1f4q4f1jea3s8
+        <div class="node_value">    1f4q4g02tfplo
 </div>
       </li>
       <li class="not_collapseable">