diff --git a/monitoring/monitorlib/schema_validation.py b/monitoring/monitorlib/schema_validation.py index 8865e09914..4e79bf0c00 100644 --- a/monitoring/monitorlib/schema_validation.py +++ b/monitoring/monitorlib/schema_validation.py @@ -1,13 +1,13 @@ import os.path from dataclasses import dataclass +from datetime import datetime from enum import Enum from pathlib import Path -from typing import List, Dict, Type, TypeVar +from typing import List, Dict, Type import bc_jsonpath_ng import jsonschema.validators import yaml - from implicitdict import ImplicitDict from implicitdict.jsonschema import SchemaVars, make_json_schema diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md index 8de64d6ca7..7f340ba94b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/create.md @@ -7,12 +7,12 @@ This test step fragment validates that operational intent references can be crea As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to create an operational intent reference with either one or both of the start and end time missing, provided all the required parameters are valid. -## 🛑 Create operational intent reference response is correct check +## 🛑 Create operational intent reference response content is correct check -A successful operational intent reference creation query is expected to return a well-defined body, the content of which reflects the created operational intent reference. -If the format and content of the response are not conforming, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. +A successful operational intent reference creation query is expected to return a body, the content of which reflects the created operational intent reference. +If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. -This check will usually be performing a series of sub-checks from the [validate](../validate) fragments, as well as the format conformance check in this fragment. +This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. ## ⚠️ Create operational intent reference response format conforms to spec check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md index abc46716ee..eab08126c8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/delete.md @@ -6,12 +6,12 @@ This test step fragment validates that operational intent references can be dele A query to delete an operational intent reference, by its owner and when the correct OVN is provided, should succeed, otherwise the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. -## 🛑 Delete operational intent reference response is correct check +## 🛑 Delete operational intent reference response content is correct check -A successful operational intent reference deletion query is expected to return a well-defined body, the content of which reflects the operational intent reference at the moment of deletion. -If the format and content of the response are not conforming, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. +A successful operational intent reference deletion query is expected to return a body, the content of which reflects the operational intent reference at the moment of deletion. +If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. -This check will usually be performing a series of sub-checks from the [validate](../validate) fragments, as well as the format conformance check in this fragment. +This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. ## ⚠️ Delete operational intent reference response format conforms to spec check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md index 615c655547..e3e66d3e00 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/update.md @@ -6,12 +6,12 @@ This test step fragment validates that operational intent references can be upda As per **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**, the DSS API must allow callers to mutate an operational intent reference. -## 🛑 Mutate operational intent reference response is correct check +## 🛑 Mutate operational intent reference response content is correct check A successful operational intent reference mutation query is expected to return a well-defined body, the content of which reflects the updated operational intent reference. -If the format and content of the response are not conforming, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. +If the content of the response does not correspond to what was requested, the DSS is failing to implement **[astm.f3548.v21.DSS0005,1](../../../../../../../requirements/astm/f3548/v21.md)**. -This check will usually be performing a series of sub-checks from the [validate](../validate) fragments, as well as the format conformance check in this fragment. +This check will usually be performing a series of sub-checks from the [validate](../validate) fragments. ## ⚠️ Mutate operational intent reference response format conforms to spec check diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py index 2fd5766144..743ec9df6b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py @@ -118,3 +118,27 @@ def cleanup_active_oirs( for oir in oirs: if oir.manager == manager_identity: remove_op_intent(scenario, dss, oir.id, oir.ovn) + + +def cleanup_op_intent( + scenario: TestScenarioType, dss: DSSInstance, oi_id: EntityID +) -> None: + """Remove the specified operational intent reference from the DSS, if it exists.""" + + with scenario.check( + "Operational intent references can be queried by ID", [dss.participant_id] + ) as check: + try: + oir, q = dss.get_op_intent_reference(oi_id) + except fetch.QueryError as e: + scenario.record_queries(e.queries) + if e.cause_status_code != 404: + check.record_failed( + summary="OIR Get query returned code different from 200 or 404", + details=e.msg, + query_timestamps=e.query_timestamps, + ) + else: + return + + remove_op_intent(scenario, dss, oi_id, oir.ovn) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py index b6070467a0..2b073f2c70 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/validators/oir_validator.py @@ -65,6 +65,9 @@ def _fail_sub_check( Fail the passed sub check with the passed summary and details, and fail the main check with the passed details. + Note that this method should only be used to fail sub-checks related to the CONTENT of the OIR, + but not its FORMAT, as the main-check should only be pertaining to the content. + The provided timestamp is forwarded into the query_timestamps of the check failure. """ sub_check.record_failed( @@ -79,23 +82,6 @@ def _fail_sub_check( query_timestamps=[t_dss], ) - def _fail_with_schema_errors( - self, - check: PendingCheck, - errors: List[schema_validation.ValidationError], - t_dss: datetime, - ) -> None: - """Fail the passed check with the passed schema validation errors, and fail - the main check with the passed details.""" - details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) - self._fail_sub_check( - check, - summary="Response format was invalid", - details="Found the following schema validation errors in the DSS response:\n" - + details, - t_dss=t_dss, - ) - def _validate_oir( self, expected_entity_id: EntityID, @@ -302,7 +288,7 @@ def _validate_put_oir_response_schema( oir_query.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + _fail_with_schema_errors(check, errors, t_dss) return False return True @@ -389,7 +375,7 @@ def validate_fetched_oir( fetched_oir.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + _fail_with_schema_errors(check, errors, t_dss) parsed_resp = fetched_oir.parse_json_result( GetOperationalIntentReferenceResponse @@ -469,7 +455,7 @@ def validate_searched_oir_format( search_response.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + _fail_with_schema_errors(check, errors, t_dss) def validate_deleted_oir( self, @@ -492,7 +478,7 @@ def validate_deleted_oir( deleted_oir.response.json, ) if errors: - self._fail_with_schema_errors(check, errors, t_dss) + _fail_with_schema_errors(check, errors, t_dss) oir_resp = deleted_oir.parse_json_result( ChangeOperationalIntentReferenceResponse @@ -508,3 +494,23 @@ def validate_deleted_oir( previous_version=None, expected_version=expected_version, ) + + +def _fail_with_schema_errors( + check: PendingCheck, + errors: List[schema_validation.ValidationError], + t_dss: datetime, +) -> None: + """ + Fail the passed check with the passed schema validation errors. + Note: + The main check IS NOT failed: + The main check pertains to the CONTENT of the response but not its FORMAT. + """ + details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) + check.record_failed( + summary="Response format was invalid", + details="Found the following schema validation errors in the DSS response:\n" + + details, + query_timestamps=[t_dss], + )