From d0163daee50e46b199591d2a1932007f84e1e727 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Fri, 16 Jun 2023 11:53:24 +0200 Subject: [PATCH] Fix #745 - Add RPKI test results to overall score --- checks/categories.py | 17 +++- checks/migrations/0015_add_rpki_scoring.py | 78 +++++++++++++++++++ checks/models.py | 14 +++- checks/scoring.py | 21 +++-- checks/tasks/rpki.py | 31 ++++---- integration_tests/develop/test_website.py | 2 +- integration_tests/integration/test_batch.py | 2 +- integration_tests/integration/test_email.py | 2 +- integration_tests/integration/test_website.py | 2 +- 9 files changed, 138 insertions(+), 31 deletions(-) create mode 100644 checks/migrations/0015_add_rpki_scoring.py diff --git a/checks/categories.py b/checks/categories.py index d4a2829b0..0eaeb6483 100644 --- a/checks/categories.py +++ b/checks/categories.py @@ -260,6 +260,7 @@ class RpkiExists(Subtest): _label = None _test_label = "exists" _test_name = None + _score_field = None def __init__(self): super().__init__( @@ -268,7 +269,9 @@ def __init__(self): explanation=f"detail {self._label} rpki {self._test_label} exp", tech_string=f"detail {self._label} rpki {self._test_label} tech table", init_tech_type="table_multi_col", - worst_status=STATUS_INFO, + worst_status=STATUS_FAIL, + full_score=scoring.RPKI_EXISTS_GOOD, + model_score_field=self._score_field, ) def was_tested(self): @@ -299,6 +302,7 @@ def result_validator_error(self): class WebRpkiExists(RpkiExists): _label = "web" _test_name = "web_rpki_exists" + _score_field = "web_exists_score" def was_tested(self): self.worst_status = scoring.WEB_RPKI_EXISTENCE_WORST_STATUS @@ -307,6 +311,7 @@ def was_tested(self): class MailRpkiExists(RpkiExists): _label = "mail" _test_name = "mail_rpki_exists" + _score_field = "mail_exists_score" def was_tested(self): self.worst_status = scoring.MAIL_RPKI_EXISTENCE_WORST_STATUS @@ -316,6 +321,7 @@ class RpkiValid(Subtest): _label = None _test_label = "valid" _test_name = None + _score_field = None def __init__(self): super().__init__( @@ -324,7 +330,8 @@ def __init__(self): explanation=f"detail {self._label} rpki {self._test_label} exp", tech_string=f"detail {self._label} rpki {self._test_label} tech table", init_tech_type="table", - worst_status=STATUS_INFO, + full_score=scoring.RPKI_VALID_GOOD, + model_score_field=self._score_field, ) def was_tested(self): @@ -365,6 +372,7 @@ def result_validator_error(self): class WebRpkiValid(RpkiValid): _label = "web" _test_name = "web_rpki_valid" + _score_field = "web_valid_score" def was_tested(self): self.worst_status = scoring.WEB_RPKI_VALIDITY_WORST_STATUS @@ -373,6 +381,7 @@ def was_tested(self): class MailRpkiValid(RpkiValid): _label = "mail" _test_name = "mail_rpki_valid" + _score_field = "mail_valid_score" def was_tested(self): self.worst_status = scoring.MAIL_RPKI_VALIDITY_WORST_STATUS @@ -382,6 +391,7 @@ class NsRpkiExists(RpkiExists): _label = "web-mail" _test_label = "ns-exists" _test_name = "ns_rpki_exists" + _score_field = "ns_exists_score" def was_tested(self): self.worst_status = scoring.RPKI_NS_EXISTENCE_WORST_STATUS @@ -391,6 +401,7 @@ class NsRpkiValid(RpkiValid): _label = "web-mail" _test_label = "ns-valid" _test_name = "ns_rpki_valid" + _score_field = "ns_valid_score" def was_tested(self): self.worst_status = scoring.RPKI_NS_VALIDITY_WORST_STATUS @@ -400,6 +411,7 @@ class MailMxNsRpkiExists(RpkiExists): _label = "mail" _test_label = "mx-ns-exists" _test_name = "mail_mx_ns_rpki_exists" + _score_field = "mx_ns_exists_score" def was_tested(self): self.worst_status = scoring.MAIL_RPKI_MX_NS_EXISTENCE_WORST_STATUS @@ -409,6 +421,7 @@ class MailMxNsRpkiValid(RpkiValid): _label = "mail" _test_label = "mx-ns-valid" _test_name = "mail_mx_ns_rpki_valid" + _score_field = "mx_ns_valid_score" def was_tested(self): self.worst_status = scoring.MAIL_RPKI_MX_NS_VALIDITY_WORST_STATUS diff --git a/checks/migrations/0015_add_rpki_scoring.py b/checks/migrations/0015_add_rpki_scoring.py new file mode 100644 index 000000000..b66dd9911 --- /dev/null +++ b/checks/migrations/0015_add_rpki_scoring.py @@ -0,0 +1,78 @@ +# Generated by Django 3.2.19 on 2023-06-15 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("checks", "0015_auto_20240212_1616"), + ] + + operations = [ + migrations.RemoveField( + model_name="mailtestrpki", + name="mail_score", + ), + migrations.RemoveField( + model_name="mailtestrpki", + name="ns_score", + ), + migrations.RemoveField( + model_name="webtestrpki", + name="ns_score", + ), + migrations.RemoveField( + model_name="webtestrpki", + name="web_score", + ), + migrations.AddField( + model_name="mailtestrpki", + name="mail_exists_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="mailtestrpki", + name="mail_valid_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="mailtestrpki", + name="mx_ns_exists_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="mailtestrpki", + name="mx_ns_valid_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="mailtestrpki", + name="ns_exists_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="mailtestrpki", + name="ns_valid_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="webtestrpki", + name="ns_exists_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="webtestrpki", + name="ns_valid_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="webtestrpki", + name="web_exists_score", + field=models.IntegerField(null=True), + ), + migrations.AddField( + model_name="webtestrpki", + name="web_valid_score", + field=models.IntegerField(null=True), + ), + ] diff --git a/checks/models.py b/checks/models.py index 4eb9ca19a..fbf40c1ff 100644 --- a/checks/models.py +++ b/checks/models.py @@ -783,8 +783,10 @@ class WebTestRpki(BaseTestModel): timestamp = models.DateTimeField(auto_now_add=True) domain = models.CharField(max_length=255, default="") report = ListField(default="") - web_score = models.IntegerField(null=True) - ns_score = models.IntegerField(null=True) + web_exists_score = models.IntegerField(null=True) + web_valid_score = models.IntegerField(null=True) + ns_exists_score = models.IntegerField(null=True) + ns_valid_score = models.IntegerField(null=True) score = models.IntegerField(null=True) max_score = models.IntegerField(null=True) @@ -799,8 +801,12 @@ class MailTestRpki(BaseTestModel): timestamp = models.DateTimeField(auto_now_add=True) domain = models.CharField(max_length=255, default="") report = ListField(default="") - mail_score = models.IntegerField(null=True) - ns_score = models.IntegerField(null=True) + mail_exists_score = models.IntegerField(null=True) + mail_valid_score = models.IntegerField(null=True) + ns_exists_score = models.IntegerField(null=True) + ns_valid_score = models.IntegerField(null=True) + mx_ns_exists_score = models.IntegerField(null=True) + mx_ns_valid_score = models.IntegerField(null=True) score = models.IntegerField(null=True) max_score = models.IntegerField(null=True) diff --git a/checks/scoring.py b/checks/scoring.py index e716656c8..4b5606819 100644 --- a/checks/scoring.py +++ b/checks/scoring.py @@ -63,8 +63,13 @@ IPV6_NS_CONN_FAIL = NO_POINTS IPV6_NS_CONN_WORST_STATUS = STATUS_FAIL -RPKI_NS_EXISTENCE_WORST_STATUS = STATUS_NOTICE -RPKI_NS_VALIDITY_WORST_STATUS = STATUS_NOTICE +RPKI_NS_EXISTENCE_WORST_STATUS = STATUS_FAIL +RPKI_NS_VALIDITY_WORST_STATUS = STATUS_FAIL + +RPKI_EXISTS_GOOD = HALF_WEIGHT_POINTS +RPKI_EXISTS_FAIL = NO_POINTS +RPKI_VALID_GOOD = FULL_WEIGHT_POINTS +RPKI_VALID_FAIL = NO_POINTS # --- WEBTEST @@ -84,8 +89,8 @@ WEB_DNSSEC_ERROR = NO_POINTS WEB_DNSSEC_WORST_STATUS = STATUS_FAIL -WEB_RPKI_EXISTENCE_WORST_STATUS = STATUS_NOTICE -WEB_RPKI_VALIDITY_WORST_STATUS = STATUS_NOTICE +WEB_RPKI_EXISTENCE_WORST_STATUS = STATUS_FAIL +WEB_RPKI_VALIDITY_WORST_STATUS = STATUS_FAIL # The HTTPS_EXISTS test is not scored. The scoring of the category relies on # the subsequent tests. @@ -240,10 +245,10 @@ MAIL_AUTH_SPF_POLICY_FAIL = NO_POINTS MAIL_AUTH_SPF_POLICY_WORST_STATUS = STATUS_FAIL -MAIL_RPKI_EXISTENCE_WORST_STATUS = STATUS_NOTICE -MAIL_RPKI_MX_NS_EXISTENCE_WORST_STATUS = STATUS_NOTICE -MAIL_RPKI_MX_NS_VALIDITY_WORST_STATUS = STATUS_NOTICE -MAIL_RPKI_VALIDITY_WORST_STATUS = STATUS_NOTICE +MAIL_RPKI_EXISTENCE_WORST_STATUS = STATUS_FAIL +MAIL_RPKI_MX_NS_EXISTENCE_WORST_STATUS = STATUS_FAIL +MAIL_RPKI_MX_NS_VALIDITY_WORST_STATUS = STATUS_FAIL +MAIL_RPKI_VALIDITY_WORST_STATUS = STATUS_FAIL MAIL_TLS_STARTTLS_EXISTS_GOOD = FULL_WEIGHT_POINTS MAIL_TLS_STARTTLS_EXISTS_WORST_STATUS = STATUS_FAIL diff --git a/checks/tasks/rpki.py b/checks/tasks/rpki.py index 134224413..9614facff 100644 --- a/checks/tasks/rpki.py +++ b/checks/tasks/rpki.py @@ -21,7 +21,7 @@ TeamCymruIPtoASN, RelyingPartyUnvailableError, ) -from .. import categories +from .. import categories, scoring from ..models import ( MailTestRpki, WebTestRpki, @@ -249,7 +249,7 @@ def batch_mail_mx_ns_rpki(self, mx_ips_pairs, url, *args, **kwargs): return do_mx_ns_rpki(mx_ips_pairs, url, self, *args, **kwargs) -def generate_roa_existence_report(subtestname, category, hostset) -> None: +def generate_roa_existence_report(subtestname, category, hostset) -> int: """Generate a test report for the existence of ROAs.""" def gen_tech_data(host, ip, validity) -> List[List[str]]: @@ -296,11 +296,13 @@ def gen_tech_data(host, ip, validity) -> List[List[str]]: category.subtests[subtestname].result_no_addresses() elif missing_count > 0: category.subtests[subtestname].result_bad(tech_data) + return scoring.RPKI_EXISTS_FAIL else: category.subtests[subtestname].result_good(tech_data) + return scoring.RPKI_EXISTS_GOOD -def generate_validity_report(subtestname, category, hostset) -> None: +def generate_validity_report(subtestname, category, hostset) -> int: """Generate a test report based on Route Origin Validation. This compares routing data from BGP with published ROAs. @@ -368,12 +370,15 @@ def gen_tech_data(host, asn, prefix, validity, errors) -> List[str]: category.subtests[subtestname].result_no_addresses() elif invalid_count > 0: category.subtests[subtestname].result_invalid(tech_data) + return scoring.RPKI_VALID_FAIL elif not_valid_count > 0: category.subtests[subtestname].result_bad(tech_data) + return scoring.RPKI_VALID_FAIL elif not_routed_count == count: # no BGP data for all IPs category.subtests[subtestname].result_not_routed(tech_data) else: category.subtests[subtestname].result_good(tech_data) + return scoring.RPKI_VALID_GOOD def build_summary_report(parent, parent_name, category) -> None: @@ -382,22 +387,22 @@ def build_summary_report(parent, parent_name, category) -> None: webset = parent.webhosts.all().order_by("host") nsset = parent.nshosts.all().order_by("host") - generate_roa_existence_report("web_rpki_exists", category, webset) - generate_validity_report("web_rpki_valid", category, webset) - generate_roa_existence_report("ns_rpki_exists", category, nsset) - generate_validity_report("ns_rpki_valid", category, nsset) + parent.web_exists_score = generate_roa_existence_report("web_rpki_exists", category, webset) + parent.web_valid_score = generate_validity_report("web_rpki_valid", category, webset) + parent.ns_exists_score = generate_roa_existence_report("ns_rpki_exists", category, nsset) + parent.ns_valid_score = generate_validity_report("ns_rpki_valid", category, nsset) elif parent_name == "mailtestrpki": mxset = parent.mxhosts.all().order_by("host") nsset = parent.nshosts.all().order_by("host") mxnsset = parent.mxnshosts.all().order_by("host") - generate_roa_existence_report("mail_rpki_exists", category, mxset) - generate_validity_report("mail_rpki_valid", category, mxset) - generate_roa_existence_report("ns_rpki_exists", category, nsset) - generate_validity_report("ns_rpki_valid", category, nsset) - generate_roa_existence_report("mail_mx_ns_rpki_exists", category, mxnsset) - generate_validity_report("mail_mx_ns_rpki_valid", category, mxnsset) + parent.mail_exists_score = generate_roa_existence_report("mail_rpki_exists", category, mxset) + parent.mail_valid_score = generate_validity_report("mail_rpki_valid", category, mxset) + parent.ns_exists_score = generate_roa_existence_report("ns_rpki_exists", category, nsset) + parent.ns_valid_score = generate_validity_report("ns_rpki_valid", category, nsset) + parent.mx_ns_exists_score = generate_roa_existence_report("mail_mx_ns_rpki_exists", category, mxnsset) + parent.mx_ns_valid_score = generate_validity_report("mail_mx_ns_rpki_valid", category, mxnsset) parent.report = category.gen_report() parent.save() diff --git a/integration_tests/develop/test_website.py b/integration_tests/develop/test_website.py index 0a55f23b6..49e7be9ba 100644 --- a/integration_tests/develop/test_website.py +++ b/integration_tests/develop/test_website.py @@ -7,7 +7,7 @@ INVALID_DOMAIN = "invalid-domain.example.com" TEST_DOMAIN_EXPECTED_SCORE = 100 -TEST_DOMAIN_EXPECTED_SCORE_NO_IPV6 = 48 +TEST_DOMAIN_EXPECTED_SCORE_NO_IPV6 = 61 LONG_TIMEOUT = 1000 * 3 * 60 diff --git a/integration_tests/integration/test_batch.py b/integration_tests/integration/test_batch.py index 909d5b6d4..e60135ccc 100644 --- a/integration_tests/integration/test_batch.py +++ b/integration_tests/integration/test_batch.py @@ -10,7 +10,7 @@ TEST_DOMAIN = "target.test" TEST_DOMAIN_EXPECTED_SCORE = 100 # TODO: improve test environment to allow 100% score result -TEST_DOMAIN_EXPECTED_SCORE = 48 +TEST_DOMAIN_EXPECTED_SCORE = 49 def wait_for_request_status(url, expected_status, timeout=10, interval=1, auth=None): diff --git a/integration_tests/integration/test_email.py b/integration_tests/integration/test_email.py index 015f80dd5..691dfda96 100644 --- a/integration_tests/integration/test_email.py +++ b/integration_tests/integration/test_email.py @@ -7,7 +7,7 @@ ALL_EMAIL_PROBES = {"ipv6", "dnssec", "tls", "auth", "rpki"} TEST_EMAIL_EXPECTED_SCORE = 100 # TODO: improve test environment to allow 100% score result -TEST_EMAIL_EXPECTED_SCORE = 17 +TEST_EMAIL_EXPECTED_SCORE = 20 def test_your_email_score(page, app_url, test_email): diff --git a/integration_tests/integration/test_website.py b/integration_tests/integration/test_website.py index fcbb74344..6b2dd2c87 100644 --- a/integration_tests/integration/test_website.py +++ b/integration_tests/integration/test_website.py @@ -9,7 +9,7 @@ ALL_PROBES = {"ipv6", "dnssec", "tls", "appsecpriv", "rpki"} TEST_DOMAIN_EXPECTED_SCORE = 100 # TODO: improve test environment to allow 100% score result -TEST_DOMAIN_EXPECTED_SCORE = 48 +TEST_DOMAIN_EXPECTED_SCORE = 49 def test_reject_invalid_domain(page, app_url):