fix several bugs and further optimze uploads of large domain lists

CHANGELOG.md

@@ -4,6 +4,49 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## V4.3.0 - t.b.d.
+
+Version 4.3.0 targets to support creating lists and viewing reports up to 10.000 domains.
+
+### Added
+- Password change functionality
+
+### Changed
+- Reduced the amount of memory needed to view reports
+- Improved interaction of uploading spreadsheets with domain
+- Excel spreadsheets are now formatted > 5000 rows
+- Improved a11y
+- Some changes on the signup form and e-mail
+- The results table in the report is now much higher
+- Footer links have changed
+- API 2.4.0 support
+
+### Bugfixes
+- 409 Deleting lists also deletes associated reports
+- 338 domain was missing in the example sharing url
+- 486 password was not set when sharing a report with a password the first time
+- 344 label fuction was not working
+- 471 Timeline becomes ugly with a lot of reports
+- Several other small issues
+
+
+## V4.2.2 - 6 nov 2023
+
+### Bugfixes
+- 487 Spreadsheet uploads without tags crash
+- 483 Tags don't overwrite on upload
+- Fixed crash INTERNET-NL-DASHBOARD-69: loading translations of a non existing language, template did not exist. Falls back to a supported language if no language is supplied
+- Update websecmap and other dependencies to fix vulnerabilities
+
+
+## V4.2.1 - 19 sept 2023
+
+### Bugfixes
+- Add tags to spreadsheet exports
+- 463 Shared reports without passwords are not loaded
+- 466 Filtering on report removes entire table
+
+
 ## V4.2.0 - 10 July 2023
 
 ### Added

dashboard/internet_nl_dashboard/admin.py

@@ -302,14 +302,24 @@ class TaggedUrlInUrllistAdmin(ImportExportModelAdmin, admin.ModelAdmin):
 @admin.register(AccountInternetNLScan)
 class AccountInternetNLScanAdmin(ImportExportModelAdmin, admin.ModelAdmin):
 
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+
+        # this column prevents loading of long lists
+        qs.defer('')
+
+        return qs
+
     list_display = ('id', 'account', 'account__name', 'state', 'internetnl_scan', 'internetnl_scan_type',
-                    'urllist', 'domains', 'started_on', 'finished_on')
+                    'urllist', 'started_on', 'finished_on')
 
     list_filter = ['account', 'urllist', 'state', 'started_on', 'finished_on'][::-1]
     search_fields = ('urllist__name', 'account__name')
 
     fields = ('state', 'state_changed_on', 'account', 'scan', 'urllist', 'started_on', 'finished_on')
 
+    readonly_fields = ('account', 'scan', 'urllist')
+
     @staticmethod
     def account__name(obj):
         return obj.account.internet_nl_api_username
@@ -324,7 +334,8 @@ def internetnl_scan_type(obj):
 
     @staticmethod
     def domains(obj):
-        return obj.scan.subject_urls.count()
+        return "?"
+        # return obj.scan.subject_urls.count()
 
     actions = []
 

dashboard/internet_nl_dashboard/logic/__init__.py

@@ -245,3 +245,101 @@ def operation_response(
 
 MAIL_CATEGORIES = MAIL_IPV6_CATEGORY + MAIL_DNSSEC_CATEGORY + MAIL_AUTH_CATEGORY + MAIL_TLS_CATEGORY + \
     MAIL_RPKI_CATEGORY + MAIL_LEGACY_CATEGORY
+
+# When exporting, it also needs to be clear what field belongs to what category, otherwise duplicate
+# field names will make it unclear what is meant. See examples in issue:
+# This mapping is maintained in report_mixin.vue where the hierarchy is setup
+# https://github.com/internetstandards/Internet.nl-dashboard/issues/397
+# perhaps at a later state the mapping from the frontend can be copied 1 to 1, but that's more mental luggage
+FIELD_TO_CATEGORY_MAP = {
+    'internet_nl_web_ipv6_ns_address': 'category_web_ipv6_name_server',
+    'internet_nl_web_ipv6_ns_reach': 'category_web_ipv6_name_server',
+    'internet_nl_web_ipv6_ws_address': 'category_web_ipv6_web_server',
+    'internet_nl_web_ipv6_ws_reach': 'category_web_ipv6_web_server',
+    'internet_nl_web_ipv6_ws_similar': 'category_web_ipv6_web_server',
+
+    'internet_nl_web_dnssec_exist': 'category_web_dnssec_dnssec',
+    'internet_nl_web_dnssec_valid': 'category_web_dnssec_dnssec',
+
+    'internet_nl_web_https_http_available': 'category_web_tls_http',
+    'internet_nl_web_https_http_redirect': 'category_web_tls_http',
+    'internet_nl_web_https_http_compress': 'category_web_tls_http',
+    'internet_nl_web_https_http_hsts': 'category_web_tls_http',
+
+    'internet_nl_web_https_tls_version': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_ciphers': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_cipherorder': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_keyexchange': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_keyexchangehash': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_compress': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_secreneg': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_clientreneg': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_0rtt': 'category_web_tls_tls',
+    'internet_nl_web_https_tls_ocsp': 'category_web_tls_tls',
+
+    'internet_nl_web_https_cert_chain': 'category_web_tls_certificate',
+    'internet_nl_web_https_cert_pubkey': 'category_web_tls_certificate',
+    'internet_nl_web_https_cert_sig': 'category_web_tls_certificate',
+    'internet_nl_web_https_cert_domain': 'category_web_tls_certificate',
+
+    'internet_nl_web_https_dane_exist': 'category_web_tls_dane',
+    'internet_nl_web_https_dane_valid': 'category_web_tls_dane',
+
+    'internet_nl_web_appsecpriv_x_frame_options': 'category_web_security_options_appsecpriv',
+    'internet_nl_web_appsecpriv_x_content_type_options': 'category_web_security_options_appsecpriv',
+    'internet_nl_web_appsecpriv_csp': 'category_web_security_options_appsecpriv',
+    'internet_nl_web_appsecpriv_referrer_policy': 'category_web_security_options_appsecpriv',
+    'internet_nl_web_appsecpriv_securitytxt': 'category_web_security_options_other',
+
+    'internet_nl_web_rpki_exists': 'category_web_rpki_name_server',
+    'internet_nl_web_rpki_valid': 'category_web_rpki_name_server',
+    'internet_nl_web_ns_rpki_exists': 'category_web_rpki_web_server',
+    'internet_nl_web_ns_rpki_valid': 'category_web_rpki_web_server',
+
+    'internet_nl_mail_ipv6_ns_address': 'category_mail_ipv6_name_servers',
+    'internet_nl_mail_ipv6_ns_reach': 'category_mail_ipv6_name_servers',
+
+    'internet_nl_mail_ipv6_mx_address': 'category_mail_ipv6_mail_servers',
+    'internet_nl_mail_ipv6_mx_reach': 'category_mail_ipv6_mail_servers',
+
+    'internet_nl_mail_dnssec_mailto_exist': 'category_mail_dnssec_email_address_domain',
+    'internet_nl_mail_dnssec_mailto_valid': 'category_mail_dnssec_email_address_domain',
+    'internet_nl_mail_dnssec_mx_exist': 'category_mail_dnssec_mail_server_domain',
+    'internet_nl_mail_dnssec_mx_valid': 'category_mail_dnssec_mail_server_domain',
+
+    'internet_nl_mail_auth_dmarc_exist': 'category_mail_dashboard_auth_dmarc',
+    'internet_nl_mail_auth_dmarc_policy': 'category_mail_dashboard_auth_dmarc',
+    'internet_nl_mail_auth_dkim_exist': 'category_mail_dashboard_aut_dkim',
+    'internet_nl_mail_auth_spf_exist': 'category_mail_dashboard_aut_spf',
+    'internet_nl_mail_auth_spf_policy': 'category_mail_dashboard_aut_spf',
+
+    'internet_nl_mail_starttls_tls_available': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_version': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_ciphers': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_cipherorder': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_keyexchange': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_keyexchangehash': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_compress': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_secreneg': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_clientreneg': 'category_mail_starttls_tls',
+    'internet_nl_mail_starttls_tls_0rtt': 'category_mail_starttls_tls',
+
+    'internet_nl_mail_starttls_cert_chain': 'category_mail_starttls_certificate',
+    'internet_nl_mail_starttls_cert_pubkey': 'category_mail_starttls_certificate',
+    'internet_nl_mail_starttls_cert_sig': 'category_mail_starttls_certificate',
+    'internet_nl_mail_starttls_cert_domain': 'category_mail_starttls_certificate',
+
+    'internet_nl_mail_starttls_dane_exist': 'category_mail_starttls_dane',
+    'internet_nl_mail_starttls_dane_valid': 'category_mail_starttls_dane',
+    'internet_nl_mail_starttls_dane_rollover': 'category_mail_starttls_dane',
+
+    'internet_nl_mail_rpki_exists': 'category_mail_rpki_name_server',
+    'internet_nl_mail_rpki_valid': 'category_mail_rpki_name_server',
+    'internet_nl_mail_ns_rpki_exists': 'category_mail_rpki_name_mail_server',
+    'internet_nl_mail_ns_rpki_valid': 'category_mail_rpki_name_mail_server',
+    'internet_nl_mail_mx_ns_rpki_exists': 'category_mail_rpki_mail_server',
+    'internet_nl_mail_mx_ns_rpki_valid': 'category_mail_rpki_mail_server',
+
+    # 'internet_nl_web_legacy_category': 'internet_nl_web_legacy_category',
+    # 'internet_nl_mail_legacy_category': 'internet_nl_mail_legacy_category',
+}