Skip to content

Commit

Permalink
fix column confusion in export, #397
Browse files Browse the repository at this point in the history
  • Loading branch information
@stitch
stitch committed Mar 12, 2024
1 parent fbffc9a commit 90f7b0a
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 15 deletions.
98 changes: 98 additions & 0 deletions dashboard/internet_nl_dashboard/logic/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,3 +245,101 @@ def operation_response(

MAIL_CATEGORIES = MAIL_IPV6_CATEGORY + MAIL_DNSSEC_CATEGORY + MAIL_AUTH_CATEGORY + MAIL_TLS_CATEGORY + \
MAIL_RPKI_CATEGORY + MAIL_LEGACY_CATEGORY

# When exporting, it also needs to be clear what field belongs to what category, otherwise duplicate
# field names will make it unclear what is meant. See examples in issue:
# This mapping is maintained in report_mixin.vue where the hierarchy is setup
# https://github.com/internetstandards/Internet.nl-dashboard/issues/397
# perhaps at a later state the mapping from the frontend can be copied 1 to 1, but that's more mental luggage
FIELD_TO_CATEGORY_MAP = {
'internet_nl_web_ipv6_ns_address': 'category_web_ipv6_name_server',
'internet_nl_web_ipv6_ns_reach': 'category_web_ipv6_name_server',
'internet_nl_web_ipv6_ws_address': 'category_web_ipv6_web_server',
'internet_nl_web_ipv6_ws_reach': 'category_web_ipv6_web_server',
'internet_nl_web_ipv6_ws_similar': 'category_web_ipv6_web_server',

'internet_nl_web_dnssec_exist': 'category_web_dnssec_dnssec',
'internet_nl_web_dnssec_valid': 'category_web_dnssec_dnssec',

'internet_nl_web_https_http_available': 'category_web_tls_http',
'internet_nl_web_https_http_redirect': 'category_web_tls_http',
'internet_nl_web_https_http_compress': 'category_web_tls_http',
'internet_nl_web_https_http_hsts': 'category_web_tls_http',

'internet_nl_web_https_tls_version': 'category_web_tls_tls',
'internet_nl_web_https_tls_ciphers': 'category_web_tls_tls',
'internet_nl_web_https_tls_cipherorder': 'category_web_tls_tls',
'internet_nl_web_https_tls_keyexchange': 'category_web_tls_tls',
'internet_nl_web_https_tls_keyexchangehash': 'category_web_tls_tls',
'internet_nl_web_https_tls_compress': 'category_web_tls_tls',
'internet_nl_web_https_tls_secreneg': 'category_web_tls_tls',
'internet_nl_web_https_tls_clientreneg': 'category_web_tls_tls',
'internet_nl_web_https_tls_0rtt': 'category_web_tls_tls',
'internet_nl_web_https_tls_ocsp': 'category_web_tls_tls',

'internet_nl_web_https_cert_chain': 'category_web_tls_certificate',
'internet_nl_web_https_cert_pubkey': 'category_web_tls_certificate',
'internet_nl_web_https_cert_sig': 'category_web_tls_certificate',
'internet_nl_web_https_cert_domain': 'category_web_tls_certificate',

'internet_nl_web_https_dane_exist': 'category_web_tls_dane',
'internet_nl_web_https_dane_valid': 'category_web_tls_dane',

'internet_nl_web_appsecpriv_x_frame_options': 'category_web_security_options_appsecpriv',
'internet_nl_web_appsecpriv_x_content_type_options': 'category_web_security_options_appsecpriv',
'internet_nl_web_appsecpriv_csp': 'category_web_security_options_appsecpriv',
'internet_nl_web_appsecpriv_referrer_policy': 'category_web_security_options_appsecpriv',
'internet_nl_web_appsecpriv_securitytxt': 'category_web_security_options_other',

'internet_nl_web_rpki_exists': 'category_web_rpki_name_server',
'internet_nl_web_rpki_valid': 'category_web_rpki_name_server',
'internet_nl_web_ns_rpki_exists': 'category_web_rpki_web_server',
'internet_nl_web_ns_rpki_valid': 'category_web_rpki_web_server',

'internet_nl_mail_ipv6_ns_address': 'category_mail_ipv6_name_servers',
'internet_nl_mail_ipv6_ns_reach': 'category_mail_ipv6_name_servers',

'internet_nl_mail_ipv6_mx_address': 'category_mail_ipv6_mail_servers',
'internet_nl_mail_ipv6_mx_reach': 'category_mail_ipv6_mail_servers',

'internet_nl_mail_dnssec_mailto_exist': 'category_mail_dnssec_email_address_domain',
'internet_nl_mail_dnssec_mailto_valid': 'category_mail_dnssec_email_address_domain',
'internet_nl_mail_dnssec_mx_exist': 'category_mail_dnssec_mail_server_domain',
'internet_nl_mail_dnssec_mx_valid': 'category_mail_dnssec_mail_server_domain',

'internet_nl_mail_auth_dmarc_exist': 'category_mail_dashboard_auth_dmarc',
'internet_nl_mail_auth_dmarc_policy': 'category_mail_dashboard_auth_dmarc',
'internet_nl_mail_auth_dkim_exist': 'category_mail_dashboard_aut_dkim',
'internet_nl_mail_auth_spf_exist': 'category_mail_dashboard_aut_spf',
'internet_nl_mail_auth_spf_policy': 'category_mail_dashboard_aut_spf',

'internet_nl_mail_starttls_tls_available': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_version': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_ciphers': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_cipherorder': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_keyexchange': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_keyexchangehash': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_compress': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_secreneg': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_clientreneg': 'category_mail_starttls_tls',
'internet_nl_mail_starttls_tls_0rtt': 'category_mail_starttls_tls',

'internet_nl_mail_starttls_cert_chain': 'category_mail_starttls_certificate',
'internet_nl_mail_starttls_cert_pubkey': 'category_mail_starttls_certificate',
'internet_nl_mail_starttls_cert_sig': 'category_mail_starttls_certificate',
'internet_nl_mail_starttls_cert_domain': 'category_mail_starttls_certificate',

'internet_nl_mail_starttls_dane_exist': 'category_mail_starttls_dane',
'internet_nl_mail_starttls_dane_valid': 'category_mail_starttls_dane',
'internet_nl_mail_starttls_dane_rollover': 'category_mail_starttls_dane',

'internet_nl_mail_rpki_exists': 'category_mail_rpki_name_server',
'internet_nl_mail_rpki_valid': 'category_mail_rpki_name_server',
'internet_nl_mail_ns_rpki_exists': 'category_mail_rpki_name_mail_server',
'internet_nl_mail_ns_rpki_valid': 'category_mail_rpki_name_mail_server',
'internet_nl_mail_mx_ns_rpki_exists': 'category_mail_rpki_mail_server',
'internet_nl_mail_mx_ns_rpki_valid': 'category_mail_rpki_mail_server',

# 'internet_nl_web_legacy_category': 'internet_nl_web_legacy_category',
# 'internet_nl_mail_legacy_category': 'internet_nl_mail_legacy_category',
}
36 changes: 36 additions & 0 deletions dashboard/internet_nl_dashboard/logic/internet_nl_translations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,32 @@ def translate_field(field_label, translation_dictionary: Dict[str, str]):
'internet_nl_mail_dashboard_ipv6': 'test_mailipv6_label',
'internet_nl_mail_dashboard_rpki': 'test_mailrpki_label',

"category_web_ipv6_name_server": 'results_domain_mail_ipv6_name_servers_label',
"category_web_ipv6_web_server": 'results_domain_ipv6_web_server_label',
"category_web_dnssec_dnssec": 'test_sitednssec_label',
"category_web_tls_http": 'results_domain_tls_https_label',
"category_web_tls_tls": 'results_domain_tls_tls_label',
"category_web_tls_certificate": 'results_domain_mail_tls_certificate_label',
"category_web_tls_dane": 'results_domain_mail_tls_dane_label',
"category_web_security_options_appsecpriv": 'results_domain_appsecpriv_http_headers_label',

'category_mail_ipv6_name_servers': 'results_domain_mail_ipv6_name_servers_label',
'category_mail_ipv6_mail_servers': 'results_mail_ipv6_mail_servers_label',
'category_mail_dnssec_email_address_domain': 'results_mail_dnssec_domain_label',
'category_mail_dnssec_mail_server_domain': 'results_mail_dnssec_mail_servers_label',
'category_mail_dashboard_auth_dmarc': 'results_mail_auth_dmarc_label',
'category_mail_dashboard_aut_dkim': 'results_mail_auth_dkim_label',
'category_mail_dashboard_aut_spf': 'results_mail_auth_spf_label',
'category_mail_starttls_tls': 'results_mail_tls_starttls_label',
'category_mail_starttls_certificate': 'results_domain_mail_tls_certificate_label',
'category_mail_starttls_dane': 'results_domain_mail_tls_dane_label',
'category_web_security_options_other': 'results_domain_appsecpriv_other_options_label',
'category_web_rpki_name_server': 'results_domain_mail_rpki_name_servers_label',
'category_web_rpki_web_server': 'results_domain_rpki_web_server_label',
'category_mail_rpki_name_server': 'results_domain_mail_rpki_name_servers_label',
'category_mail_rpki_name_mail_server': 'results_mail_rpki_mx_name_servers_label',
'category_mail_rpki_mail_server': 'results_mail_rpki_mail_servers_label',

'internet_nl_score': '% Score',
'internet_nl_score_report': 'Report',

Expand All @@ -376,6 +402,7 @@ def translate_field(field_label, translation_dictionary: Dict[str, str]):
'internet_nl_mail_legacy_category_ipv6': 'IPv6',
'internet_nl_mail_legacy_ipv6_nameserver': 'IPv6 nameserver',
'internet_nl_mail_legacy_ipv6_mailserver': 'IPv6 mailserver',
'internet_nl_mail_legacy_category': 'Extra Fields',

'internet_nl_web_legacy_dnssec': 'DNSSEC',
'internet_nl_web_legacy_tls_available': 'TLS available',
Expand All @@ -385,6 +412,7 @@ def translate_field(field_label, translation_dictionary: Dict[str, str]):
'internet_nl_web_legacy_category_ipv6': 'IPv6',
'internet_nl_web_legacy_ipv6_nameserver': 'IPv6 nameserver',
'internet_nl_web_legacy_ipv6_webserver': 'IPv6 webserver',
'internet_nl_web_legacy_category': 'Extra Fields',
# Deleted on request
# 'internet_nl_web_legacy_dane': 'DANE',

Expand All @@ -398,6 +426,14 @@ def translate_field(field_label, translation_dictionary: Dict[str, str]):
'legacy': 'Extra Fields',
'internet_nl_mail_dashboard_overall_score': 'Score',
'internet_nl_web_overall_score': 'Score',

'overall': "Score",
"ipv6": "Modern address (IPv6)",
"dnssec": "DNSSEC",
"tls": "Secure connection (HTTPS)",
"appsecpriv": "HTTP security headers / Other",
"rpki": "Route authorisation (RPKI)",
"auth": "Authenticity marks against phishing (DMARC, DKIM and SPF)",
}

# handle inconsistent naming and (why cannot i load something else than django.po?)
Expand Down
42 changes: 27 additions & 15 deletions dashboard/internet_nl_dashboard/logic/report_to_spreadsheet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,17 @@
from openpyxl.styles import Font, PatternFill
from websecmap.reporting.diskreport import retrieve_report

from dashboard.internet_nl_dashboard.logic import MAIL_AUTH_CATEGORY # pylint: disable=duplicate-code
from dashboard.internet_nl_dashboard.logic import (MAIL_AUTH_FIELDS, MAIL_DNSSEC_CATEGORY, MAIL_DNSSEC_FIELDS,
MAIL_IPV6_CATEGORY, MAIL_IPV6_FIELDS, MAIL_LEGACY_FIELDS,
MAIL_OVERALL_FIELDS, MAIL_RPKI_CATEGORY, MAIL_RPKI_FIELDS,
MAIL_TLS_CATEGORY, MAIL_TLS_CERTIFICATE_FIELDS, MAIL_TLS_DANE_FIELDS,
MAIL_TLS_TLS_FIELDS, WEB_APPSECPRIV_CATEGORY, WEB_APPSECPRIV_FIELDS,
WEB_DNSSEC_CATEGORY, WEB_DNSSEC_FIELDS, WEB_IPV6_CATEGORY,
WEB_IPV6_FIELDS, WEB_LEGACY_CATEGORY, WEB_LEGACY_FIELDS,
WEB_OVERALL_FIELDS, WEB_RPKI_CATEGORY, WEB_RPKI_FIELDS,
WEB_TLS_CATEGORY, WEB_TLS_CERTIFICATE_FIELDS, WEB_TLS_DANE_FIELDS,
WEB_TLS_HTTP_FIELDS, WEB_TLS_TLS_FIELDS)
from dashboard.internet_nl_dashboard.logic import FIELD_TO_CATEGORY_MAP # pylint: disable=duplicate-code
from dashboard.internet_nl_dashboard.logic import (MAIL_AUTH_CATEGORY, MAIL_AUTH_FIELDS, MAIL_DNSSEC_CATEGORY,
MAIL_DNSSEC_FIELDS, MAIL_IPV6_CATEGORY, MAIL_IPV6_FIELDS,
MAIL_LEGACY_FIELDS, MAIL_OVERALL_FIELDS, MAIL_RPKI_CATEGORY,
MAIL_RPKI_FIELDS, MAIL_TLS_CATEGORY, MAIL_TLS_CERTIFICATE_FIELDS,
MAIL_TLS_DANE_FIELDS, MAIL_TLS_TLS_FIELDS, WEB_APPSECPRIV_CATEGORY,
WEB_APPSECPRIV_FIELDS, WEB_DNSSEC_CATEGORY, WEB_DNSSEC_FIELDS,
WEB_IPV6_CATEGORY, WEB_IPV6_FIELDS, WEB_LEGACY_CATEGORY,
WEB_LEGACY_FIELDS, WEB_OVERALL_FIELDS, WEB_RPKI_CATEGORY,
WEB_RPKI_FIELDS, WEB_TLS_CATEGORY, WEB_TLS_CERTIFICATE_FIELDS,
WEB_TLS_DANE_FIELDS, WEB_TLS_HTTP_FIELDS, WEB_TLS_TLS_FIELDS)
from dashboard.internet_nl_dashboard.logic.internet_nl_translations import get_po_as_dictionary_v2, translate_field
from dashboard.internet_nl_dashboard.models import Account, TaggedUrlInUrllist, Url, UrlListReport

Expand Down Expand Up @@ -152,6 +152,7 @@ def create_spreadsheet(account: Account, report_id: int):
# add an empty row for clarity
data += [[]]
data += [category_headers(protocol)]
data += [subcategory_headers(protocol)]
data += [headers(protocol)]
data += urllistreport_to_spreadsheet_data(category_name=report.urllist.name, urls=urls, protocol=protocol,
tags=url_tag_mapping)
Expand Down Expand Up @@ -238,9 +239,10 @@ def upgrade_excel_spreadsheet(spreadsheet_data):
for cell in data_columns:
worksheet[f'{cell}11'].font = Font(bold=True)
worksheet[f'{cell}12'].font = Font(bold=True)
worksheet[f'{cell}13'].font = Font(bold=True)

# Freeze pane to make navigation easier.
worksheet.freeze_panes = worksheet['K13']
worksheet.freeze_panes = worksheet['K14']

# there is probably a feature that puts this in a single conditional value.
conditional_rules = {
Expand Down Expand Up @@ -309,6 +311,18 @@ def category_headers(protocol: str = 'dns_soa'):
return sheet_headers


def subcategory_headers(protocol: str = 'dns_soa'):
sheet_headers = ['', '', '', '', '', '', '']
for group in SANE_COLUMN_ORDER[protocol]:
sheet_headers += SANE_COLUMN_ORDER[protocol][group]
# add empty thing after each group to make distinction per group clearer
sheet_headers += ['']

# translate them:
return [translate_field(FIELD_TO_CATEGORY_MAP.get(header, ''),
translation_dictionary=po_file_as_dictionary) for header in sheet_headers]


def headers(protocol: str = 'dns_soa'):
sheet_headers = ['List', 'Url', "Subdomain", "Domain", "Suffix", 'Tags', 'InStats']
for group in SANE_COLUMN_ORDER[protocol]:
Expand All @@ -317,9 +331,7 @@ def headers(protocol: str = 'dns_soa'):
sheet_headers += ['']

# translate them:
sheet_headers = [translate_field(header, translation_dictionary=po_file_as_dictionary) for header in sheet_headers]

return sheet_headers
return [translate_field(header, translation_dictionary=po_file_as_dictionary) for header in sheet_headers]


def formula_row(function: str, protocol: str = 'dns_soa'):
Expand Down

0 comments on commit 90f7b0a

Please sign in to comment.