From 4bbb137d019205fce9a63e6e180859ec3207dd01 Mon Sep 17 00:00:00 2001 From: stitch1 Date: Wed, 6 Nov 2024 13:37:19 +0100 Subject: [PATCH 1/4] add subdomain suggestion settings, call and tests --- .../internet_nl_dashboard/logic/domains.py | 19 +++++++++++ .../tests/test_domain_management.py | 17 +++++++++- dashboard/internet_nl_dashboard/urls.py | 1 + dashboard/internet_nl_dashboard/views/app.py | 11 +++++++ .../internet_nl_dashboard/views/domains.py | 9 ++++- dashboard/settings.py | 33 ++++++++++++++++++- requirements-dev.in | 3 ++ requirements-dev.txt | 6 ++-- 8 files changed, 94 insertions(+), 5 deletions(-) diff --git a/dashboard/internet_nl_dashboard/logic/domains.py b/dashboard/internet_nl_dashboard/logic/domains.py index 859b927a..3e7e4489 100644 --- a/dashboard/internet_nl_dashboard/logic/domains.py +++ b/dashboard/internet_nl_dashboard/logic/domains.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List, Set, Tuple, Union import pyexcel as p +import requests import tldextract from actstream import action from celery import group @@ -28,6 +29,24 @@ log = logging.getLogger(__package__) +def suggest_subdomains(domain: str, period: int = 370): + extract = tldextract.extract(domain) + + # ip address or garbage + if not extract.domain or not extract.suffix: + return [] + + # call SUBDOMAIN_SUGGESTION_SERVER_ADDRESS + response = requests.get(config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS, + params={'domain': extract.domain, 'suffix': extract.suffix, 'period': period}) + + if response.status_code != 200: + log.error("Failed to retrieve subdomain suggestions from %s.", config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS) + return [] + + return response.json() + + # todo: write test def alter_url_in_urllist(account, data) -> Dict[str, Any]: # data = {'list_id': list.id, 'url_id': url.id, 'new_url_string': url.url} diff --git a/dashboard/internet_nl_dashboard/tests/test_domain_management.py b/dashboard/internet_nl_dashboard/tests/test_domain_management.py index f7326feb..212858c5 100644 --- a/dashboard/internet_nl_dashboard/tests/test_domain_management.py +++ b/dashboard/internet_nl_dashboard/tests/test_domain_management.py @@ -14,8 +14,23 @@ get_urllist_content, get_urllists_from_account, keys_are_present_in_object, rename_list, retrieve_possible_urls_from_unfiltered_input, - save_urllist_content_by_name) + save_urllist_content_by_name, suggest_subdomains) from dashboard.internet_nl_dashboard.models import Account +import responses + + +@responses.activate +def test_suggest_subdomains(db, caplog): + responses.add(responses.GET, 'http://localhost:8001/?domain=test&suffix=nl&period=370', json=["test"], status=200) + responses.add(responses.GET, 'http://localhost:8001/?domain=broken&suffix=nl&period=370', json=[], status=404) + + assert suggest_subdomains("test.nl", 370) == ["test"] + assert suggest_subdomains("broken.nl") == [] + assert "Failed to retrieve" in caplog.text + + # incomplete requests: + assert suggest_subdomains("192.168.1.1") == [] + assert suggest_subdomains("a") == [] @override_config(DASHBOARD_MAXIMUM_DOMAINS_PER_LIST=5000) diff --git a/dashboard/internet_nl_dashboard/urls.py b/dashboard/internet_nl_dashboard/urls.py index 4e17ce0c..b9273b46 100644 --- a/dashboard/internet_nl_dashboard/urls.py +++ b/dashboard/internet_nl_dashboard/urls.py @@ -54,6 +54,7 @@ def to_url(value): path('data/urllist/url/delete/', domains.delete_url_from_urllist_), path('data/urllist/download/', domains.download_list_), path('data/urllist/upload//', spreadsheet.upload_list_), + path('data/urllist/suggest-subdomains/', domains.suggest_subdomains_), path('data/urllist/tag/add/', tags.add_tag_), path('data/urllist/tag/remove/', tags.remove_tag_), diff --git a/dashboard/internet_nl_dashboard/views/app.py b/dashboard/internet_nl_dashboard/views/app.py index 1a1b72a7..72257769 100644 --- a/dashboard/internet_nl_dashboard/views/app.py +++ b/dashboard/internet_nl_dashboard/views/app.py @@ -12,6 +12,17 @@ def config_content(): "show": { "signup_form": configuration["SHOW_SIGNUP_FORM"], }, + "app": { + "subdomain_suggestion": { + "enabled": configuration["SUBDOMAIN_SUGGESTION_ENABLED"], + "default_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_PERIOD"], + "default_extend_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD"], + }, + # in the future we'll support this + "signup": { + "enabled": configuration["SHOW_SIGNUP_FORM"], + } + } } diff --git a/dashboard/internet_nl_dashboard/views/domains.py b/dashboard/internet_nl_dashboard/views/domains.py index 56c6fa28..1d8b1f7e 100644 --- a/dashboard/internet_nl_dashboard/views/domains.py +++ b/dashboard/internet_nl_dashboard/views/domains.py @@ -11,10 +11,17 @@ delete_url_from_urllist, download_as_spreadsheet, get_scan_status_of_list, get_urllist_content, get_urllists_from_account, save_urllist_content_by_name, - scan_now, update_list_settings) + scan_now, suggest_subdomains, update_list_settings) from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_json_body +@login_required(login_url=LOGIN_URL) +def suggest_subdomains_(request) -> JsonResponse: + return JsonResponse( + suggest_subdomains(request.GET.get("domain", ""), request.GET.get("period", 370) + ), encoder=JSEncoder, safe=False) + + @login_required(login_url=LOGIN_URL) def get_lists(request) -> JsonResponse: return JsonResponse(get_urllists_from_account(account=get_account(request)), encoder=JSEncoder, safe=False) diff --git a/dashboard/settings.py b/dashboard/settings.py index 903dc2f3..90335d06 100644 --- a/dashboard/settings.py +++ b/dashboard/settings.py @@ -672,6 +672,28 @@ "Do not send in subdomains. To reduce the number of tests while still getting an impression on a broader scope", bool, ), + + "SUBDOMAIN_SUGGESTION_ENABLED": ( + False, + "Do you want subdomain suggestions to become available in the web interface?", + bool, + ), + "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS": ( + "http://localhost:8001/", + "Server address of the suggestions API. To run this API, go to: " + "https://github.com/internetstandards/Internet.nl-ct-log-subdomain-suggestions-api", + str, + ), + "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD": ( + 120, + "The amount of days the domain has to be last seen", + int, + ), + "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD": ( + 90, + "The amount of days to extend the range to search for available subdomains", + int, + ), } CONSTANCE_CONFIG_FIELDSETS = OrderedDict( @@ -713,6 +735,15 @@ ) ), + ( + "Subdomain suggestions", ( + "SUBDOMAIN_SUGGESTION_ENABLED", + "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS", + "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD", + "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD" + ) + ), + ( 'Signup Settings (internet.nl only)', ( 'SHOW_SIGNUP_FORM', @@ -785,7 +816,7 @@ ]}, {'label': _('🔬 Scan'), 'items': [ - {'name': 'scanners.internetnlscaninspection', 'label': 'Scan Inspections'}, + {'name': 'scanners.internetnlscaninspection', 'label': 'Scan Inspections'}, {'name': 'internet_nl_dashboard.accountinternetnlscan'}, {'name': 'internet_nl_dashboard.accountinternetnlscanlog'}, {'name': 'scanners.internetnlv2scan', 'label': 'Internet.nl Scans Tasks'}, diff --git a/requirements-dev.in b/requirements-dev.in index 460856a0..3fd915e6 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -67,3 +67,6 @@ packaging>=23.1 # using unreleased version from github which has better support for SNI server_hostname which prevents connection # resets on some servers requests_toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0 + +# requests testing +responses diff --git a/requirements-dev.txt b/requirements-dev.txt index 2acb28e1..28c92ea8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -725,7 +725,9 @@ requests-toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd561 # -r requirements-dev.in # websecmap responses==0.25.0 - # via pytest-responses + # via + # -r requirements-dev.in + # pytest-responses retry==0.9.2 # via # -c requirements.txt @@ -968,7 +970,7 @@ zope-event==5.0 # -c requirements.txt # gevent # websecmap -zope.interface==6.0 +zope-interface==6.0 # via # -c requirements.txt # gevent From 449c50147c09080226984035756ee624fcc9fae4 Mon Sep 17 00:00:00 2001 From: stitch1 Date: Wed, 6 Nov 2024 13:37:35 +0100 Subject: [PATCH 2/4] fix lint --- dashboard/internet_nl_dashboard/tests/test_domain_management.py | 2 +- dashboard/internet_nl_dashboard/views/domains.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/internet_nl_dashboard/tests/test_domain_management.py b/dashboard/internet_nl_dashboard/tests/test_domain_management.py index 212858c5..e0e0eb99 100644 --- a/dashboard/internet_nl_dashboard/tests/test_domain_management.py +++ b/dashboard/internet_nl_dashboard/tests/test_domain_management.py @@ -6,6 +6,7 @@ """ import timeit +import responses from constance.test import override_config from websecmap.organizations.models import Url @@ -16,7 +17,6 @@ retrieve_possible_urls_from_unfiltered_input, save_urllist_content_by_name, suggest_subdomains) from dashboard.internet_nl_dashboard.models import Account -import responses @responses.activate diff --git a/dashboard/internet_nl_dashboard/views/domains.py b/dashboard/internet_nl_dashboard/views/domains.py index 1d8b1f7e..63e5dd2f 100644 --- a/dashboard/internet_nl_dashboard/views/domains.py +++ b/dashboard/internet_nl_dashboard/views/domains.py @@ -19,7 +19,7 @@ def suggest_subdomains_(request) -> JsonResponse: return JsonResponse( suggest_subdomains(request.GET.get("domain", ""), request.GET.get("period", 370) - ), encoder=JSEncoder, safe=False) + ), encoder=JSEncoder, safe=False) @login_required(login_url=LOGIN_URL) From fe812a9478b578e7c50d3369ae22dbfbea1c9698 Mon Sep 17 00:00:00 2001 From: stitch1 Date: Fri, 8 Nov 2024 11:41:52 +0100 Subject: [PATCH 3/4] fix config object, cleaner code for suggestions --- dashboard/internet_nl_dashboard/views/app.py | 2 +- dashboard/internet_nl_dashboard/views/domains.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dashboard/internet_nl_dashboard/views/app.py b/dashboard/internet_nl_dashboard/views/app.py index 72257769..b547f5ee 100644 --- a/dashboard/internet_nl_dashboard/views/app.py +++ b/dashboard/internet_nl_dashboard/views/app.py @@ -15,7 +15,7 @@ def config_content(): "app": { "subdomain_suggestion": { "enabled": configuration["SUBDOMAIN_SUGGESTION_ENABLED"], - "default_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_PERIOD"], + "default_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD"], "default_extend_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD"], }, # in the future we'll support this diff --git a/dashboard/internet_nl_dashboard/views/domains.py b/dashboard/internet_nl_dashboard/views/domains.py index 63e5dd2f..14994b38 100644 --- a/dashboard/internet_nl_dashboard/views/domains.py +++ b/dashboard/internet_nl_dashboard/views/domains.py @@ -17,9 +17,10 @@ @login_required(login_url=LOGIN_URL) def suggest_subdomains_(request) -> JsonResponse: - return JsonResponse( - suggest_subdomains(request.GET.get("domain", ""), request.GET.get("period", 370) - ), encoder=JSEncoder, safe=False) + request = get_json_body(request) + domain = request.get("domain", "") + period = request.get("period", 370) + return JsonResponse(suggest_subdomains(domain, period), encoder=JSEncoder, safe=False) @login_required(login_url=LOGIN_URL) From 6892e6a32f9a5c8db809f17b695a2b138a50b090 Mon Sep 17 00:00:00 2001 From: stitch1 Date: Fri, 8 Nov 2024 12:42:06 +0100 Subject: [PATCH 4/4] fix qa issues, clean up settings because of that --- Makefile | 2 +- .../internet_nl_dashboard/logic/domains.py | 3 +- dashboard/settings.py | 503 ++---------------- dashboard/settings_constance.py | 355 ++++++++++++ dashboard/settings_jet.py | 58 ++ pyproject.toml | 12 +- 6 files changed, 453 insertions(+), 480 deletions(-) create mode 100644 dashboard/settings_constance.py create mode 100644 dashboard/settings_jet.py diff --git a/Makefile b/Makefile index 3156db0b..e817690b 100644 --- a/Makefile +++ b/Makefile @@ -253,7 +253,7 @@ vulture: ${app} ## Check for unused code ${python} -m vulture ${pysrcdirs} ruff: ${app} ## Faster than black, might autoformat some things - ${python} -m ruff ${pysrcdirs} + ${python} -m ruff check ${pysrcdirs} bandit: ${app} ## Run basic security audit ${python} -m bandit --configfile bandit.yaml -r ${pysrcdirs} diff --git a/dashboard/internet_nl_dashboard/logic/domains.py b/dashboard/internet_nl_dashboard/logic/domains.py index 3e7e4489..51e0e940 100644 --- a/dashboard/internet_nl_dashboard/logic/domains.py +++ b/dashboard/internet_nl_dashboard/logic/domains.py @@ -38,7 +38,8 @@ def suggest_subdomains(domain: str, period: int = 370): # call SUBDOMAIN_SUGGESTION_SERVER_ADDRESS response = requests.get(config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS, - params={'domain': extract.domain, 'suffix': extract.suffix, 'period': period}) + params={'domain': extract.domain, 'suffix': extract.suffix, 'period': period}, + timeout=10) if response.status_code != 200: log.error("Failed to retrieve subdomain suggestions from %s.", config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS) diff --git a/dashboard/settings.py b/dashboard/settings.py index 90335d06..d6089ec1 100644 --- a/dashboard/settings.py +++ b/dashboard/settings.py @@ -1,16 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 import os -from collections import OrderedDict from datetime import timedelta import sentry_sdk from corsheaders.defaults import default_headers -from django.utils.translation import gettext_lazy as _ from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration from . import __version__ +from .settings_constance import CONSTANCE_CONFIG_FIELDSETS # noqa pylint: disable=unused-import +from .settings_constance import (CONSTANCE_ADDITIONAL_FIELDS, # noqa # pylint: disable=unused-import + CONSTANCE_BACKEND, CONSTANCE_CONFIG) +# import all of this and don't auto-lint it away because there are no references here. +# most code refers to these settings. +from .settings_jet import JET_SIDE_MENU_COMPACT, JET_SIDE_MENU_ITEMS # noqa # pylint: disable=unused-import """ Django settings for dashboard project. @@ -143,9 +147,7 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - BASE_DIR + '/', - ], + 'DIRS': [f'{BASE_DIR}/'], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -157,7 +159,7 @@ 'dashboard.internet_nl_dashboard.context_processors.template_settings_processor', ], }, - }, + } ] WSGI_APPLICATION = 'dashboard.wsgi.application' @@ -175,7 +177,7 @@ 'mysql': 'dashboard.app.backends.mysql', } DATABASES_SETTINGS = { - # persisten local database used during development (runserver) + # persist local database used during development (runserver) 'dev': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.environ.get('DB_NAME', 'db.sqlite3'), @@ -187,13 +189,13 @@ }, # for production get database settings from environment (eg: docker) 'production': { - 'ENGINE': DATABASE_ENGINES.get(DB_ENGINE, 'django.db.backends.' + DB_ENGINE), + 'ENGINE': DATABASE_ENGINES.get(DB_ENGINE, f'django.db.backends.{DB_ENGINE}'), 'NAME': os.environ.get('DB_NAME', 'dashboard'), 'USER': os.environ.get('DB_USER', 'dashboard'), 'PASSWORD': os.environ.get('DB_PASSWORD', 'dashboard'), 'HOST': os.environ.get('DB_HOST', 'mysql'), - 'OPTIONS': DATABASE_OPTIONS.get(os.environ.get('DB_ENGINE', 'mysql'), {}) - } + 'OPTIONS': DATABASE_OPTIONS.get(os.environ.get('DB_ENGINE', 'mysql'), {}), + }, } # allow database to be selected through environment variables DATABASE = os.environ.get('DJANGO_DATABASE', 'dev') @@ -249,8 +251,8 @@ else: STATIC_ROOT = os.environ.get('STATIC_ROOT', '/srv/dashboard/static/') # noqa -MEDIA_ROOT = os.environ.get("MEDIA_ROOT", os.path.abspath(os.path.dirname(__file__)) + "/uploads/") -UPLOAD_ROOT: str = os.environ.get('MEDIA_ROOT', os.path.abspath(os.path.dirname(__file__)) + '/uploads/') +MEDIA_ROOT = os.environ.get("MEDIA_ROOT", f"{os.path.abspath(os.path.dirname(__file__))}/uploads/") +UPLOAD_ROOT: str = os.environ.get('MEDIA_ROOT', f'{os.path.abspath(os.path.dirname(__file__))}/uploads/') # Two factor auth LOGIN_URL = "two_factor:login" @@ -261,6 +263,9 @@ TWO_FACTOR_TOTP_DIGITS = 6 TWO_FACTOR_PATCH_ADMIN = True +# something from websecmap +TOOLS = {'organizations': {'import_data_dir': ''}} + # Encrypted fields # Note that this key is not stored in the database, that would be a security risk. # The key can be generated with the following routine: @@ -419,448 +424,14 @@ if not DEBUG: STATSD_PATCHES = ['django_statsd.patches.db', ] -TOOLS = { - 'organizations': { - 'import_data_dir': '', - }, -} -OUTPUT_DIR = os.environ.get('OUTPUT_DIR', os.path.abspath(os.path.dirname(__file__)) + '/') -VENDOR_DIR = os.environ.get('VENDOR_DIR', os.path.abspath(os.path.dirname(__file__) + '/../vendor/') + '/') +OUTPUT_DIR = os.environ.get('OUTPUT_DIR', f'{os.path.abspath(os.path.dirname(__file__))}/') +VENDOR_DIR = os.environ.get('VENDOR_DIR', f"{os.path.abspath(f'{os.path.dirname(__file__)}/../vendor/')}/") if DEBUG: # too many sql variables.... DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000 -# Constance settings: -CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' - -CONSTANCE_ADDITIONAL_FIELDS = { - "json": ["django.forms.fields.JSONField", {"required": False}], -} - -CONSTANCE_CONFIG = { - # general settings - 'DASHBOARD_FRONTEND_URL': ( - 'https://dashboard.example.com', - 'Url where the frontend is reachable on for end users. This url is references in a few parts of the frontend.', - str - ), - 'DASHBOARD_MAXIMUM_DOMAINS_PER_LIST': ( - 1000, - 'The maximum amount of domains that can be in a list. There will be no crash when somebody imports more ' - 'via a spreadsheet: domains will be added but the list will refuse to scan and show a warning.' - 'In normal use cases these limits will not be reached as average lists are about 300 domains. Lists ' - 'with 600 domains are unusual. Lists with 10.000+ domains are exceptional.', - int - ), - 'DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET': ( - 1000, - 'The maximum amount of domains that can be imported via a spreadsheet at one time. ' - 'In normal use cases these limits will not be reached.', - int - ), - 'DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET': ( - 20, - 'The maximum amount of lists that can be imported via a spreadsheet at one time. ' - 'In normal usec ases these limits will not be reached.', - int - ), - 'DASHBOARD_FRONT_PAGE_URL_LISTS': ( - '', - 'Comma separated list of urllists of which all reports will automatically be shared on the front page. ' - 'For example: 1,2,3. No data means the front page will not show any lists, just the usual information.', - str - ), - - # scan settings - 'SCAN_AT_ALL': ( - True, - 'This enables or disabled all scans. Note that scans that are picked up will still be processed.', - bool - ), - 'INTERNET_NL_API_URL': ( - 'https://batch.example.com/api/batch/v2', - 'The internet address for the Internet.nl API installation. This is commonly called a "batch server".', - str - ), - 'INTERNET_NL_SCAN_TRACKING_NAME': ( - 'Dashboard InternetNL [countrycode]', - 'This setting is used when sending API requests for tracking purposes. Setting this value make it clear who ' - 'is sending API requests. A good setting contains something unique about this installation, such as an ' - 'organization name. The maximum length is 40 characters.', - str - ), - "SCANNER_NAMESERVERS": ( - ["193.17.47.1", "185.43.135.1", "193.110.81.0", "185.253.5.0", "9.9.9.9", "149.112.112.112", - "2001:148f:ffff::1", "2001:148f:fffe::1", "2a0f:fc80::", "2a0f:fc81::", "2620:fe::fe", "2620:fe::9"], - "Nameservers used during scans (dns endpoints and subdomains). This string is loaded as JSON, but not validated" - " due to limitations of this settings library. Be careful when editing(!). " - "This information is cached and loaded only once every 10 minutes.", - "json", - ), - "CREDENTIAL_CHECK_URL": ( - "https://batch.example.com/api/", - "The url where internet.nl api credentials are checked. This is usually the bare INTERNET_NL_API_URL endpoint. " - "This feature is used in the admin interface at account management. " - "There the option 'check credentials' can be performed for each account.", - str - ), - - # email settings - 'EMAIL_NOTIFICATION_SENDER': ( - 'noreply@example.com', - 'The sender of email report notification: this is the e-mail that contains the current scan results and a ' - 'summary. It also compares the result to the previous results. Use an e-mail address that is in use.', - str - ), - 'EMAIL_FALLBACK_LANGUAGE': ( - 'en', - 'Default language used for templates. Template should end with _en in lowercase. Example e-mail templates are ' - 'included and can be found in the menu of the admin interface.', - str - ), - 'EMAIL_TEST_RECIPIENT': ( - 'info@example.com', - 'Which e-mail address receives the testmail from the command "dashboard send_testmail". This command tests if ' - 'the e-mail outbox is properly configured.', - str - ), - 'EMAIL_DASHBOARD_ADDRESS': ( - 'https://example.com', - 'The address of the dashboard, can be set to any url. Available in email template at {{dashboard_address}}. ' - 'This is probably the same as the DASHBOARD_FRONTEND_URL. Only in rare cases this would differ.', - str - ), - - # security.txt - "SECURITY_TXT_IS_REDIRECTED": ( - False, - "Security.txt is used to allow security researchers to report vulnerabilities. This can be either set to a " - "redirect to an existing security.txt or configured with your own security.txt policy.", - bool - ), - "SECURITY_TXT_REDIRECT_URL": ( - "https://example.com/.well-known/security.txt", - "The url where the security.txt files redirect to. This is usually an external site.", - str - ), - "SECURITY_TXT_CONTENT": ( - "", - "The content of the security.txt file, located at .well-known/security.txt. Only " - "used when redirect is disabled. Go to securitytxt.org to create a configuration " - "for this installation.", - str - ), - - # signup settings - "SHOW_SIGNUP_FORM": ( - False, - "Show the signup form on the front page, so visitors of the dashboard can sign up for an account. Currently " - "only internet.nl signup questions are available. So this might not be useful for most installations.", - bool, - ), - 'EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP': ( - 'noreply@example.com', - 'The sender of the "thank you" e-mail after signing up. The template for this e-mail can be found in the ' - 'E-Mail templates menu of the admin interface.', - str - ), - 'DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES': ( - 'support@example.com', - 'Comma separated list of email addresses to notify about new signups. Don\'t add extra spaces in between.', - str - ), - - # timeouts - "SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS": ( - 10000, - 'timeout for phase DISCOVERING_ENDPOINTS', - int - ), - "SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS": ( - 1440, - 'timeout for phase RETRIEVING_SCANABLE_URLS', - int - ), - "SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL": ( - 1440, - 'timeout for phase REGISTERING_SCAN_AT_INTERNET_NL', - int - ), - "SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS": ( - 10000, - 'timeout for phase IMPORTING_SCAN_RESULTS', - int - ), - "SCAN_TIMEOUT_MINUTES_CREATING_REPORT": ( - 10000, - 'timeout for phase CREATING_REPORT', - int - ), - "SCAN_TIMEOUT_MINUTES_SENDING_MAIL": ( - 1440, - 'timeout for phase SENDING_MAIL', - int - ), - "SCAN_TIMEOUT_MINUTES_SERVER_ERROR": ( - 1440, - 'timeout for phase SERVER_ERROR', - int - ), - - # other stuff - 'INTERNET_NL_API_USERNAME': ( - 'dummy', - 'Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep ' - 'this value set to dummy for legacy reasons.', - str), - 'INTERNET_NL_API_PASSWORD': ( - '', - 'Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep ' - 'this value set to dummy for legacy reasons.', - str - ), - 'INTERNET_NL_MAXIMUM_URLS': ( - 1000, - 'The maximum amount of domains per scan, not relevant for dashboard, only for websecmap.', - int - ), - - "SCANNER_LOG_PLANNED_SCANS": ( - False, - "Used when debugging, logs all changes to planned scans to a separate table. Causes millions of records a day", - bool, - ), - "SCANNER_AUTO_PURGE_FINISHED_SCANS": ( - True, - "Removes the scan record from the planned scan table, which reduces the amount of data stored.", - bool, - ), - "CONNECTIVITY_TEST_DOMAIN": ( - "internet.nl", - "A server that is reachable over IPv4. This is used by a worker to determine what kind of scans it can do. " - "Enter an address that you own or manage.", - str, - ), - "IPV6_TEST_DOMAIN": ( - "internet.nl", - "A server that is reachable over IPv6. This is used by a worker to determine " - "what kind of scans it can do. Enter an address that you own or manage.", - str, - ), - "INTERNET_NL_ADD_CALCULATED_RESULTS_WEBSECMAP": ( - False, - "Add calculated results for web security map. This is used only for installations by the " - "Internet Cleanup Foundation.", - bool, - ), - "INTERNET_NL_ADD_CALCULATED_RESULTS_FORUM_STANDAARDISATIE": ( - False, - "Add calculated results for forum standaardisatie, the internet.nl dashboard. These calculations are created " - "on top of the internet.nl metrics. These are used for official publications. You probably do not need these.", - bool, - ), - "INTERNET_NL_ADD_CALCULATED_RESULTS_VNG_V6": ( - False, - "Add calculated results for VNG, obsoleted IPv6 derived conclusions. No need to enable these and will be " - "removed in a future release.", - bool, - ), - "INTERNET_NL_WEB_ONLY_TOP_LEVEL": ( - False, - "Do not send in subdomains. To reduce the number of tests while still getting an impression on a broader scope", - bool, - ), - - "SUBDOMAIN_SUGGESTION_ENABLED": ( - False, - "Do you want subdomain suggestions to become available in the web interface?", - bool, - ), - "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS": ( - "http://localhost:8001/", - "Server address of the suggestions API. To run this API, go to: " - "https://github.com/internetstandards/Internet.nl-ct-log-subdomain-suggestions-api", - str, - ), - "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD": ( - 120, - "The amount of days the domain has to be last seen", - int, - ), - "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD": ( - 90, - "The amount of days to extend the range to search for available subdomains", - int, - ), -} - -CONSTANCE_CONFIG_FIELDSETS = OrderedDict( - [ - ( - 'General Dashboard Settings', ( - 'DASHBOARD_FRONTEND_URL', - 'DASHBOARD_MAXIMUM_DOMAINS_PER_LIST', - 'DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET', - 'DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET', - 'DASHBOARD_FRONT_PAGE_URL_LISTS' - ) - ), - - ( - 'Internet.nl Scan Settings', ( - 'SCAN_AT_ALL', - 'INTERNET_NL_API_URL', - "INTERNET_NL_SCAN_TRACKING_NAME", - "SCANNER_NAMESERVERS", - "CREDENTIAL_CHECK_URL", - ) - ), - - ( - 'E-Mail Settings', ( - 'EMAIL_NOTIFICATION_SENDER', - 'EMAIL_FALLBACK_LANGUAGE', - 'EMAIL_TEST_RECIPIENT', - 'EMAIL_DASHBOARD_ADDRESS', - ), - ), - - ( - "Security.txt", ( - "SECURITY_TXT_IS_REDIRECTED", - "SECURITY_TXT_REDIRECT_URL", - "SECURITY_TXT_CONTENT" - ) - ), - - ( - "Subdomain suggestions", ( - "SUBDOMAIN_SUGGESTION_ENABLED", - "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS", - "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD", - "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD" - ) - ), - - ( - 'Signup Settings (internet.nl only)', ( - 'SHOW_SIGNUP_FORM', - 'EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP', - 'DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES' - ) - ), - - ( - "Timeouts (advanced)", ( - 'SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS', - 'SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS', - 'SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL', - 'SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS', - 'SCAN_TIMEOUT_MINUTES_CREATING_REPORT', - 'SCAN_TIMEOUT_MINUTES_SENDING_MAIL', - 'SCAN_TIMEOUT_MINUTES_SERVER_ERROR', - ) - ), - - ( - "Logging settings (advanced)", ( - "SCANNER_LOG_PLANNED_SCANS", - "SCANNER_AUTO_PURGE_FINISHED_SCANS", - ) - ), - - ( - "Unused / Expert settings", ( - 'INTERNET_NL_API_USERNAME', - 'INTERNET_NL_API_PASSWORD', - 'INTERNET_NL_MAXIMUM_URLS', - "INTERNET_NL_ADD_CALCULATED_RESULTS_WEBSECMAP", - "INTERNET_NL_ADD_CALCULATED_RESULTS_FORUM_STANDAARDISATIE", - "INTERNET_NL_ADD_CALCULATED_RESULTS_VNG_V6", - "INTERNET_NL_WEB_ONLY_TOP_LEVEL", - "IPV6_TEST_DOMAIN", - "CONNECTIVITY_TEST_DOMAIN" - ) - ) - ] -) - -# the try-except makes sure autofix doesn't move the import to the top of the file. -# Loaded here, otherwise: django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet. -try: - from websecmap.scanners.constance import add_scanner_fields, add_scanner_fieldsets # NOQA - - CONSTANCE_CONFIG = add_scanner_fields(CONSTANCE_CONFIG) - CONSTANCE_CONFIG_FIELDSETS = add_scanner_fieldsets(CONSTANCE_CONFIG_FIELDSETS) -except ImportError: - pass - -JET_SIDE_MENU_ITEMS = [ - - {'label': '', 'items': [ - {'name': 'constance.config', 'label': '🎛️ Dashboard Configuration'}, - {'name': 'django_mail_admin.emailtemplate', 'label': '📨 E-Mail Templates'}, - {'name': 'django_mail_admin.outbox', 'label': '📨 Outboxes'}, - {'name': 'django_celery_beat.periodictask', 'label': '⏰ Periodic Tasks'}, - {'name': 'auth.user', 'label': '👤 Users'}, - {'name': 'internet_nl_dashboard.account', 'label': '🏢 Accounts'}, - {'name': 'otp_totp.totpdevice', 'label': '📱 TOTP Devices'}, - ]}, - - {'label': _('📘 Dashboard'), 'items': [ - {'name': 'internet_nl_dashboard.urllist', 'label': "Domain lists"}, - {'name': 'internet_nl_dashboard.taggedurlinurllist', 'label': 'Tagged Url'}, - {'name': 'internet_nl_dashboard.uploadlog', 'label': 'Uploads'}, - ]}, - - {'label': _('🔬 Scan'), 'items': [ - {'name': 'scanners.internetnlscaninspection', 'label': 'Scan Inspections'}, - {'name': 'internet_nl_dashboard.accountinternetnlscan'}, - {'name': 'internet_nl_dashboard.accountinternetnlscanlog'}, - {'name': 'scanners.internetnlv2scan', 'label': 'Internet.nl Scans Tasks'}, - {'name': 'scanners.internetnlv2statelog', 'label': 'Internet.nl Scans Log'}, - {'name': 'internet_nl_dashboard.subdomaindiscoveryscan', 'label': 'Subdomain Discovery'} - ]}, - - {'label': _('💽 Data'), 'items': [ - {'name': 'organizations.url', 'label': 'Urls'}, - {'name': 'scanners.endpoint', 'label': 'Endpoints'}, - {'name': 'scanners.endpointgenericscan', 'label': 'Endpoint Scans'}, - ]}, - - {'label': _('📊 Report'), 'items': [ - {'name': 'reporting.urlreport', 'label': 'Url Reports'}, - {'name': 'internet_nl_dashboard.urllistreport', 'label': 'Full Reports'} - ]}, - - {'label': _('🕒 Periodic Tasks'), 'items': [ - {'name': 'django_celery_beat.periodictask'}, - {'name': 'django_celery_beat.crontabschedule'}, - ]}, - - {'label': _('📨 E-Mail'), 'items': [ - {'name': 'django_mail_admin.emailtemplate', 'label': 'Templates'}, - {'name': 'django_mail_admin.outgoingemail', 'label': 'Sent mail'}, - {'name': 'django_mail_admin.outbox', 'label': 'Outboxes'}, - {'name': 'django_mail_admin.log', 'label': 'Logs'}, - ]}, - - {'label': _('✨ Activity'), 'items': [ - {'name': 'actstream.action'}, - ]}, -] - -JET_SIDE_MENU_COMPACT = True - -# Allows to see all details of websecmap. -# Loaded here, otherwise: django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet. -# try: -# from websecmap.jet import websecmap_menu_items # NOQA -# JET_SIDE_MENU_ITEMS += websecmap_menu_items() -# except ImportError: -# pass # Security options if not DEBUG: @@ -869,8 +440,7 @@ SESSION_COOKIE_AGE = 1209600 # two weeks, could be longer CSRF_COOKIE_SECURE = True # insecure by default -SENTRY_DSN = os.environ.get('SENTRY_DSN') -if SENTRY_DSN: +if SENTRY_DSN := os.environ.get('SENTRY_DSN'): # new sentry_sdk implementation, with hopes to also get exceptions from workers. sentry_sdk.init( # pylint: disable=abstract-class-instantiated # (following the documentation) # type: ignore dsn=SENTRY_DSN, @@ -894,25 +464,14 @@ # email settings... # django_mail_admin.backends.OutboxEmailBackend = Store sent mails in outbox, so we know what has been sent. # It's not a log -> this is just a way to test things, and it will hang send_queued_mail. -if DEBUG: - EMAIL_BACKEND = 'django_mail_admin.backends.CustomEmailBackend' - # As there are sanity checks, these settings need to be present during debugging too. - EMAIL_HOST = '' - EMAIL_PORT = '' - EMAIL_HOST_USER = '' - EMAIL_HOST_PASSWORD = '' # nosec - EMAIL_USE_TLS = False - EMAIL_USE_SSL = False -else: - EMAIL_BACKEND = 'django_mail_admin.backends.CustomEmailBackend' - # todo: get these settings from internet.nl - EMAIL_HOST = '' - EMAIL_PORT = '' - EMAIL_HOST_USER = '' - EMAIL_HOST_PASSWORD = '' # nosec - EMAIL_USE_TLS = False - EMAIL_USE_SSL = False - +EMAIL_USE_SSL = False +EMAIL_USE_TLS = False +EMAIL_HOST_PASSWORD = '' # nosec +EMAIL_HOST_USER = '' +EMAIL_PORT = '' +# As there are sanity checks, these settings need to be present during debugging too. +EMAIL_HOST = '' +EMAIL_BACKEND = 'django_mail_admin.backends.CustomEmailBackend' if DEBUG: # 25 megs for importing reports from live situations DATA_UPLOAD_MAX_MEMORY_SIZE = 26214400 @@ -948,7 +507,7 @@ # allow cookies to be sent as well, we have to, because there are logins and such. CORS_ALLOW_CREDENTIALS = True -LOCKFILE_DIR = os.environ.get('LOCKFILE_DIR', os.path.abspath(os.path.dirname(__file__)) + '/lockfiles/') +LOCKFILE_DIR = os.environ.get('LOCKFILE_DIR', f'{os.path.abspath(os.path.dirname(__file__))}/lockfiles/') TAGGIT_CASE_INSENSITIVE = True @@ -980,11 +539,11 @@ os.environ.get("CSRF_TRUSTED_ORIGINS_WILDCARD_DOMAIN", 'https://*.internet.nl') ] -REPORT_STORAGE_DIR = os.environ.get("REPORT_STORAGE_DIR", MEDIA_ROOT + "diskreports/") +REPORT_STORAGE_DIR = os.environ.get("REPORT_STORAGE_DIR", f"{MEDIA_ROOT}diskreports/") # todo: should be handled better, this is a fix to know for sure reports can be written to disk... # check diskreport dir, todo: move to django file field -full_report_storage_dir = REPORT_STORAGE_DIR + "original/UrlListReport/" +full_report_storage_dir = f"{REPORT_STORAGE_DIR}original/UrlListReport/" if not os.path.isdir(full_report_storage_dir): os.makedirs(full_report_storage_dir, exist_ok=True) diff --git a/dashboard/settings_constance.py b/dashboard/settings_constance.py new file mode 100644 index 00000000..253549c7 --- /dev/null +++ b/dashboard/settings_constance.py @@ -0,0 +1,355 @@ +from collections import OrderedDict + +CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' + +CONSTANCE_ADDITIONAL_FIELDS = { + "json": ["django.forms.fields.JSONField", {"required": False}], +} + +CONSTANCE_CONFIG = { + # general settings + 'DASHBOARD_FRONTEND_URL': ( + 'https://dashboard.example.com', + 'Url where the frontend is reachable on for end users. This url is references in a few parts of the frontend.', + str + ), + 'DASHBOARD_MAXIMUM_DOMAINS_PER_LIST': ( + 1000, + 'The maximum amount of domains that can be in a list. There will be no crash when somebody imports more ' + 'via a spreadsheet: domains will be added but the list will refuse to scan and show a warning.' + 'In normal use cases these limits will not be reached as average lists are about 300 domains. Lists ' + 'with 600 domains are unusual. Lists with 10.000+ domains are exceptional.', + int + ), + 'DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET': ( + 1000, + 'The maximum amount of domains that can be imported via a spreadsheet at one time. ' + 'In normal use cases these limits will not be reached.', + int + ), + 'DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET': ( + 20, + 'The maximum amount of lists that can be imported via a spreadsheet at one time. ' + 'In normal usec ases these limits will not be reached.', + int + ), + 'DASHBOARD_FRONT_PAGE_URL_LISTS': ( + '', + 'Comma separated list of urllists of which all reports will automatically be shared on the front page. ' + 'For example: 1,2,3. No data means the front page will not show any lists, just the usual information.', + str + ), + + # scan settings + 'SCAN_AT_ALL': ( + True, + 'This enables or disabled all scans. Note that scans that are picked up will still be processed.', + bool + ), + 'INTERNET_NL_API_URL': ( + 'https://batch.example.com/api/batch/v2', + 'The internet address for the Internet.nl API installation. This is commonly called a "batch server".', + str + ), + 'INTERNET_NL_SCAN_TRACKING_NAME': ( + 'Dashboard InternetNL [countrycode]', + 'This setting is used when sending API requests for tracking purposes. Setting this value make it clear who ' + 'is sending API requests. A good setting contains something unique about this installation, such as an ' + 'organization name. The maximum length is 40 characters.', + str + ), + "SCANNER_NAMESERVERS": ( + ["193.17.47.1", "185.43.135.1", "193.110.81.0", "185.253.5.0", "9.9.9.9", "149.112.112.112", + "2001:148f:ffff::1", "2001:148f:fffe::1", "2a0f:fc80::", "2a0f:fc81::", "2620:fe::fe", "2620:fe::9"], + "Nameservers used during scans (dns endpoints and subdomains). This string is loaded as JSON, but not validated" + " due to limitations of this settings library. Be careful when editing(!). " + "This information is cached and loaded only once every 10 minutes.", + "json", + ), + "CREDENTIAL_CHECK_URL": ( + "https://batch.example.com/api/", + "The url where internet.nl api credentials are checked. This is usually the bare INTERNET_NL_API_URL endpoint. " + "This feature is used in the admin interface at account management. " + "There the option 'check credentials' can be performed for each account.", + str + ), + + # email settings + 'EMAIL_NOTIFICATION_SENDER': ( + 'noreply@example.com', + 'The sender of email report notification: this is the e-mail that contains the current scan results and a ' + 'summary. It also compares the result to the previous results. Use an e-mail address that is in use.', + str + ), + 'EMAIL_FALLBACK_LANGUAGE': ( + 'en', + 'Default language used for templates. Template should end with _en in lowercase. Example e-mail templates are ' + 'included and can be found in the menu of the admin interface.', + str + ), + 'EMAIL_TEST_RECIPIENT': ( + 'info@example.com', + 'Which e-mail address receives the testmail from the command "dashboard send_testmail". This command tests if ' + 'the e-mail outbox is properly configured.', + str + ), + 'EMAIL_DASHBOARD_ADDRESS': ( + 'https://example.com', + 'The address of the dashboard, can be set to any url. Available in email template at {{dashboard_address}}. ' + 'This is probably the same as the DASHBOARD_FRONTEND_URL. Only in rare cases this would differ.', + str + ), + + # security.txt + "SECURITY_TXT_IS_REDIRECTED": ( + False, + "Security.txt is used to allow security researchers to report vulnerabilities. This can be either set to a " + "redirect to an existing security.txt or configured with your own security.txt policy.", + bool + ), + "SECURITY_TXT_REDIRECT_URL": ( + "https://example.com/.well-known/security.txt", + "The url where the security.txt files redirect to. This is usually an external site.", + str + ), + "SECURITY_TXT_CONTENT": ( + "", + "The content of the security.txt file, located at .well-known/security.txt. Only " + "used when redirect is disabled. Go to securitytxt.org to create a configuration " + "for this installation.", + str + ), + + # signup settings + "SHOW_SIGNUP_FORM": ( + False, + "Show the signup form on the front page, so visitors of the dashboard can sign up for an account. Currently " + "only internet.nl signup questions are available. So this might not be useful for most installations.", + bool, + ), + 'EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP': ( + 'noreply@example.com', + 'The sender of the "thank you" e-mail after signing up. The template for this e-mail can be found in the ' + 'E-Mail templates menu of the admin interface.', + str + ), + 'DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES': ( + 'support@example.com', + 'Comma separated list of email addresses to notify about new signups. Don\'t add extra spaces in between.', + str + ), + + # timeouts + "SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS": ( + 10000, + 'timeout for phase DISCOVERING_ENDPOINTS', + int + ), + "SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS": ( + 1440, + 'timeout for phase RETRIEVING_SCANABLE_URLS', + int + ), + "SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL": ( + 1440, + 'timeout for phase REGISTERING_SCAN_AT_INTERNET_NL', + int + ), + "SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS": ( + 10000, + 'timeout for phase IMPORTING_SCAN_RESULTS', + int + ), + "SCAN_TIMEOUT_MINUTES_CREATING_REPORT": ( + 10000, + 'timeout for phase CREATING_REPORT', + int + ), + "SCAN_TIMEOUT_MINUTES_SENDING_MAIL": ( + 1440, + 'timeout for phase SENDING_MAIL', + int + ), + "SCAN_TIMEOUT_MINUTES_SERVER_ERROR": ( + 1440, + 'timeout for phase SERVER_ERROR', + int + ), + + # other stuff + 'INTERNET_NL_API_USERNAME': ( + 'dummy', + 'Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep ' + 'this value set to dummy for legacy reasons.', + str), + 'INTERNET_NL_API_PASSWORD': ( + '', + 'Username for the internet.nl API. This option is ignored as every account uses their own credentials. Keep ' + 'this value set to dummy for legacy reasons.', + str + ), + 'INTERNET_NL_MAXIMUM_URLS': ( + 1000, + 'The maximum amount of domains per scan, not relevant for dashboard, only for websecmap.', + int + ), + + "SCANNER_LOG_PLANNED_SCANS": ( + False, + "Used when debugging, logs all changes to planned scans to a separate table. Causes millions of records a day", + bool, + ), + "SCANNER_AUTO_PURGE_FINISHED_SCANS": ( + True, + "Removes the scan record from the planned scan table, which reduces the amount of data stored.", + bool, + ), + "CONNECTIVITY_TEST_DOMAIN": ( + "internet.nl", + "A server that is reachable over IPv4. This is used by a worker to determine what kind of scans it can do. " + "Enter an address that you own or manage.", + str, + ), + "IPV6_TEST_DOMAIN": ( + "internet.nl", + "A server that is reachable over IPv6. This is used by a worker to determine " + "what kind of scans it can do. Enter an address that you own or manage.", + str, + ), + "INTERNET_NL_ADD_CALCULATED_RESULTS_WEBSECMAP": ( + False, + "Add calculated results for web security map. This is used only for installations by the " + "Internet Cleanup Foundation.", + bool, + ), + "INTERNET_NL_ADD_CALCULATED_RESULTS_FORUM_STANDAARDISATIE": ( + False, + "Add calculated results for forum standaardisatie, the internet.nl dashboard. These calculations are created " + "on top of the internet.nl metrics. These are used for official publications. You probably do not need these.", + bool, + ), + "INTERNET_NL_ADD_CALCULATED_RESULTS_VNG_V6": ( + False, + "Add calculated results for VNG, obsoleted IPv6 derived conclusions. No need to enable these and will be " + "removed in a future release.", + bool, + ), + "INTERNET_NL_WEB_ONLY_TOP_LEVEL": ( + False, + "Do not send in subdomains. To reduce the number of tests while still getting an impression on a broader scope", + bool, + ), + + "SUBDOMAIN_SUGGESTION_ENABLED": ( + False, + "Do you want subdomain suggestions to become available in the web interface?", + bool, + ), + "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS": ( + "http://localhost:8001/", + "Server address of the suggestions API. To run this API, go to: " + "https://github.com/internetstandards/Internet.nl-ct-log-subdomain-suggestions-api", + str, + ), + "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD": ( + 120, + "The amount of days the domain has to be last seen", + int, + ), + "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD": ( + 90, + "The amount of days to extend the range to search for available subdomains", + int, + ), +} + +CONSTANCE_CONFIG_FIELDSETS = OrderedDict( + [ + ( + 'General Dashboard Settings', ( + 'DASHBOARD_FRONTEND_URL', + 'DASHBOARD_MAXIMUM_DOMAINS_PER_LIST', + 'DASHBOARD_MAXIMUM_DOMAINS_PER_SPREADSHEET', + 'DASHBOARD_MAXIMUM_LISTS_PER_SPREADSHEET', + 'DASHBOARD_FRONT_PAGE_URL_LISTS' + ) + ), + + ( + 'Internet.nl Scan Settings', ( + 'SCAN_AT_ALL', + 'INTERNET_NL_API_URL', + "INTERNET_NL_SCAN_TRACKING_NAME", + "SCANNER_NAMESERVERS", + "CREDENTIAL_CHECK_URL", + ) + ), + + ( + 'E-Mail Settings', ( + 'EMAIL_NOTIFICATION_SENDER', + 'EMAIL_FALLBACK_LANGUAGE', + 'EMAIL_TEST_RECIPIENT', + 'EMAIL_DASHBOARD_ADDRESS', + ), + ), + + ( + "Security.txt", ( + "SECURITY_TXT_IS_REDIRECTED", + "SECURITY_TXT_REDIRECT_URL", + "SECURITY_TXT_CONTENT" + ) + ), + + ( + "Subdomain suggestions", ( + "SUBDOMAIN_SUGGESTION_ENABLED", + "SUBDOMAIN_SUGGESTION_SERVER_ADDRESS", + "SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD", + "SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD" + ) + ), + + ( + 'Signup Settings (internet.nl only)', ( + 'SHOW_SIGNUP_FORM', + 'EMAIL_NOTIFICATION_SENDER_FOR_SIGNUP', + 'DASHBOARD_SIGNUP_NOTIFICATION_EMAIL_ADRESSES' + ) + ), + + ( + "Timeouts (advanced)", ( + 'SCAN_TIMEOUT_MINUTES_DISCOVERING_ENDPOINTS', + 'SCAN_TIMEOUT_MINUTES_RETRIEVING_SCANABLE_URLS', + 'SCAN_TIMEOUT_MINUTES_REGISTERING_SCAN_AT_INTERNET_NL', + 'SCAN_TIMEOUT_MINUTES_IMPORTING_SCAN_RESULTS', + 'SCAN_TIMEOUT_MINUTES_CREATING_REPORT', + 'SCAN_TIMEOUT_MINUTES_SENDING_MAIL', + 'SCAN_TIMEOUT_MINUTES_SERVER_ERROR', + ) + ), + + ( + "Logging settings (advanced)", ( + "SCANNER_LOG_PLANNED_SCANS", + "SCANNER_AUTO_PURGE_FINISHED_SCANS", + ) + ), + + ( + "Unused / Expert settings", ( + 'INTERNET_NL_API_USERNAME', + 'INTERNET_NL_API_PASSWORD', + 'INTERNET_NL_MAXIMUM_URLS', + "INTERNET_NL_ADD_CALCULATED_RESULTS_WEBSECMAP", + "INTERNET_NL_ADD_CALCULATED_RESULTS_FORUM_STANDAARDISATIE", + "INTERNET_NL_ADD_CALCULATED_RESULTS_VNG_V6", + "INTERNET_NL_WEB_ONLY_TOP_LEVEL", + "IPV6_TEST_DOMAIN", + "CONNECTIVITY_TEST_DOMAIN" + ) + ) + ] +) diff --git a/dashboard/settings_jet.py b/dashboard/settings_jet.py new file mode 100644 index 00000000..c7548403 --- /dev/null +++ b/dashboard/settings_jet.py @@ -0,0 +1,58 @@ +from django.utils.translation import gettext_lazy as _ + +JET_SIDE_MENU_ITEMS = [ + + {'label': '', 'items': [ + {'name': 'constance.config', 'label': '🎛️ Dashboard Configuration'}, + {'name': 'django_mail_admin.emailtemplate', 'label': '📨 E-Mail Templates'}, + {'name': 'django_mail_admin.outbox', 'label': '📨 Outboxes'}, + {'name': 'django_celery_beat.periodictask', 'label': '⏰ Periodic Tasks'}, + {'name': 'auth.user', 'label': '👤 Users'}, + {'name': 'internet_nl_dashboard.account', 'label': '🏢 Accounts'}, + {'name': 'otp_totp.totpdevice', 'label': '📱 TOTP Devices'}, + ]}, + + {'label': _('📘 Dashboard'), 'items': [ + {'name': 'internet_nl_dashboard.urllist', 'label': "Domain lists"}, + {'name': 'internet_nl_dashboard.taggedurlinurllist', 'label': 'Tagged Url'}, + {'name': 'internet_nl_dashboard.uploadlog', 'label': 'Uploads'}, + ]}, + + {'label': _('🔬 Scan'), 'items': [ + {'name': 'scanners.internetnlscaninspection', 'label': 'Scan Inspections'}, + {'name': 'internet_nl_dashboard.accountinternetnlscan'}, + {'name': 'internet_nl_dashboard.accountinternetnlscanlog'}, + {'name': 'scanners.internetnlv2scan', 'label': 'Internet.nl Scans Tasks'}, + {'name': 'scanners.internetnlv2statelog', 'label': 'Internet.nl Scans Log'}, + {'name': 'internet_nl_dashboard.subdomaindiscoveryscan', 'label': 'Subdomain Discovery'} + ]}, + + {'label': _('💽 Data'), 'items': [ + {'name': 'organizations.url', 'label': 'Urls'}, + {'name': 'scanners.endpoint', 'label': 'Endpoints'}, + {'name': 'scanners.endpointgenericscan', 'label': 'Endpoint Scans'}, + ]}, + + {'label': _('📊 Report'), 'items': [ + {'name': 'reporting.urlreport', 'label': 'Url Reports'}, + {'name': 'internet_nl_dashboard.urllistreport', 'label': 'Full Reports'} + ]}, + + {'label': _('🕒 Periodic Tasks'), 'items': [ + {'name': 'django_celery_beat.periodictask'}, + {'name': 'django_celery_beat.crontabschedule'}, + ]}, + + {'label': _('📨 E-Mail'), 'items': [ + {'name': 'django_mail_admin.emailtemplate', 'label': 'Templates'}, + {'name': 'django_mail_admin.outgoingemail', 'label': 'Sent mail'}, + {'name': 'django_mail_admin.outbox', 'label': 'Outboxes'}, + {'name': 'django_mail_admin.log', 'label': 'Logs'}, + ]}, + + {'label': _('✨ Activity'), 'items': [ + {'name': 'actstream.action'}, + ]}, +] + +JET_SIDE_MENU_COMPACT = True diff --git a/pyproject.toml b/pyproject.toml index dffc71fe..f519baed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,12 +30,12 @@ max_line_length = 120 [tool.ruff] # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. -select = ["E", "F"] -ignore = [] +lint.select = ["E", "F"] +lint.ignore = [] # Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] -unfixable = [] +lint.fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +lint.unfixable = [] # Exclude a variety of commonly ignored directories. exclude = [ @@ -66,11 +66,11 @@ exclude = [ line-length = 120 # Allow unused variables when underscore-prefixed. -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" # Assume Python 3.10. target-version = "py310" -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] # Unlike Flake8, default to a complexity level of 10. max-complexity = 10