Merge pull request #512 from internetstandards/50

4.4.0

Makefile

@@ -172,7 +172,7 @@ app: ${app}  ## perform arbitrary app commands
 	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} ${cmd}
 
 run-worker: ${app}  ## only run worker component
-	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} celery worker -ldebug -Q storage,celery,reporting,ipv4,ipv6,4and6,internet,isolated
+	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} celery worker -ldebug -Q storage,celery,reporting,ipv4,ipv6,4and6,internet,isolated,database,kickoff,default,database_deprecate
 
 run-broker:  ## only run broker
 	docker run --rm --name=redis -p 6379:6379 redis
@@ -223,7 +223,7 @@ clean:  ## cleanup build artifacts, caches, databases, etc.
 	# remove build artifacts
 	-rm -rf *.egg-info dist/ pip-wheel-metadata/
 	# remove runtime state files
-	-rm -rf *.sqlite3
+	# -rm -rf *.sqlite3
 
 clean_virtualenv:  ## cleanup virtualenv and installed app/dependencies
 	# remove virtualenv

README.md

@@ -301,3 +301,18 @@ and run: `arch -x86_64 sh`, `cd /usr/local/bin`, `/brew install --build-from-sou
 
 and run: `arch -x86_64 brew install libmagic`
 You'll get an error but at least there is now an x64/intel file at: `/usr/local/lib/libmagic.dylib`
+
+
+# issues  with celery < 5in
+pkg_resources.extern.packaging.requirements.InvalidRequirement: Expected closing RIGHT_PARENTHESIS
+    pytz (>dev)
+
+add the following line:
+`requirement_string = requirement_string.replace(">dev", "")`
+just before
+`parsed = parse_requirement(requirement_string)`
+in
+`/site-packages/pkg_resources/_vendor/packaging/requirements.py`
+Probably line 36...
+
+`/Users/stitch/Library/Caches/virtualenvs/internet.nl-dashboard/lib/python3.10/site-packages/pkg_resources/_vendor/packaging/`

dashboard/internet_nl_dashboard/scanners/scan_internet_nl_per_account.py

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
+import json
 import logging
 from copy import copy
 from datetime import datetime, timedelta, timezone
@@ -344,7 +345,25 @@ def registering_scan_at_internet_nl(scan_id: int):
     internet_nl_websecmap.update_state(
         scan.scan.id, "requested", "requested a scan to be performed on internet.nl api")
 
-    return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id)
+    # use our own tracking information for this scan, based on #451
+    # max length = 255.
+    tracking_info = {
+        # add the configured tracking name:
+        'origin': constance_cached_value("INTERNET_NL_SCAN_TRACKING_NAME")[:40],  # + 6
+        # lists:
+        'list': {  # + 4
+            'id': scan.urllist.id,  # 6 chars + 2
+            'name': scan.urllist.name[:80],  # 60 + 4
+            'interval': scan.urllist.automated_scan_frequency,  # 30 chars + 8
+            'account': {  # + 6
+                'id': scan.urllist.account.id,  # 6 chars + 2
+                'name': scan.urllist.account.name[:50],  # 40 + 4
+            }
+        },
+    }
+
+    # total json overhead is about 40 characters.
+    return chain(internet_nl_websecmap.progress_running_scan(scan.scan.id, tracking_info=json.dumps(tracking_info))
                  | copy_state_from_websecmap_scan.si(scan.id))
 
 

dashboard/settings.py

@@ -48,7 +48,8 @@
 INSTALLED_APPS = [
     # Constance
     'constance',
-    'constance.backends.database',
+    # not needed since 3.0
+    # 'constance.backends.database',
 
     # Jet
     'jet.dashboard',
@@ -487,7 +488,6 @@
         'Comma separated list of urllists of which all reports will be shared on the front page',
         str
     ),
-
     'INTERNET_NL_API_USERNAME': (
         'dummy',
         'Username for the internet.nl API. You can request one via the contact '
@@ -502,6 +502,21 @@
                             'The internet address for the Internet.nl API installation. Defaults to a version from '
                             '2020.', str),
     'INTERNET_NL_MAXIMUM_URLS': (1000, 'The maximum amount of domains per scan.', int),
+    'INTERNET_NL_SCAN_TRACKING_NAME': (
+        'Dashboard Internet NL',
+        'What dashboard installation is sending API requests?',
+        str
+    ),
+    "SCANNER_LOG_PLANNED_SCANS": (
+        True,
+        "Used when debugging, logs all changes to planned scans to a separate table. Causes millions of records a day",
+        bool,
+    ),
+    "SCANNER_AUTO_PURGE_FINISHED_SCANS": (
+        False,
+        "Removes the scan record from the planned scan table, which reduces the amount of data stored.",
+        bool,
+    ),
     'EMAIL_FALLBACK_LANGUAGE': (
         'en',
         'Default language used for templates. Template should end with _en in lowercase.',
@@ -609,6 +624,8 @@
                                "INTERNET_NL_ADD_CALCULATED_RESULTS_FORUM_STANDAARDISATIE",
                                "INTERNET_NL_ADD_CALCULATED_RESULTS_VNG_V6",
                                "INTERNET_NL_WEB_ONLY_TOP_LEVEL",
+                               "INTERNET_NL_SCAN_TRACKING_NAME",
+                               "SCANNER_LOG_PLANNED_SCANS", "SCANNER_AUTO_PURGE_FINISHED_SCANS"
                                )),
         ("Scanning preferences", ("SCANNER_NAMESERVERS",)),
         (
@@ -789,6 +806,7 @@
 # Django 3.2
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
+MEDIA_URL = os.environ.get("MEDIA_URL", "/admin/uploads/")
 
 # New in django 4.2:
 STORAGES = {
@@ -798,6 +816,7 @@
     "staticfiles": {
         "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
     },
+    "paper_trail": {"BACKEND": "django.core.files.storage.FileSystemStorage", "OPTIONS": {"base_url": MEDIA_URL}},
 }
 
 # required from django 4.0

requirements-deploy.in

@@ -3,7 +3,7 @@
 # make sure there are no version conflicts with requirements.txt
 -c requirements.txt
 
-websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@d60d90d787d94de122dfe139df4790b82eee7554
+websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@6e360ed6c3d3205a4c2caddb04d946e28420fed4
 
 # include (security) version constraints for non primary dependencies
 -c security-constraints.in

requirements-deploy.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements-deploy.txt requirements-deploy.in
 #
-aiohttp==3.8.6
+aiohttp==3.9.5
     # via
     #   -c requirements.txt
     #   geoip2
@@ -97,7 +97,7 @@ colorlog==6.7.0
     # via
     #   -c requirements.txt
     #   websecmap
-cryptography==41.0.5
+cryptography==41.0.7
     # via
     #   -c requirements.txt
     #   -c security-constraints.in
@@ -123,7 +123,7 @@ diff-match-patch==20230430
     #   -c requirements.txt
     #   django-import-export
     #   websecmap
-django==4.2.7
+django==4.2.13
     # via
     #   -c requirements.txt
     #   -c security-constraints.in
@@ -149,7 +149,7 @@ django-colorful==1.3
     # via
     #   -c requirements.txt
     #   websecmap
-django-constance[database]==2.9.1
+django-constance==3.1.0
     # via
     #   -c requirements.txt
     #   websecmap
@@ -569,7 +569,7 @@ tldextract==3.4.4
     # via
     #   -c requirements.txt
     #   websecmap
-tornado==6.3.3
+tornado==6.4.0
     # via
     #   -c requirements.txt
     #   flower
@@ -607,7 +607,7 @@ vine==1.3.0
     #   celery
     #   flower
     #   websecmap
-websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@d60d90d787d94de122dfe139df4790b82eee7554
+websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@6e360ed6c3d3205a4c2caddb04d946e28420fed4
     # via
     #   -c requirements.txt
     #   -r requirements-deploy.in
@@ -632,12 +632,12 @@ yarl==1.9.2
     #   -c requirements.txt
     #   aiohttp
     #   websecmap
-zope-event==5.0
+zope.event==5.0
     # via
     #   -c requirements.txt
     #   gevent
     #   websecmap
-zope-interface==6.0
+zope.interface==6.0
     # via
     #   -c requirements.txt
     #   gevent

requirements-dev.in

@@ -18,7 +18,7 @@ django-extensions
 django-debug-toolbar
 django-debug-toolbar-request-history
 pytest-mock
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@d60d90d787d94de122dfe139df4790b82eee7554
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@6e360ed6c3d3205a4c2caddb04d946e28420fed4
 # use the version with relaxed attr requirements
 pytest-docker @ git+https://github.com/avast/pytest-docker@519b155009b6f3570c01f1f56e7c4e6ce3c5c760
 

requirements-dev.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements-dev.txt --resolver=backtracking requirements-dev.in
 #
-aiohttp==3.8.6
+aiohttp==3.9.5
     # via
     #   -c requirements.txt
     #   geoip2
@@ -121,7 +121,7 @@ coverage[toml]==7.2.7
     # via
     #   -r requirements-dev.in
     #   pytest-cov
-cryptography==41.0.5
+cryptography==41.0.7
     # via
     #   -c security-constraints.in
     #   pyopenssl
@@ -150,7 +150,7 @@ diff-match-patch==20230430
     #   websecmap
 dill==0.3.6
     # via pylint
-django==4.2.7
+django==4.2.13
     # via
     #   -c security-constraints.in
     #   django-celery-beat
@@ -179,7 +179,7 @@ django-colorful==1.3
     # via
     #   -c requirements.txt
     #   websecmap
-django-constance[database]==2.9.1
+django-constance==3.1.0
     # via
     #   -c requirements.txt
     #   websecmap
@@ -788,7 +788,7 @@ tomli==2.0.1
     #   pytest
 tomlkit==0.11.8
     # via pylint
-tornado==6.3.3
+tornado==6.4.0
     # via
     #   -c requirements.txt
     #   flower
@@ -851,7 +851,7 @@ vine==1.3.0
     #   websecmap
 vulture==2.7
     # via -r requirements-dev.in
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@d60d90d787d94de122dfe139df4790b82eee7554
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@6e360ed6c3d3205a4c2caddb04d946e28420fed4
     # via -r requirements-dev.in
 wikidata==0.7.0
     # via
@@ -878,12 +878,12 @@ yarl==1.9.2
     #   websecmap
 zipp==3.15.0
     # via importlib-metadata
-zope-event==5.0
+zope.event==5.0
     # via
     #   -c requirements.txt
     #   gevent
     #   websecmap
-zope-interface==6.0
+zope.interface==6.0
     # via
     #   -c requirements.txt
     #   gevent

requirements.in

@@ -9,7 +9,7 @@
 # - no version pinning, unless it is required and explained
 
 # to update websecmap, update the SHA hash and run: make update_requirement_websecmap
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@d60d90d787d94de122dfe139df4790b82eee7554
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@6e360ed6c3d3205a4c2caddb04d946e28420fed4
 
 django-otp
 django-two-factor-auth>1.15