diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..c9adf60 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,49 @@ +name: QA and image building + +on: + pull_request: + push: + branches: + - main + +jobs: + build: + environment: Default + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + # This helps with adding version information to the build in the admin on production. CI usually works shallow. + # https://stackoverflow.com/questions/4916492/git-describe-fails-with-fatal-no-names-found-cannot-describe-anything + # this will fetch the whole repository, which is more expensive. + - run: git fetch --prune --unshallow --tags + + - name: build images + run: | + VERSION=$(git describe --tags --abbrev=0)-$(git log -1 --pretty=format:%h) + make build build_args=--build-arg=VERSION=${VERSION:-0.0.0-dev0} + + - name: check + run: make check + + - name: lint + run: make lint + + - name: test + run: make test + + - name: Login to Docker Hub + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push image to docker hub + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + make push_images + env: + DOCKER_USERNAME: internetstandards + secure: U7kSLhWutEWne8w0zMUN3XzWWWHnK0NKuDxrKQBtYpH5VA+ROd0js7y9uFYflqdJO/doiXE4updVM8D1aGrxfJ8Wp2Ywp5vC/6Ua+Wn+pdIi2WNBqOgCeq3+2NUMND7DpChFIA9dBiEOCnnOlRKYaz5bBhOR7trruV2E+QSCG+Scxejg+pToF/cRsQDI9zTktAr9rfWDfOjz0jfxC2NixOP8THFZhDW/vXlRWWGtmwvnawwXMdyhRYnIV/fvb2RnHeVlDXckMm+GTEcRwKZTp1Cy/lCuk8zEzxPkSvrYnAeC8Dto/KPFo9XnRNKNPygf4t7YlJ8eGvw7sp5QKH327D8+1kjspq20Aqq7IoHSHnhmYGrJSsgmoWsod6Be8PaFV/FjjfDbMvcpNspVxSPQu/h0P0POdK3+0HJBQ0j1qaYlOwlKGFyoNEN+hPOg7A/RWeV+P5wn5SxZCepL4T95uS4JIsb7SMmD1GxRTr+6ygRrKOnduBtBXFV48LALxvAwvBTka9muo924HDNsRQiI0r5gizoNPPTw+T8tzynoc7QRalih03UnEVvRxuPPz+kF+GyXQYLjVRIuPlrubnprOuW31VYEgwzAv32Ec3OZoUvx6MQvSvsignklSJgg9pucMu6apQPNkyt8GPRgUKUqQscPY1mSMhblWnaPAu7my9w= diff --git a/.gitignore b/.gitignore index 23bd691..819af31 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ suggestions_domain_25_10_2024* *export_merklemap_nl_zone_2024_20_25* .root/ *.egg-info +.build diff --git a/.gitmodules b/.gitmodules index cc09f98..583e24b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "vendor/certstream"] path = vendor/certstream - url = git@github.com:CaliDog/certstream-server.git + url = https://github.com/CaliDog/certstream-server.git diff --git a/Dockerfile b/Dockerfile index 9fa7ab3..2a71dbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,6 @@ RUN pip install -r requirements.txt ADD README.md /src ADD pyproject.toml /src -ADD src /src/src - -RUN pip install . FROM build AS dev @@ -22,6 +19,9 @@ WORKDIR /src ADD requirements-dev.txt /src RUN pip install -r requirements-dev.txt +ADD src /src/src +RUN pip install --editable . + ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings ENTRYPOINT [ "bash", "-c" ] @@ -29,12 +29,19 @@ CMD [ "bash" ] FROM build AS app +ADD src /src/src +RUN pip install --editable . + ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings ENV UWSGI_MODULE ctlssa.app.wsgi -ENV UWSGI_HTTP_SOCKET=:8000 +ENV UWSGI_HTTP_SOCKET=:8001 ENV UWSGI_MASTER=1 ENV UWSGI_UID=nobody +ENV DJANGO_PORT=8001 + +EXPOSE 8001 -EXPOSE 8000 +ARG VERSION=0.0.0-dev0 +ENV SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION ENTRYPOINT [ "ctlssa" ] diff --git a/Makefile b/Makefile index 6e7d135..7992727 100644 --- a/Makefile +++ b/Makefile @@ -1,38 +1,71 @@ SHELL = /bin/bash -ctlssa = docker compose run -ti app -dev = docker compose run -ti -v ${PWD}/.root:/root --rm dev +dev = docker compose --profile=dev run -i --rm dev +db = docker compose exec -i db # run this before/after checking in/out the source all: build lint test # run the entire project -run up: requirements - docker compose up --build --remove-orphans --watch +run up: + docker compose up --remove-orphans --watch # make migration files -makemigrations: build - ${ctlssa} makemigrations +makemigrations: + ${dev} "ctlssa makemigrations" + +# load testdatta fixture +testdata: + ${dev} "ctlssa loaddata testdata" # run development shell -dev dev-shell shell: build +dev dev-shell shell: ${dev} +dbshell: + ${db} psql --user ctlssa + # fix/check linting issues -lint fix: build +lint fix: ${dev} "isort ." ${dev} "black ." # run test suite -test: build - ${dev} "pytest ./tests/" +test: + ${dev} "pytest --verbose ." -# build docker container images -build: requirements - docker compose build +test-watch: + ${dev} "ptw --clear . -- --verbose" + +check: requirements + @if [ ! -z "$(shell git status --porcelain requirements*.txt)" ];then \ + echo "Requirements .in files have not all been compiled into .txt files and commited to Git!"; \ + git status --porcelain requirements*.txt; \ + exit 1; \ + fi + +testcase: + # support extra verbosity in testcases so differences can be copy-pasted when needed and to see exactly + # what is happening. + # make testcase case=filter condition + ${dev} "pytest -vvv -k ${case} ./tests/" # updates requirements.txt files when requirements.in files have changes requirements_files = $(subst .in,.txt,$(wildcard requirements*.in)) requirements: ${requirements_files} ${requirements_files}: %.txt: %.in - ${dev} pip-compile $< --output-file $@ + ${dev} "pip-compile $< --output-file $@" + +# build docker container images +.PHONY: build +build: + docker compose build ${build_args} + +push_images: + docker compose --push app certstream + +# remove all runtime state and cache from the project +mrproper: + docker compose rm --volumes --force --stop + docker system prune --filter label=com.docker.compose.project="$(shell docker compose config --format yaml | sed -nE 's/^name: (.*)/\1/p')" --all --force --volumes + rm -fr build/ src/*.egg-info diff --git a/README.md b/README.md index f61463e..024e9a9 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,13 @@ fixture might be present. This contains 5.991.724 records. -> Git refused this, ## What are the limits of this tool The limits have not yet been discovered and no optimizations have been performed yet, aside from a few proactive -database indexes. It is expected to being able to store about a years worth of data from the .nl zone. This means -about 5 million domains with an estimated 50 million subdomains, each which will have a new certificate every 90 days. -In total about 200 million records per year. This is the same in most EU countries. There is no expectation that this -tool will work quickly on the combined com/net/org zones. Although some partitioning and smarter inserting might just -do the trick. For the Netherlands the total number of certificate renewals seems to be much lower for subdomains, -between 0.5 to 2 per second. +database indexes. It is expected to being able to store about a years worth of data from the .nl zone. The preload +from merklemap will load up 6 million subdomains (which is much lower than the expected 50 million). + +For the Netherlands the total number of certificate renewals seems to be much lower for subdomains, +between 0.5 to 2 per second. Each subdomain which will have a new certificate every 90 days. This is the same in +most EU countries. There is no expectation that this tool will work quickly on the combined com/net/org zones. +Although some partitioning and smarter inserting might just do the trick. The goal is to being able to run this on medium sized virtual machines with just a few cores and a few gigabytes of ram. That should be enough for the Netherlands and most EU countries. We've not tried to see if this solution is 'web @@ -52,8 +53,8 @@ This command should run forever. In case your certstream server is down it will The webserver can be started with the command: ```python manage.py runserver``` -When you visit the web interface at http://localhost:8000/ you will see a blank JSON response. Use the following -parameters to retrieve data: `http://localhost:8000/?domain=example&suffix=nl&period=365` +When you visit the web interface at http://localhost:8001/ you will see a blank JSON response. Use the following +parameters to retrieve data: `http://localhost:8001/?domain=example&suffix=nl&period=365` ## Further configuration options @@ -123,19 +124,44 @@ Requirements for development are: - Compose - GNU make +### Checking out the code + +This repository contains submodules, so after the Git clone this submodule should be initialized. + + git clone git@github.com:internetstandards/Internet.nl-ct-log-subdomain-suggestions-api.git + git submodule update --init + +### Building the application + +Before running the application, or whenever code changes, the Docker images need to be built. For this run: + + make build + +Running the build step before every command is recommended to ensure the images are up to date, that's why it's included in all the command below. It can be omitted if you know you don't need it. + ### Running application -To run the application for development run: +To run the application for development use: + + make build run + +This will start a full development stack with: - make run + - uWSGI application + - ingest + - certstream server + - postgreSQL server + - database migration -A web interface will be available at `http://localhost:8000`. +A web interface will be available at `http://localhost:8001`. + +Compose will watch for file changes and restart the required services accordingly. ### Linting Run these commands before checking in. These should all pass without error. - make lint + make build lint *notice*: this command autofixes trivial issues in formatting and updates the source files @@ -143,16 +169,44 @@ Run these commands before checking in. These should all pass without error. To run the test suite use: - make test + make build test + +To rerun tests every time a file changes use: + + make build test-watch ### Development shell -To open a shell with all dependencies and development tools installed run: +To open a shell with all dependencies and development tools installed, first bring the project up, then run: - make dev + make build dev + +From here you can run the `ctlssa` command to perform Django `./manage.py` functions like: + + ctlssa makemigrations + + ctlssa shell + + ctlssa loadfixtures testdata + + ctlssa bulk_ingest --file data.json + +For all commands run: + + ctlssa --help ### Dependency management After changing requirements in any of the `.in` files update the `.txt` files using: - make requirements + make build requirements + +### Database shell (postgresql) + + make dbshell + +### Reset + +If nothing else works, try resetting the project state by deleting all runtime state and caches using: + + make mrproper diff --git a/compose.yml b/compose.yml index b633344..8d81a02 100644 --- a/compose.yml +++ b/compose.yml @@ -1,29 +1,51 @@ services: app: image: internetstandards/ctlssa:latest - build: . + build: &app_build + context: . + cache_from: + - internetstandards/ctlssa:latest + - type=gha + cache_to: + - type=gha + args: + BUILDKIT_INLINE_CACHE: 1 + develop: &app_develop + watch: + - path: src/ + action: sync+restart + target: /src/src/ entrypoint: uwsgi ports: - - 8000:8000 + - 8001:8001 environment: CTLSSA_SECRET_KEY: '1' CTLSSA_DJANGO_DATABASE: production CTLSSA_DB_ENGINE: postgresql_psycopg2 CTLSSA_DB_HOST: db CTLSSA_CERTSTREAM_SERVER_URL: ws://certstream + # uwsgi reloads on SIGTERM, so use SIGINT instead + # https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi + stop_signal: SIGINT + restart: always depends_on: + db: + condition: service_healthy db-migrate: # wait for DB migration to be completed condition: service_completed_successfully + app-ingest: + condition: service_started healthcheck: - test: curl --silent --fail http://127.0.0.1:8000 + test: curl --silent --fail http://127.0.0.1:8001 interval: 5m start_period: 30s start_interval: 1s app-ingest: - build: . - image: internetstandards/ctlssa + build: *app_build + develop: *app_develop + image: internetstandards/ctlssa:latest environment: CTLSSA_SECRET_KEY: '1' CTLSSA_DJANGO_DATABASE: production @@ -32,10 +54,19 @@ services: CTLSSA_CERTSTREAM_SERVER_URL: ws://certstream entrypoint: ctlssa command: ingest + restart: always + # certstream library only listens to SIGINT + # https://github.com/CaliDog/certstream-python/blob/master/certstream/core.py#L56 + stop_signal: SIGINT + stop_grace_period: 3s depends_on: + db: + condition: service_healthy db-migrate: # wait for DB migration to be completed condition: service_completed_successfully + certstream: + condition: service_healthy certstream: build: @@ -43,13 +74,16 @@ services: image: internetstandards/certstream environment: LOG_LEVEL: error - depends_on: - # no need starting certstream before we can ingest - app-ingest: - condition: service_started + restart: always + healthcheck: + test: wget -q 127.0.0.1:4000 -O /dev/null + interval: 5m + start_period: 30s + start_interval: 1s db: image: postgres:15 + restart: always environment: POSTGRES_DB: ctlssa POSTGRES_USER: ctlssa @@ -59,10 +93,14 @@ services: interval: 5m start_period: 30s start_interval: 1s + cpu_percent: ${CERTSTREAM_CPU_PERCENT:-10} + volumes: + - postgres:/var/lib/postgresql/data db-migrate: - build: . - image: internetstandards/ctlssa + build: *app_build + develop: *app_develop + image: internetstandards/ctlssa:latest restart: on-failure environment: CTLSSA_SECRET_KEY: '1' @@ -78,5 +116,20 @@ services: dev: build: target: dev + environment: + CTLSSA_SECRET_KEY: '1' + CTLSSA_DJANGO_DATABASE: production + CTLSSA_DB_ENGINE: postgresql_psycopg2 + CTLSSA_DB_HOST: db volumes: - .:/src + - ./.root:/root + depends_on: + app: + condition: service_healthy + + profiles: + - dev + +volumes: + postgres: diff --git a/pyproject.toml b/pyproject.toml index ff7ea60..e4fbafb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,8 @@ requires-python = ">=3.10" dynamic = ["dependencies", "optional-dependencies"] name = "ctlssa" readme = "README.md" -version = "1.0.0" +# overwritten during CI build using VERSION/SETUPTOOLS_SCM_PRETEND_VERSION +version = "0.0.0-dev0" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} @@ -13,7 +14,7 @@ optional-dependencies = {dev = {file = ["requirements-dev.txt"]}, deploy = {file ctlssa = "ctlssa.manage:main" [tool.setuptools.package-data] -ctlssa = ["*.json"] +"*" = ["*.json"] [build-system] requires = [ diff --git a/requirements-dev.in b/requirements-dev.in index c23de7d..3d5a7fc 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -8,3 +8,4 @@ coverage django-coverage pytest-cov pip-tools +pytest-watch diff --git a/requirements-dev.txt b/requirements-dev.txt index a4b84e1..82317fe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements-dev.in +# pip-compile --output-file=requirements-dev.txt requirements-dev.in # black==24.10.0 # via -r requirements-dev.in @@ -12,12 +12,16 @@ click==8.1.7 # via # black # pip-tools +colorama==0.4.6 + # via pytest-watch coverage[toml]==7.6.4 # via # -r requirements-dev.in # pytest-cov django-coverage==1.2.4 # via -r requirements-dev.in +docopt==0.6.2 + # via pytest-watch freezegun==1.5.1 # via -r requirements-dev.in iniconfig==2.0.0 @@ -47,14 +51,19 @@ pytest==8.3.3 # via # pytest-cov # pytest-django + # pytest-watch pytest-cov==5.0.0 # via -r requirements-dev.in pytest-django==4.9.0 # via -r requirements-dev.in +pytest-watch==4.2.0 + # via -r requirements-dev.in python-dateutil==2.9.0.post0 # via freezegun six==1.16.0 # via python-dateutil +watchdog==6.0.0 + # via pytest-watch wheel==0.44.0 # via pip-tools diff --git a/requirements.txt b/requirements.txt index 4159105..3e9c832 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,76 +5,44 @@ # pip-compile # asgiref==3.8.1 - # via - # -c requirements.txt - # django + # via django certifi==2024.8.30 - # via - # -c requirements.txt - # requests + # via requests certstream==1.12 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in charset-normalizer==3.4.0 - # via - # -c requirements.txt - # requests + # via requests colorlog==6.8.2 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in django==4.2.16 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in filelock==3.16.1 - # via - # -c requirements.txt - # tldextract + # via tldextract idna==3.10 # via - # -c requirements.txt # requests # tldextract +psycopg==3.2.3 + # via -r requirements.in pysimdjson==6.0.2 # via -r requirements.in python-xz==0.5.0 # via -r requirements.in -psycopg==3.2.3 - # via - # -c requirements.txt - # -r requirements.in requests==2.32.3 # via - # -c requirements.txt # requests-file # tldextract requests-file==2.1.0 - # via - # -c requirements.txt - # tldextract + # via tldextract sqlparse==0.5.1 - # via - # -c requirements.txt - # django + # via django termcolor==2.5.0 - # via - # -c requirements.txt - # certstream + # via certstream tldextract==5.1.2 - # via - # -c requirements.txt - # -r requirements.in + # via -r requirements.in typing-extensions==4.12.2 - # via - # -c requirements.txt - # psycopg + # via psycopg urllib3==2.2.3 - # via - # -c requirements.txt - # requests + # via requests websocket-client==1.8.0 - # via - # -c requirements.txt - # certstream + # via certstream diff --git a/src/ctlssa/app/settings.py b/src/ctlssa/app/settings.py index 04488b7..ec245b4 100644 --- a/src/ctlssa/app/settings.py +++ b/src/ctlssa/app/settings.py @@ -17,7 +17,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ diff --git a/src/ctlssa/suggestions/fixtures/testdata.json b/src/ctlssa/suggestions/fixtures/testdata.json index 2cedfbb..df26863 100644 --- a/src/ctlssa/suggestions/fixtures/testdata.json +++ b/src/ctlssa/suggestions/fixtures/testdata.json @@ -1 +1,533 @@ -[{"model": "suggestions.domain", "pk": 30116, "fields": {"subdomain": "plan", "domain": "roughmen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30117, "fields": {"subdomain": "planning", "domain": "roughmen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30119, "fields": {"subdomain": "www", "domain": "praktijkbosveld", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30121, "fields": {"subdomain": "www", "domain": "bureauhofmeijer", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30123, "fields": {"subdomain": "www", "domain": "odeaanlent", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30125, "fields": {"subdomain": "www", "domain": "odeaanlent", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30127, "fields": {"subdomain": "www", "domain": "qrisicoadviseurs", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30128, "fields": {"subdomain": "r.fin-acc", "domain": "newomij", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30129, "fields": {"subdomain": "lime", "domain": "wwwdeltalloyd", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30131, "fields": {"subdomain": "www", "domain": "byaniek", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30133, "fields": {"subdomain": "www", "domain": "byaniek", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30135, "fields": {"subdomain": "www", "domain": "erciyeshalal", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30136, "fields": {"subdomain": "www", "domain": "yaabba", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30138, "fields": {"subdomain": "www", "domain": "scentful", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30139, "fields": {"subdomain": "pod", "domain": "hanos", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30142, "fields": {"subdomain": "www", "domain": "erciyeshalal", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30144, "fields": {"subdomain": "pub.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30145, "fields": {"subdomain": "www.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30146, "fields": {"subdomain": "pub.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30147, "fields": {"subdomain": "www.math", "domain": "leidenuniv", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30148, "fields": {"subdomain": "qoroyo", "domain": "etcbc", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30287, "fields": {"subdomain": "www", "domain": "bomenriddershardenberg", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30289, "fields": {"subdomain": "www", "domain": "crossfitharderwijk", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30290, "fields": {"subdomain": "dev", "domain": "afotos", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30291, "fields": {"subdomain": "dev", "domain": "afotos", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30293, "fields": {"subdomain": "www", "domain": "eigenbenen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30295, "fields": {"subdomain": "www", "domain": "eigenbenen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30297, "fields": {"subdomain": "www", "domain": "eigenbenen", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30298, "fields": {"subdomain": "autodiscover", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30299, "fields": {"subdomain": "cpanel", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30300, "fields": {"subdomain": "cpcalendars", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30301, "fields": {"subdomain": "cpcontacts", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30303, "fields": {"subdomain": "ipv6", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30304, "fields": {"subdomain": "mail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30305, "fields": {"subdomain": "webdisk", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30306, "fields": {"subdomain": "webmail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30307, "fields": {"subdomain": "www", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30308, "fields": {"subdomain": "autodiscover", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30309, "fields": {"subdomain": "cpanel", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30310, "fields": {"subdomain": "cpcalendars", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30311, "fields": {"subdomain": "cpcontacts", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30313, "fields": {"subdomain": "ipv6", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30314, "fields": {"subdomain": "mail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30315, "fields": {"subdomain": "webdisk", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30316, "fields": {"subdomain": "webmail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30317, "fields": {"subdomain": "www", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30318, "fields": {"subdomain": "autodiscover", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30319, "fields": {"subdomain": "cpanel", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30320, "fields": {"subdomain": "cpcalendars", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30321, "fields": {"subdomain": "cpcontacts", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30323, "fields": {"subdomain": "ipv6", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30324, "fields": {"subdomain": "mail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30325, "fields": {"subdomain": "webdisk", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30326, "fields": {"subdomain": "webmail", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30327, "fields": {"subdomain": "www", "domain": "example", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30329, "fields": {"subdomain": "www", "domain": "henkbexkens", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30331, "fields": {"subdomain": "www", "domain": "henkbexkens", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30333, "fields": {"subdomain": "www", "domain": "kiefttweewielers", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30335, "fields": {"subdomain": "www", "domain": "kiefttweewielers", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30336, "fields": {"subdomain": "sacraea-saturna", "domain": "mattdrums", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30337, "fields": {"subdomain": "sacraea-saturna", "domain": "mattdrums", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30339, "fields": {"subdomain": "www", "domain": "jelgerhuis", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30341, "fields": {"subdomain": "www", "domain": "jelgerhuis", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30343, "fields": {"subdomain": "www", "domain": "doppinie", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30345, "fields": {"subdomain": "www", "domain": "doppinie", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30347, "fields": {"subdomain": "www", "domain": "doppinie", "suffix": "nl", "last_seen": "2024-10-28"}}, {"model": "suggestions.domain", "pk": 30348, "fields": {"subdomain": "www", "domain": "waterspektakel", "suffix": "nl", "last_seen": "2024-10-28"}}] \ No newline at end of file +[ + { + "model": "suggestions.domain", + "fields": { + "subdomain": "plan", + "domain": "test", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "planning", + "domain": "test", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "nos", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "nu", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "dubbelresultaat", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "dubbelresultaat", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "autodiscover", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpanel", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcalendars", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcontacts", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "ipv6", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webdisk", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webmail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "autodiscover", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpanel", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcalendars", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcontacts", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "ipv6", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webdisk", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webmail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "autodiscover", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpanel", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcalendars", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "cpcontacts", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "ipv6", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webdisk", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "webmail", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "nl", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "com", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "www", + "domain": "example", + "suffix": "politie", + "last_seen": "2024-10-28" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "januari", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-01-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "februari", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-02-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "maart", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-03-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "april", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-04-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mei", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-05-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juni", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-06-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juli", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-07-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "augustus", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-08-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "september", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-09-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "oktober", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-10-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "november", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-11-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "december", + "domain": "2023", + "suffix": "nl", + "last_seen": "2023-12-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "januari", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-01-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "februari", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-02-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "maart", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-03-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "april", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-04-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "mei", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-05-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juni", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-06-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "juli", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-07-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "augustus", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-08-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "september", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-09-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "oktober", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-10-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "november", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-11-01" + } + }, + { + "model": "suggestions.domain", + "fields": { + "subdomain": "december", + "domain": "2024", + "suffix": "nl", + "last_seen": "2024-12-01" + } + } +] \ No newline at end of file diff --git a/src/ctlssa/suggestions/logic/bulk_ingest.py b/src/ctlssa/suggestions/logic/bulk_ingest.py index 17d5bd7..46d32c0 100644 --- a/src/ctlssa/suggestions/logic/bulk_ingest.py +++ b/src/ctlssa/suggestions/logic/bulk_ingest.py @@ -2,6 +2,7 @@ import simdjson import xz + from ..logic import log from ..logic.domains import CaseOptimizedBulkInsert @@ -67,6 +68,8 @@ def ingest_merklemap(file: str = "merklemap_data.jsonl"): with my_open(file) as f: for line in f: data = parser.parse(line) + # If you load up the django fixture here instead of the merklemap file, you'll get an integer + # error here. But no try-except as that takes extra time to process. hostname = data["hostname"] # performance trick from the simdjson manual del data diff --git a/src/ctlssa/suggestions/logic/ingest.py b/src/ctlssa/suggestions/logic/ingest.py index 71c40ec..3114db3 100644 --- a/src/ctlssa/suggestions/logic/ingest.py +++ b/src/ctlssa/suggestions/logic/ingest.py @@ -1,3 +1,5 @@ +import time + import certstream from django.conf import settings @@ -18,6 +20,22 @@ def certstream_callback(message, context): add_domains(all_domains) +def listen_for_events( + message_callback, url, skip_heartbeats=True, setup_logger=True, on_open=None, on_error=None, **kwargs +): + try: + while True: + c = certstream.core.CertStreamClient( + message_callback, url, skip_heartbeats=skip_heartbeats, on_open=on_open, on_error=on_error + ) + c.run_forever(ping_interval=15, **kwargs) + time.sleep(5) + except KeyboardInterrupt: + log.info("Kill command received, exiting!!") + except: + log.exception("unhandled exception") + + def ingest_certstream(): log.info("Starting certstream ingestion from %s", settings.CERTSTREAM_SERVER_URL) - certstream.listen_for_events(certstream_callback, url=settings.CERTSTREAM_SERVER_URL) + listen_for_events(certstream_callback, url=settings.CERTSTREAM_SERVER_URL) diff --git a/src/ctlssa/suggestions/logic/suggest.py b/src/ctlssa/suggestions/logic/suggest.py index ddb7e68..56cb3f7 100644 --- a/src/ctlssa/suggestions/logic/suggest.py +++ b/src/ctlssa/suggestions/logic/suggest.py @@ -1,27 +1,48 @@ -from datetime import datetime, timedelta, timezone +import contextlib +from datetime import date, datetime, timedelta, timezone from ..models import Domain -def suggest_subdomains(domain: str, suffix: str = "nl", period_in_days: int = 365): +def suggest_subdomains(domain: str, suffix: str = "nl", period_in_days: int = 365, max_date: str = ""): if not domain or not suffix or not period_in_days: return [] - - some_time_ago = datetime.now(timezone.utc) - timedelta(days=period_in_days) - - # These are not distinct, as ct logs contain certificate requests and requests can be processed multiple times + # Results are not distinct, as ct logs contain certificate requests and requests can be processed multiple times # per day. But even if a request is performed a few times per day for a domain the below logic will be # more than fast enough to reduce this set to unique values. A 'distinct' query is not needed and if performance # issues arise, distinct might be worth considering. - return sorted( - set( - Domain.objects.filter( - # not performing a last_seen__lte=now() as that is yet another thing for the database to process, - # which makes querying slower. There is no strict need for windowing over time, only a need for showing - # accurate results today. - domain=domain, - suffix=suffix, - last_seen__gte=some_time_ago, - ).values_list("subdomain", flat=True) - ) - ) + + # The reason for using two distinct queries is to see if there are performance differences between the two for + # current and historical data. The assumption is that less filtering results in better performance. We'll see + # on the long run how this pans out. + + # silently fall back if the date value is not according to specification + with contextlib.suppress(ValueError): + # suppress "isoformat must be a string" + if not max_date: + max_date = "" + max_date = date.fromisoformat(max_date) + + if max_date: + some_time_ago = max_date - timedelta(days=period_in_days) + query = Domain.objects.filter( + # not performing a last_seen__lte=now() as that is yet another thing for the database to process, + # which makes querying slower. There is no strict need for windowing over time, only a need for showing + # accurate results today. + domain=domain, + suffix=suffix, + last_seen__gte=some_time_ago, + last_seen__lte=max_date, + ).values_list("subdomain", flat=True) + else: + some_time_ago = datetime.now(timezone.utc) - timedelta(days=period_in_days) + query = Domain.objects.filter( + # not performing a last_seen__lte=now() as that is yet another thing for the database to process, + # which makes querying slower. There is no strict need for windowing over time, only a need for showing + # accurate results today. + domain=domain, + suffix=suffix, + last_seen__gte=some_time_ago, + ).values_list("subdomain", flat=True) + + return sorted(set(query)) diff --git a/src/ctlssa/suggestions/views.py b/src/ctlssa/suggestions/views.py index 72b126a..97e5c0b 100644 --- a/src/ctlssa/suggestions/views.py +++ b/src/ctlssa/suggestions/views.py @@ -9,6 +9,7 @@ def suggest(request): request.GET.get("domain", None), request.GET.get("suffix", "nl"), request.GET.get("period_in_days", 365), + request.GET.get("max_date", None), ), safe=False, ) diff --git a/tests/conftest.py b/tests/conftest.py index cc0bbab..775aea3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ -import pytest import pathlib +import pytest + + @pytest.fixture def testdata(): return pathlib.Path(__file__).parent / "testdata" diff --git a/tests/sample_merklemap_data.jsonl b/tests/sample_merklemap_data.jsonl new file mode 100644 index 0000000..aed9a43 --- /dev/null +++ b/tests/sample_merklemap_data.jsonl @@ -0,0 +1,33 @@ +{"hostname":"nl.dev.batch.internet.nl","results":[{"success":{"query":"nl.dev.batch.internet.nl IN A","query_timestamp":"2024-10-22T22:33:11.792584228Z","records":{"A":["62.204.66.17"],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"nl.dev.batch.internet.nl IN AAAA","query_timestamp":"2024-10-22T22:33:11.792588008Z","records":{"CNAME":["dev.batch.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:17"]}}},{"success":{"query":"nl.dev.batch.internet.nl IN MX","query_timestamp":"2024-10-22T22:33:11.792592598Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"nl.dev.batch.internet.nl IN TXT","query_timestamp":"2024-10-22T22:33:11.792596298Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev.batch.internet.nl."]}}}]} +{"hostname":"ipv6kaartmetingen.internet.nl","results":[{"success":{"query":"ipv6kaartmetingen.internet.nl IN A","query_timestamp":"2024-10-22T22:37:21.799320664Z","records":{"A":["136.144.246.3"]}}},{"success":{"query":"ipv6kaartmetingen.internet.nl IN AAAA","query_timestamp":"2024-10-22T22:37:21.799324664Z","records":{"AAAA":["2a01:7c8:d006:35f:5054:ff:fe0a:8dd8"]}}},{"success":{"query":"ipv6kaartmetingen.internet.nl IN MX","query_timestamp":"2024-10-22T22:37:21.799328144Z","records":{"MX":["0 ."]}}},{"success":{"query":"ipv6kaartmetingen.internet.nl IN TXT","query_timestamp":"2024-10-22T22:37:21.799331544Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"matomo.internet.nl","results":[{"success":{"query":"matomo.internet.nl IN A","query_timestamp":"2024-10-23T01:20:16.052324787Z","records":{"CNAME":["emailveilig.internet.nl."],"A":["62.204.66.21"]}}},{"success":{"query":"matomo.internet.nl IN AAAA","query_timestamp":"2024-10-23T01:20:16.052327407Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"matomo.internet.nl IN MX","query_timestamp":"2024-10-23T01:20:16.052329957Z","records":{"MX":["0 ."],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"matomo.internet.nl IN TXT","query_timestamp":"2024-10-23T01:20:16.052334647Z","records":{"TXT":["v=spf1 -all"],"CNAME":["emailveilig.internet.nl."]}}}]} +{"hostname":"helpdesk.internet.nl","results":[{"success":{"query":"helpdesk.internet.nl IN A","query_timestamp":"2024-10-23T05:50:45.075437020Z","records":{"A":["62.204.66.22"]}}},{"success":{"query":"helpdesk.internet.nl IN AAAA","query_timestamp":"2024-10-23T05:50:45.075440200Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:22"]}}},{"success":{"query":"helpdesk.internet.nl IN MX","query_timestamp":"2024-10-23T05:50:45.075445270Z","records":{"MX":["0 ."]}}},{"success":{"query":"helpdesk.internet.nl IN TXT","query_timestamp":"2024-10-23T05:50:45.075450770Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"batch.internet.nl","results":[{"success":{"query":"batch.internet.nl IN A","query_timestamp":"2024-10-23T05:53:44.985949012Z","records":{"A":["62.204.66.12"]}}},{"success":{"query":"batch.internet.nl IN AAAA","query_timestamp":"2024-10-23T05:53:44.985951812Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:12"]}}},{"success":{"query":"batch.internet.nl IN MX","query_timestamp":"2024-10-23T05:53:44.985956522Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"batch.internet.nl IN TXT","query_timestamp":"2024-10-23T05:53:44.985962642Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"en.conn.internet.nl","results":[{"success":{"query":"en.conn.internet.nl IN A","query_timestamp":"2024-10-23T07:56:34.556390504Z","records":{"CNAME":["www.internet.nl."],"A":["62.204.66.10"]}}},{"success":{"query":"en.conn.internet.nl IN AAAA","query_timestamp":"2024-10-23T07:56:34.556394824Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"en.conn.internet.nl IN MX","query_timestamp":"2024-10-23T07:56:34.556398884Z","records":{"MX":["0 ."],"CNAME":["www.internet.nl."]}}},{"success":{"query":"en.conn.internet.nl IN TXT","query_timestamp":"2024-10-23T07:56:34.556403874Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"en.conn.dev-docker.internet.nl","results":[{"success":{"query":"en.conn.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T09:33:50.987911398Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"en.conn.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T09:33:50.987915858Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"en.conn.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T09:33:50.987922848Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"en.conn.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T09:33:50.987929938Z","records":{"CNAME":["dev-docker.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"nl.dev-docker.internet.nl","results":[{"success":{"query":"nl.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T10:05:44.178092289Z","records":{"A":["62.204.66.35"],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"nl.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T10:05:44.178096929Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"nl.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T10:05:44.178102419Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx01.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx02.prolocation.nl."]}}},{"success":{"query":"nl.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T10:05:44.178107279Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"en.dev-docker.internet.nl","results":[{"success":{"query":"en.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T10:23:51.895551112Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"en.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T10:23:51.895555983Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:35"],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"en.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T10:23:51.895561852Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"en.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T10:23:51.895566622Z","records":{"CNAME":["dev-docker.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"en.internet.nl","results":[{"success":{"query":"en.internet.nl IN A","query_timestamp":"2024-10-23T14:26:05.317658431Z","records":{"CNAME":["www.internet.nl."],"A":["62.204.66.10"]}}},{"success":{"query":"en.internet.nl IN AAAA","query_timestamp":"2024-10-23T14:26:05.317662311Z","records":{"CNAME":["www.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"en.internet.nl IN MX","query_timestamp":"2024-10-23T14:26:05.317667201Z","records":{"MX":["0 ."],"CNAME":["www.internet.nl."]}}},{"success":{"query":"en.internet.nl IN TXT","query_timestamp":"2024-10-23T14:26:05.317671161Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"www.dev-docker.internet.nl","results":[{"success":{"query":"www.dev-docker.internet.nl IN A","query_timestamp":"2024-10-23T15:03:53.145159703Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"www.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-23T15:03:53.145164473Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"www.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-23T15:03:53.145170523Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"www.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-23T15:03:53.145174363Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"dev.batch.internet.nl","results":[{"success":{"query":"dev.batch.internet.nl IN A","query_timestamp":"2024-10-23T15:50:15.028538521Z","records":{"A":["62.204.66.17"]}}},{"success":{"query":"dev.batch.internet.nl IN AAAA","query_timestamp":"2024-10-23T15:50:15.028541771Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:17"]}}},{"success":{"query":"dev.batch.internet.nl IN MX","query_timestamp":"2024-10-23T15:50:15.028546701Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"dev.batch.internet.nl IN TXT","query_timestamp":"2024-10-23T15:50:15.028551271Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"conn.internet.nl","results":[{"success":{"query":"conn.internet.nl IN A","query_timestamp":"2024-10-23T21:30:26.439068901Z","records":{"CNAME":["www.internet.nl."],"A":["62.204.66.10"]}}},{"success":{"query":"conn.internet.nl IN AAAA","query_timestamp":"2024-10-23T21:30:26.439073191Z","records":{"CNAME":["www.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"conn.internet.nl IN MX","query_timestamp":"2024-10-23T21:30:26.439077411Z","records":{"CNAME":["www.internet.nl."],"MX":["0 ."]}}},{"success":{"query":"conn.internet.nl IN TXT","query_timestamp":"2024-10-23T21:30:26.439083561Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"emailveilig.internet.nl","results":[{"success":{"query":"emailveilig.internet.nl IN A","query_timestamp":"2024-10-23T23:28:24.380730112Z","records":{"A":["62.204.66.21"]}}},{"success":{"query":"emailveilig.internet.nl IN AAAA","query_timestamp":"2024-10-23T23:28:24.380732502Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"]}}},{"success":{"query":"emailveilig.internet.nl IN MX","query_timestamp":"2024-10-23T23:28:24.380735882Z","records":{"MX":["0 ."]}}},{"success":{"query":"emailveilig.internet.nl IN TXT","query_timestamp":"2024-10-23T23:28:24.380740652Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"www.dev-docker.batch.internet.nl","results":[{"success":{"query":"www.dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-23T23:45:02.835672913Z","records":{"A":["62.204.66.15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"www.dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-23T23:45:02.835676703Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"www.dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-23T23:45:02.835689873Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"www.dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-23T23:45:02.835696542Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.batch.internet.nl."]}}}]} +{"hostname":"dev-docker.internet.nl","results":[{"success":{"query":"dev-docker.internet.nl IN A","query_timestamp":"2024-10-24T00:15:10.147581691Z","records":{"A":["62.204.66.35"]}}},{"success":{"query":"dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-24T00:15:10.147584881Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"dev-docker.internet.nl IN MX","query_timestamp":"2024-10-24T00:15:10.147590471Z","records":{"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-24T00:15:10.147595061Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"dashboard.internet.nl","results":[{"success":{"query":"dashboard.internet.nl IN A","query_timestamp":"2024-10-24T00:16:35.500205670Z","records":{"A":["149.210.144.250"]}}},{"success":{"query":"dashboard.internet.nl IN AAAA","query_timestamp":"2024-10-24T00:16:35.500209260Z","records":{"AAAA":["2a01:7c8:aab1:2c::1"]}}},{"success":{"query":"dashboard.internet.nl IN TXT","query_timestamp":"2024-10-24T00:16:35.500216210Z","records":{"TXT":["v=spf1 include:spfinc.prolocation.net ~all"]}}}]} +{"hostname":"acc.dashboard.internet.nl","results":[{"success":{"query":"acc.dashboard.internet.nl IN A","query_timestamp":"2024-10-24T00:33:24.828652559Z","records":{"A":["149.210.150.226"]}}},{"success":{"query":"acc.dashboard.internet.nl IN AAAA","query_timestamp":"2024-10-24T00:33:24.828655710Z","records":{"AAAA":["2a01:7c8:aab2:3f::1"]}}},{"success":{"query":"acc.dashboard.internet.nl IN MX","query_timestamp":"2024-10-24T00:33:24.828659470Z","records":{"MX":["0 ."]}}},{"success":{"query":"acc.dashboard.internet.nl IN TXT","query_timestamp":"2024-10-24T00:33:24.828664110Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"nl.conn.internet.nl","results":[{"success":{"query":"nl.conn.internet.nl IN A","query_timestamp":"2024-10-24T03:37:36.116653128Z","records":{"A":["62.204.66.10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.conn.internet.nl IN AAAA","query_timestamp":"2024-10-24T03:37:36.116657088Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.conn.internet.nl IN MX","query_timestamp":"2024-10-24T03:37:36.116660268Z","records":{"MX":["0 ."],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.conn.internet.nl IN TXT","query_timestamp":"2024-10-24T03:37:36.116665258Z","records":{"CNAME":["www.internet.nl."],"TXT":["v=spf1 -all"]}}}]} +{"hostname":"toolbox.internet.nl","results":[{"success":{"query":"toolbox.internet.nl IN A","query_timestamp":"2024-10-24T08:05:53.710558088Z","records":{"CNAME":["emailveilig.internet.nl."],"A":["62.204.66.21"]}}},{"success":{"query":"toolbox.internet.nl IN AAAA","query_timestamp":"2024-10-24T08:05:53.710562108Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"toolbox.internet.nl IN MX","query_timestamp":"2024-10-24T08:05:53.710565657Z","records":{"MX":["0 ."],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"toolbox.internet.nl IN TXT","query_timestamp":"2024-10-24T08:05:53.710570907Z","records":{"TXT":["v=spf1 -all"],"CNAME":["emailveilig.internet.nl."]}}}]} +{"hostname":"nl.conn.dev-docker.internet.nl","results":[{"success":{"query":"nl.conn.dev-docker.internet.nl IN A","query_timestamp":"2024-10-24T13:46:04.837848047Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"nl.conn.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-24T13:46:04.837850777Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"nl.conn.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-24T13:46:04.837855437Z","records":{"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."],"CNAME":["dev-docker.internet.nl."]}}},{"success":{"query":"nl.conn.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-24T13:46:04.837859677Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"www.internet.nl","results":[{"success":{"query":"www.internet.nl IN A","query_timestamp":"2024-10-24T14:56:35.674670902Z","records":{"A":["62.204.66.10"]}}},{"success":{"query":"www.internet.nl IN AAAA","query_timestamp":"2024-10-24T14:56:35.674674262Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"www.internet.nl IN MX","query_timestamp":"2024-10-24T14:56:35.674676892Z","records":{"MX":["0 ."]}}},{"success":{"query":"www.internet.nl IN TXT","query_timestamp":"2024-10-24T14:56:35.674681282Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"nl.batch.internet.nl","results":[{"success":{"query":"nl.batch.internet.nl IN A","query_timestamp":"2024-10-24T15:50:15.435279565Z","records":{"A":["62.204.66.12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"nl.batch.internet.nl IN AAAA","query_timestamp":"2024-10-24T15:50:15.435284505Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"nl.batch.internet.nl IN MX","query_timestamp":"2024-10-24T15:50:15.435292495Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"nl.batch.internet.nl IN TXT","query_timestamp":"2024-10-24T15:50:15.435298595Z","records":{"CNAME":["batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"ipv6kaart.internet.nl","results":[{"success":{"query":"ipv6kaart.internet.nl IN A","query_timestamp":"2024-10-24T17:54:46.046966244Z","records":{"A":["93.119.13.25"]}}},{"success":{"query":"ipv6kaart.internet.nl IN AAAA","query_timestamp":"2024-10-24T17:54:46.046970194Z","records":{"AAAA":["2a01:7c8:bb09:2a:5054:ff:fe24:c0f8"]}}},{"success":{"query":"ipv6kaart.internet.nl IN MX","query_timestamp":"2024-10-24T17:54:46.046972874Z","records":{"MX":["0 ."]}}},{"success":{"query":"ipv6kaart.internet.nl IN TXT","query_timestamp":"2024-10-24T17:54:46.046977594Z","records":{"TXT":["v=spf1 -all"]}}}]} +{"hostname":"nl.internet.nl","results":[{"success":{"query":"nl.internet.nl IN A","query_timestamp":"2024-10-24T19:31:40.379357304Z","records":{"A":["62.204.66.10"],"CNAME":["www.internet.nl."]}}},{"success":{"query":"nl.internet.nl IN AAAA","query_timestamp":"2024-10-24T19:31:40.379360273Z","records":{"CNAME":["www.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:10"]}}},{"success":{"query":"nl.internet.nl IN MX","query_timestamp":"2024-10-24T19:31:40.379363084Z","records":{"CNAME":["www.internet.nl."],"MX":["0 ."]}}},{"success":{"query":"nl.internet.nl IN TXT","query_timestamp":"2024-10-24T19:31:40.379366644Z","records":{"TXT":["v=spf1 -all"],"CNAME":["www.internet.nl."]}}}]} +{"hostname":"nl.dev-docker.batch.internet.nl","results":[{"success":{"query":"nl.dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-24T19:37:22.329580341Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"A":["62.204.66.15"]}}},{"success":{"query":"nl.dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-24T19:37:22.329585101Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"nl.dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-24T19:37:22.329594191Z","records":{"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"nl.dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-24T19:37:22.329600421Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"conn.dev-docker.internet.nl","results":[{"success":{"query":"conn.dev-docker.internet.nl IN A","query_timestamp":"2024-10-24T21:32:57.567926627Z","records":{"CNAME":["dev-docker.internet.nl."],"A":["62.204.66.35"]}}},{"success":{"query":"conn.dev-docker.internet.nl IN AAAA","query_timestamp":"2024-10-24T21:32:57.567931367Z","records":{"CNAME":["dev-docker.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:35"]}}},{"success":{"query":"conn.dev-docker.internet.nl IN MX","query_timestamp":"2024-10-24T21:32:57.567938457Z","records":{"CNAME":["dev-docker.internet.nl."],"MX":["10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl.","10 vmx03.prolocation.net."]}}},{"success":{"query":"conn.dev-docker.internet.nl IN TXT","query_timestamp":"2024-10-24T21:32:57.567942727Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.internet.nl."]}}}]} +{"hostname":"dev-docker.batch.internet.nl","results":[{"success":{"query":"dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-25T02:53:52.168466116Z","records":{"A":["62.204.66.15"]}}},{"success":{"query":"dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T02:53:52.168468876Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"]}}},{"success":{"query":"dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-25T02:53:52.168473436Z","records":{"MX":["10 vmx03.prolocation.net.","10 vmx02.prolocation.nl.","10 vmx01.prolocation.nl."]}}},{"success":{"query":"dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T02:53:52.168476006Z","records":{"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"en.dev.batch.internet.nl","results":[{"success":{"query":"en.dev.batch.internet.nl IN A","query_timestamp":"2024-10-25T03:24:21.659120598Z","records":{"CNAME":["dev.batch.internet.nl."],"A":["62.204.66.17"]}}},{"success":{"query":"en.dev.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T03:24:21.659125048Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:17"],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"en.dev.batch.internet.nl IN MX","query_timestamp":"2024-10-25T03:24:21.659131188Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["dev.batch.internet.nl."]}}},{"success":{"query":"en.dev.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T03:24:21.659136068Z","records":{"CNAME":["dev.batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"e-mailveilig.internet.nl","results":[{"success":{"query":"e-mailveilig.internet.nl IN A","query_timestamp":"2024-10-25T04:01:58.281706753Z","records":{"CNAME":["emailveilig.internet.nl."],"A":["62.204.66.21"]}}},{"success":{"query":"e-mailveilig.internet.nl IN AAAA","query_timestamp":"2024-10-25T04:01:58.281710553Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:21"],"CNAME":["emailveilig.internet.nl."]}}},{"success":{"query":"e-mailveilig.internet.nl IN MX","query_timestamp":"2024-10-25T04:01:58.281714263Z","records":{"CNAME":["emailveilig.internet.nl."],"MX":["0 ."]}}},{"success":{"query":"e-mailveilig.internet.nl IN TXT","query_timestamp":"2024-10-25T04:01:58.281720063Z","records":{"TXT":["v=spf1 -all"],"CNAME":["emailveilig.internet.nl."]}}}]} +{"hostname":"en.dev-docker.batch.internet.nl","results":[{"success":{"query":"en.dev-docker.batch.internet.nl IN A","query_timestamp":"2024-10-25T09:56:45.958339736Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"A":["62.204.66.15"]}}},{"success":{"query":"en.dev-docker.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T09:56:45.958343986Z","records":{"AAAA":["2a00:d00:ff:162:62:204:66:15"],"CNAME":["dev-docker.batch.internet.nl."]}}},{"success":{"query":"en.dev-docker.batch.internet.nl IN MX","query_timestamp":"2024-10-25T09:56:45.958349686Z","records":{"CNAME":["dev-docker.batch.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl."]}}},{"success":{"query":"en.dev-docker.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T09:56:45.958354706Z","records":{"TXT":["v=spf1 a -all"],"CNAME":["dev-docker.batch.internet.nl."]}}}]} +{"hostname":"en.batch.internet.nl","results":[{"success":{"query":"en.batch.internet.nl IN A","query_timestamp":"2024-10-25T10:46:16.049155460Z","records":{"A":["62.204.66.12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"en.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T10:46:16.049161069Z","records":{"CNAME":["batch.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:12"]}}},{"success":{"query":"en.batch.internet.nl IN MX","query_timestamp":"2024-10-25T10:46:16.049168889Z","records":{"MX":["10 vmx02.prolocation.nl.","10 vmx03.prolocation.net.","10 vmx01.prolocation.nl."],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"en.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T10:46:16.049174329Z","records":{"CNAME":["batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} +{"hostname":"www.batch.internet.nl","results":[{"success":{"query":"www.batch.internet.nl IN A","query_timestamp":"2024-10-25T10:51:49.580512543Z","records":{"A":["62.204.66.12"],"CNAME":["batch.internet.nl."]}}},{"success":{"query":"www.batch.internet.nl IN AAAA","query_timestamp":"2024-10-25T10:51:49.580516143Z","records":{"CNAME":["batch.internet.nl."],"AAAA":["2a00:d00:ff:162:62:204:66:12"]}}},{"success":{"query":"www.batch.internet.nl IN MX","query_timestamp":"2024-10-25T10:51:49.580521363Z","records":{"CNAME":["batch.internet.nl."],"MX":["10 vmx03.prolocation.net.","10 vmx01.prolocation.nl.","10 vmx02.prolocation.nl."]}}},{"success":{"query":"www.batch.internet.nl IN TXT","query_timestamp":"2024-10-25T10:51:49.580526103Z","records":{"CNAME":["batch.internet.nl."],"TXT":["v=spf1 a -all"]}}}]} \ No newline at end of file diff --git a/tests/sample_merklemap_data.xz b/tests/sample_merklemap_data.xz new file mode 100644 index 0000000..f0f2cd6 Binary files /dev/null and b/tests/sample_merklemap_data.xz differ diff --git a/tests/test_suggest.py b/tests/test_suggest.py index 4800b61..e96a86c 100644 --- a/tests/test_suggest.py +++ b/tests/test_suggest.py @@ -29,25 +29,45 @@ def test_suggest_subdomains(db): "www", ] + assert suggest_subdomains("example", "com", 365) == ["www"] + assert suggest_subdomains("dubbelresultaat", "nl", 365) == ["www"] + # incomplete call of the method yields no data: assert suggest_subdomains("example", "nl", 0) == [] assert suggest_subdomains("example", "", 365) == [] assert suggest_subdomains("", "nl", 365) == [] - # this will yield the same data as above as we don't support time travel in search - with freeze_time("2024-1-30 12:00:00"): - assert suggest_subdomains("example", "nl", 10) == [ - "autodiscover", - "cpanel", - "cpcalendars", - "cpcontacts", - "ipv6", - "mail", - "webdisk", - "webmail", - "www", + with freeze_time("2024-12-30 12:00:00"): + # This will result in no data, as in 10 days no domains from 2024 have been seen after december first, + # so looking 10 days in the past results in no data. + assert suggest_subdomains("2024", "nl", 10) == [] + + # broaden the time frame to get results. + assert suggest_subdomains("2024", "nl", 30) == ["december"] + assert suggest_subdomains("2024", "nl", 65) == ["december", "november"] + + with freeze_time("2023-12-30 12:00:00"): + # Now you will get everything from 2024, because in real life you cannot go back in time :) + assert suggest_subdomains("2024", "nl", 365) == [ + "april", + "augustus", + "december", + "februari", + "januari", + "juli", + "juni", + "maart", + "mei", + "november", + "oktober", + "september", ] + # testing a time-window + with freeze_time("2024-12-30 12:00:00"): + assert suggest_subdomains("2023", "nl", 10, "2023-10-01") == ["oktober"] + assert suggest_subdomains("2023", "nl", 65, "2023-10-01") == ["augustus", "oktober", "september"] + # data is not present yet in this period, so don't suggest it. with freeze_time("2025-12-30 12:00:00"): assert suggest_subdomains("example", "nl", 10) == []