Merge pull request #4 from internetstandards/docker

Docker

.github/workflows/build.yml

@@ -0,0 +1,49 @@
+name: QA and image building
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    environment: Default
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      # This helps with adding version information to the build in the admin on production. CI usually works shallow.
+      # https://stackoverflow.com/questions/4916492/git-describe-fails-with-fatal-no-names-found-cannot-describe-anything
+      # this will fetch the whole repository, which is more expensive.
+      - run: git fetch --prune --unshallow --tags
+
+      - name: build images
+        run: |
+          VERSION=$(git describe --tags --abbrev=0)-$(git log -1 --pretty=format:%h)
+          make build build_args=--build-arg=VERSION=${VERSION:-0.0.0-dev0}
+
+      - name: check
+        run: make check
+
+      - name: lint
+        run: make lint
+
+      - name: test
+        run: make test
+
+      - name: Login to Docker Hub
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push image to docker hub
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          make push_images
+        env:
+          DOCKER_USERNAME: internetstandards
+          secure: U7kSLhWutEWne8w0zMUN3XzWWWHnK0NKuDxrKQBtYpH5VA+ROd0js7y9uFYflqdJO/doiXE4updVM8D1aGrxfJ8Wp2Ywp5vC/6Ua+Wn+pdIi2WNBqOgCeq3+2NUMND7DpChFIA9dBiEOCnnOlRKYaz5bBhOR7trruV2E+QSCG+Scxejg+pToF/cRsQDI9zTktAr9rfWDfOjz0jfxC2NixOP8THFZhDW/vXlRWWGtmwvnawwXMdyhRYnIV/fvb2RnHeVlDXckMm+GTEcRwKZTp1Cy/lCuk8zEzxPkSvrYnAeC8Dto/KPFo9XnRNKNPygf4t7YlJ8eGvw7sp5QKH327D8+1kjspq20Aqq7IoHSHnhmYGrJSsgmoWsod6Be8PaFV/FjjfDbMvcpNspVxSPQu/h0P0POdK3+0HJBQ0j1qaYlOwlKGFyoNEN+hPOg7A/RWeV+P5wn5SxZCepL4T95uS4JIsb7SMmD1GxRTr+6ygRrKOnduBtBXFV48LALxvAwvBTka9muo924HDNsRQiI0r5gizoNPPTw+T8tzynoc7QRalih03UnEVvRxuPPz+kF+GyXQYLjVRIuPlrubnprOuW31VYEgwzAv32Ec3OZoUvx6MQvSvsignklSJgg9pucMu6apQPNkyt8GPRgUKUqQscPY1mSMhblWnaPAu7my9w=

.gitignore

@@ -10,3 +10,4 @@ suggestions_domain_25_10_2024*
 *export_merklemap_nl_zone_2024_20_25*
 .root/
 *.egg-info
+.build

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "vendor/certstream"]
 	path = vendor/certstream
-	url = git@github.com:CaliDog/certstream-server.git
+	url = https://github.com/CaliDog/certstream-server.git

Dockerfile

@@ -10,9 +10,6 @@ RUN pip install -r requirements.txt
 
 ADD README.md /src
 ADD pyproject.toml /src
-ADD src /src/src
-
-RUN pip install .
 
 FROM build AS dev
 
@@ -22,19 +19,29 @@ WORKDIR /src
 ADD requirements-dev.txt /src
 RUN pip install -r requirements-dev.txt
 
+ADD src /src/src
+RUN pip install --editable .
+
 ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings
 
 ENTRYPOINT [ "bash", "-c" ]
 CMD [ "bash" ]
 
 FROM build AS app
 
+ADD src /src/src
+RUN pip install --editable .
+
 ENV DJANGO_SETTINGS_MODULE ctlssa.app.settings
 ENV UWSGI_MODULE ctlssa.app.wsgi
-ENV UWSGI_HTTP_SOCKET=:8000
+ENV UWSGI_HTTP_SOCKET=:8001
 ENV UWSGI_MASTER=1
 ENV UWSGI_UID=nobody
+ENV DJANGO_PORT=8001
+
+EXPOSE 8001
 
-EXPOSE 8000
+ARG VERSION=0.0.0-dev0
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION
 
 ENTRYPOINT [ "ctlssa" ]

Makefile

@@ -1,38 +1,71 @@
 SHELL = /bin/bash
 
-ctlssa = docker compose run -ti app
-dev = docker compose run -ti -v ${PWD}/.root:/root --rm dev
+dev = docker compose --profile=dev run -i --rm dev
+db = docker compose exec -i db
 
 # run this before/after checking in/out the source
 all: build lint test
 
 # run the entire project
-run up: requirements
-	docker compose up --build --remove-orphans --watch
+run up:
+	docker compose up --remove-orphans --watch
 
 # make migration files
-makemigrations: build
-	${ctlssa} makemigrations
+makemigrations:
+	${dev} "ctlssa makemigrations"
+
+# load testdatta fixture
+testdata:
+	${dev} "ctlssa loaddata testdata"
 
 # run development shell
-dev dev-shell shell: build
+dev dev-shell shell:
 	${dev}
 
+dbshell:
+	${db} psql --user ctlssa
+
 # fix/check linting issues
-lint fix: build
+lint fix:
 	${dev} "isort ."
 	${dev} "black ."
 
 # run test suite
-test: build
-	${dev} "pytest ./tests/"
+test:
+	${dev} "pytest --verbose ."
 
-# build docker container images
-build: requirements
-	docker compose build
+test-watch:
+	${dev} "ptw --clear . -- --verbose"
+
+check: requirements
+	@if [ ! -z "$(shell git status --porcelain requirements*.txt)" ];then \
+	  echo "Requirements .in files have not all been compiled into .txt files and commited to Git!"; \
+	  git status --porcelain requirements*.txt; \
+	  exit 1; \
+	fi
+
+testcase:
+	# support extra verbosity in testcases so differences can be copy-pasted when needed and to see exactly
+	# what is happening.
+	# make testcase case=filter condition
+	${dev} "pytest -vvv -k ${case} ./tests/"
 
 # updates requirements.txt files when requirements.in files have changes
 requirements_files = $(subst .in,.txt,$(wildcard requirements*.in))
 requirements: ${requirements_files}
 ${requirements_files}: %.txt: %.in
-	${dev} pip-compile $< --output-file $@
+	${dev} "pip-compile $< --output-file $@"
+
+# build docker container images
+.PHONY: build
+build:
+	docker compose build ${build_args}
+
+push_images:
+	docker compose --push app certstream
+
+# remove all runtime state and cache from the project
+mrproper:
+	docker compose rm --volumes --force --stop
+	docker system prune --filter label=com.docker.compose.project="$(shell docker compose config --format yaml | sed -nE 's/^name: (.*)/\1/p')" --all --force --volumes
+	rm -fr build/ src/*.egg-info

README.md

@@ -22,12 +22,13 @@ fixture might be present. This contains 5.991.724 records. -> Git refused this,
 
 ## What are the limits of this tool
 The limits have not yet been discovered and no optimizations have been performed yet, aside from a few proactive
-database indexes. It is expected to being able to store about a years worth of data from the .nl zone. This means
-about 5 million domains with an estimated 50 million subdomains, each which will have a new certificate every 90 days.
-In total about 200 million records per year. This is the same in most EU countries. There is no expectation that this
-tool will work quickly on the combined com/net/org zones. Although some partitioning and smarter inserting might just
-do the trick. For the Netherlands the total number of certificate renewals seems to be much lower for subdomains,
-between 0.5 to 2 per second.
+database indexes. It is expected to being able to store about a years worth of data from the .nl zone. The preload
+from merklemap will load up 6 million subdomains (which is much lower than the expected 50 million).
+
+For the Netherlands the total number of certificate renewals seems to be much lower for subdomains,
+between 0.5 to 2 per second. Each subdomain which will have a new certificate every 90 days. This is the same in
+most EU countries. There is no expectation that this tool will work quickly on the combined com/net/org zones.
+Although some partitioning and smarter inserting might just do the trick.
 
 The goal is to being able to run this on medium sized virtual machines with just a few cores and a few gigabytes of
 ram. That should be enough for the Netherlands and most EU countries. We've not tried to see if this solution is 'web
@@ -52,8 +53,8 @@ This command should run forever. In case your certstream server is down it will
 The webserver can be started with the command:
 ```python manage.py runserver```
 
-When you visit the web interface at http://localhost:8000/ you will see a blank JSON response. Use the following
-parameters to retrieve data: `http://localhost:8000/?domain=example&suffix=nl&period=365`
+When you visit the web interface at http://localhost:8001/ you will see a blank JSON response. Use the following
+parameters to retrieve data: `http://localhost:8001/?domain=example&suffix=nl&period=365`
 
 
 ## Further configuration options
@@ -123,36 +124,89 @@ Requirements for development are:
 - Compose
 - GNU make
 
+### Checking out the code
+
+This repository contains submodules, so after the Git clone this submodule should be initialized.
+
+    git clone [email protected]:internetstandards/Internet.nl-ct-log-subdomain-suggestions-api.git
+    git submodule update --init
+
+### Building the application
+
+Before running the application, or whenever code changes, the Docker images need to be built. For this run:
+
+    make build
+
+Running the build step before every command is recommended to ensure the images are up to date, that's why it's included in all the command below. It can be omitted if you know you don't need it.
+
 ### Running application
 
-To run the application for development run:
+To run the application for development use:
+
+    make build run
+
+This will start a full development stack with:
 
-    make run
+  - uWSGI application
+  - ingest
+  - certstream server
+  - postgreSQL server
+  - database migration
 
-A web interface will be available at `http://localhost:8000`.
+A web interface will be available at `http://localhost:8001`.
+
+Compose will watch for file changes and restart the required services accordingly.
 
 ### Linting
 
 Run these commands before checking in. These should all pass without error.
 
-    make lint
+    make build lint
 
 *notice*: this command autofixes trivial issues in formatting and updates the source files
 
 ### Testing
 
 To run the test suite use:
 
-    make test
+    make build test
+
+To rerun tests every time a file changes use:
+
+    make build test-watch
 
 ### Development shell
 
-To open a shell with all dependencies and development tools installed run:
+To open a shell with all dependencies and development tools installed, first bring the project up, then run:
 
-    make dev
+    make build dev
+
+From here you can run the `ctlssa` command to perform Django `./manage.py` functions like:
+
+    ctlssa makemigrations
+
+    ctlssa shell
+
+    ctlssa loadfixtures testdata
+
+    ctlssa bulk_ingest --file data.json
+
+For all commands run:
+
+    ctlssa --help
 
 ### Dependency management
 
 After changing requirements in any of the `.in` files update the `.txt` files using:
 
-    make requirements
+    make build requirements
+
+### Database shell (postgresql)
+
+    make dbshell
+
+### Reset
+
+If nothing else works, try resetting the project state by deleting all runtime state and caches using:
+
+    make mrproper