Updates to starlette, typing and build system

Makefile

@@ -6,67 +6,112 @@ IMAGE_LATEST_TAG ?= $(IMAGE_TAG):latest
 
 COPY ?= $(shell which cp)
 MKDIR ?= $(shell which mkdir)
-PYTHON ?= $(shell which python3)
 DOCKER ?= $(shell which docker)
+PYTHON ?= $(shell which python3)
+
+# Python virtual environment paths
+VENV_PATH := .venv
 
+# Python executable paths
+PIP := $(VENV_PATH)/bin/pip
+YAPF := $(VENV_PATH)/bin/yapf
+MYPY := $(VENV_PATH)/bin/mypy
+PYLINT := $(VENV_PATH)/bin/pylint
+PYTHON := $(VENV_PATH)/bin/python
+
+# All paths
 IMAGE_PATH := image
 BUNDLES_PATH := bundles
 EXAMPLES_PATH := examples
 RESOURCES_PATH := resources
 
+# Additional resources
 TESTS_PATH := $(RESOURCES_PATH)/tests
 SCRIPTS_PATH := $(RESOURCES_PATH)/scripts
 
+# Source paths
 BACKEND_PATH := $(IMAGE_PATH)/src/backend
 FRONTEND_PATH := $(IMAGE_PATH)/src/frontend
 ENTRYPOINT_PATH := $(IMAGE_PATH)/src/entrypoint.py
+REQUIREMENTS_PATH := $(IMAGE_PATH)/requirements.txt
 
+# Bundle paths
 HEADLESS_BUNDLE_PATH := $(BUNDLES_PATH)/headless
 BUILDLESS_BUNDLE_PATH := $(BUNDLES_PATH)/buildless
 INDEPENDENT_BUNDLE_PATH := $(BUNDLES_PATH)/independent
 
+# Headless bundle source paths
 HEADLESS_BUNDLE_INDEX_PATH := $(HEADLESS_BUNDLE_PATH)/index.html
 HEADLESS_BUNDLE_TEST_PAGE_PATH := $(HEADLESS_BUNDLE_PATH)/test-page.html
 
+# Buildless bundle source paths
 BUILDLESS_BUNDLE_BACKEND_PATH := $(BUILDLESS_BUNDLE_PATH)/src/backend
 BUILDLESS_BUNDLE_FRONTEND_PATH := $(BUILDLESS_BUNDLE_PATH)/src/frontend
 
+# Independent bundle source paths
 INDEPENDENT_BUNDLE_BACKEND_PATH := $(INDEPENDENT_BUNDLE_PATH)/application/src/backend
 INDEPENDENT_BUNDLE_FRONTEND_PATH := $(INDEPENDENT_BUNDLE_PATH)/application/src/frontend
 
+# All image sources
 IMAGE_SOURCES := $(shell find $(IMAGE_PATH) -type f)
-PYTHON_SOURCES := $(wildcard $(BACKEND_PATH)/*.py) $(wildcard $(BACKEND_PATH)/*/*.py) $(ENTRYPOINT_PATH) $(wildcard $(EXAMPLES_PATH)/*/application/src/backend/*.py) $(wildcard $(SCRIPTS_PATH)/*.py)
+
+# All python sources
+PYTHON_SOURCES := $(wildcard $(BACKEND_PATH)/*.py) $(wildcard $(BACKEND_PATH)/*/*.py) $(ENTRYPOINT_PATH) $(wildcard $(SCRIPTS_PATH)/*.py)
 
 all: bundles image
 
-prerequisites:
-	$(PYTHON) -m pip install jinja2 yapf
+# Linting and checks
 
-format: prerequisites $(PYTHON_SOURCES)
-	$(PYTHON) -m yapf -i $(PYTHON_SOURCES) --style "{based_on_style: google, column_limit: 400, indent_width: 4}"
+checks: format lint typecheck
 
-$(IMAGE_PATH)/Dockerfile-$(IMAGE_TAG): $(SCRIPTS_PATH)/create_dockerfile.py $(IMAGE_PATH)/Dockerfile.template
-	$(MKDIR) -p $(@D)
-	$(PYTHON) $(SCRIPTS_PATH)/create_dockerfile.py --python-version $(IMAGE_TAG) < $(IMAGE_PATH)/Dockerfile.template > $@
+lint: $(PYLINT) $(PYTHON_SOURCES)
+	@# Lint all of the sources
+	$(PYLINT) -d C0301 -d C0114 -d C0115 -d C0116 $(PYTHON_SOURCES)
 
-image: format $(IMAGE_PATH)/Dockerfile-$(IMAGE_TAG) $(IMAGE_SOURCES)
-	$(DOCKER) build $(IMAGE_PATH) -f $(IMAGE_PATH)/Dockerfile-$(IMAGE_TAG) -t $(IMAGE_NAME)/$(IMAGE_TAG) -t $(IMAGE_NAME)/$(IMAGE_DATE_TAG) -t $(IMAGE_NAME)/$(IMAGE_LATEST_TAG)
+typecheck: $(MYPY) $(PYTHON_SOURCES)
+	@# Typecheck all of the sources
+	$(MYPY) --strict --explicit-package-bases --no-implicit-reexport $(PYTHON_SOURCES)
 
-buildx: format $(IMAGE_PATH)/Dockerfile-$(IMAGE_TAG) $(IMAGE_SOURCES)
+format: $(YAPF) $(PYTHON_SOURCES)
+	@# Format the python sources using yapf
+	$(YAPF) -i $(PYTHON_SOURCES) --style "{based_on_style: google, column_limit: 400, indent_width: 4}"
+
+# Images
+
+image: format $(IMAGE_SOURCES)
+	@# Build the image
+	$(DOCKER) build --build-arg PYTHON_VERSION=$(IMAGE_TAG) $(IMAGE_PATH) -t $(IMAGE_NAME)/$(IMAGE_TAG) -t $(IMAGE_NAME)/$(IMAGE_DATE_TAG) -t $(IMAGE_NAME)/$(IMAGE_LATEST_TAG)
+
+buildx: format $(IMAGE_SOURCES)
+	@# Create the build context
 	$(DOCKER) buildx create --use
-	$(DOCKER) buildx build $(IMAGE_PATH) --push --platform linux/386,linux/amd64,linux/arm64/v8 -f $(IMAGE_PATH)/Dockerfile-$(IMAGE_TAG) -t $(IMAGE_NAME)/$(IMAGE_DATE_TAG) -t $(IMAGE_NAME)/$(IMAGE_LATEST_TAG)
 
-clean:
-	$(RM) $(IMAGE_PATH)/Dockerfile-*
+	@# Build the image
+	$(DOCKER) buildx build --build-arg PYTHON_VERSION=$(IMAGE_TAG) $(IMAGE_PATH) --push --platform linux/386,linux/amd64,linux/arm64/v8 -t $(IMAGE_NAME)/$(IMAGE_DATE_TAG) -t $(IMAGE_NAME)/$(IMAGE_LATEST_TAG)
 
-bundles: headless buildless independent
+# Bundles
 
+bundles: headless buildless independent
 headless: $(HEADLESS_BUNDLE_INDEX_PATH) $(HEADLESS_BUNDLE_TEST_PAGE_PATH)
-
 buildless: $(BUILDLESS_BUNDLE_BACKEND_PATH)/app.py $(BUILDLESS_BUNDLE_BACKEND_PATH)/worker.py $(BUILDLESS_BUNDLE_FRONTEND_PATH)/index.html $(BUILDLESS_BUNDLE_FRONTEND_PATH)/application/application.css $(BUILDLESS_BUNDLE_FRONTEND_PATH)/application/application.js
-
 independent: $(INDEPENDENT_BUNDLE_BACKEND_PATH)/app.py $(INDEPENDENT_BUNDLE_BACKEND_PATH)/worker.py $(INDEPENDENT_BUNDLE_FRONTEND_PATH)/index.html $(INDEPENDENT_BUNDLE_FRONTEND_PATH)/application/application.css $(INDEPENDENT_BUNDLE_FRONTEND_PATH)/application/application.js
 
+# Local prerequisites
+
+$(VENV_PATH): $(REQUIREMENTS_PATH)
+	@# Create a new virtual environment
+	python3 -m venv $(VENV_PATH)
+
+	@# Install some dependencies
+	$(PIP) install -r $(REQUIREMENTS_PATH) jinja2 yapf mypy pylint
+
+$(YAPF): $(VENV_PATH)
+$(MYPY): $(VENV_PATH)
+$(PYLINT): $(VENV_PATH)
+$(PYTHON): $(VENV_PATH)
+
+# Targets to make
+
 $(INDEPENDENT_BUNDLE_BACKEND_PATH)/%.py: $(BACKEND_PATH)/%.py
 	$(MKDIR) -p $(@D)
 	$(COPY) $^ $@
@@ -99,6 +144,8 @@ $(HEADLESS_BUNDLE_TEST_PAGE_PATH): $(TESTS_PATH)/test-page.html $(SCRIPTS_PATH)/
 	$(MKDIR) -p $(@D)
 	$(PYTHON) $(SCRIPTS_PATH)/create_headless_page.py --base-directory $(FRONTEND_PATH) < $(TESTS_PATH)/test-page.html > $@
 
+# Tests
+
 test: image
 	$(DOCKER) run --rm -p 80:80 -p 443:443 -e DEBUG=1 -e REDIS=$(REDIS) -v $(abspath $(TESTS_PATH)/test-page.html):/application/frontend/index.html:ro -v /tmp/test:/opt $(IMAGE_NAME)/$(IMAGE_TAG)
 
@@ -115,4 +162,9 @@ test-independent: independent
 	$(DOCKER) compose --project-directory $(INDEPENDENT_BUNDLE_PATH) up --build
 
 test-independent-build: independent
-	$(MAKE) -C $(INDEPENDENT_BUNDLE_PATH)
+	$(MAKE) -C $(INDEPENDENT_BUNDLE_PATH)
+
+# Cleanups
+
+clean:
+	$(RM) -r $(VENV_PATH) $(IMAGE_PATH)/Dockerfile-*

image/Dockerfile.template → image/Dockerfile

@@ -1,18 +1,16 @@
+# The default python version is 3.10
+ARG PYTHON_VERSION=3.10
+
 # Select the base image
-FROM python:{{PYTHON_VERSION}}-slim-bookworm AS python
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
-# Make the debian frontend non-interactive for apt installations
-ENV DEBIAN_FRONTEND=noninteractive
+# Copy the requirements file
+COPY requirements.txt /tmp/requirements.txt
 
 # Upgrade pip and install dependencies
-RUN pip install --upgrade \ 
-	pip ipython \
-	uvicorn==0.30.3 gunicorn==22.0.0 \
-	starlette==0.38.2 websockets==12.0 python-multipart==0.0.9 \
-	redis==5.0.8 hiredis==3.0.0 munch==4.0.0 rednest==0.5.0 runtypes==0.6.1 guardify==0.2.3
+RUN pip install -U -r /tmp/requirements.txt pip ipython
 
 # Copy default configurations
-COPY configurations/gunicorn.conf /etc/gunicorn/gunicorn.conf
 COPY configurations/entrypoint.conf /etc/entrypoint/entrypoint.conf
 
 # Copy the sources

image/configurations/entrypoint.conf

@@ -3,11 +3,11 @@ command=python worker.py
 directory=/application/backend
 
 [http]
-command=gunicorn --workers 4 --forwarded-allow-ips * --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:80 app:app
+command=gunicorn --workers 4 --forwarded-allow-ips * --worker-class uvicorn.workers.UvicornWorker --access-logfile /dev/stdout --bind 0.0.0.0:80 app:app
 directory=/application/backend
 
 [https]
-command=gunicorn --workers 4 --forwarded-allow-ips * --worker-class uvicorn.workers.UvicornWorker --bind 0.0.0.0:443 --certfile=/etc/ssl/private/server.crt --keyfile=/etc/ssl/private/server.key app:app
+command=gunicorn --workers 4 --forwarded-allow-ips * --worker-class uvicorn.workers.UvicornWorker --access-logfile /dev/stdout --bind 0.0.0.0:443 --certfile /etc/ssl/private/server.crt --keyfile /etc/ssl/private/server.key app:app
 directory=/application/backend
 
 [include]

image/requirements.txt

@@ -0,0 +1,11 @@
+uvicorn==0.30.3
+gunicorn==22.0.0
+starlette==0.38.2
+websockets==12.0
+python-multipart==0.0.9
+redis==5.0.8
+hiredis==3.0.0
+munch==4.0.0
+rednest==0.5.1
+runtypes==0.6.1
+guardify==0.2.3

image/src/backend/app.py

@@ -6,7 +6,7 @@
 
 # Import the router
 from utilities.redis import wait_for_redis_sync, broadcast_sync, broadcast_async, receive_sync, receive_async, redict
-from utilities.starlette import router
+from utilities.starlette import WebSocket, router
 
 # Wait for redis to ping back before operating on database
 wait_for_redis_sync()
@@ -25,7 +25,7 @@ def click_request() -> str:
     logging.info("User clicked - count is now %d", DATABASE.count)
 
     # Return the ping count
-    return "Click count is %d" % DATABASE.count
+    return f"Click count is {DATABASE.count}"
 
 
 @router.post("/api/relay")
@@ -39,7 +39,7 @@ async def relay_request(message: str, sender: Optional[Email] = None) -> None:
 
 
 @router.socket("/socket/relay")
-async def relay_socket(websocket) -> None:
+async def relay_socket(websocket: WebSocket) -> None:
     # Accept the websocket
     await websocket.accept()
 

image/src/backend/utilities/debug.py

@@ -1,4 +1,4 @@
 import os
 
 # Get debug state
-DEBUG = bool(int(os.environ.get("DEBUG", 0)))
+DEBUG = bool(int(os.environ.get("DEBUG", 0)))

image/src/backend/utilities/redis.py

@@ -1,10 +1,13 @@
 import os
 import time
 import json
-import munch
+import typing
 import asyncio
 import contextlib
 
+# Import munch utility
+import munch
+
 # Import redis utilities
 import redis
 import redis.asyncio
@@ -23,14 +26,21 @@
 REDIS_ASYNC = redis.asyncio.Redis.from_url(REDIS_URL, decode_responses=True)
 
 # Patch the dictionary copy type
+# pylint: disable-next=protected-access
 Dictionary._COPY_TYPE = munch.Munch
 
+
 # Create wrapper functions for databases
-relist = lambda name: List(REDIS_SYNC, name)
-redict = lambda name: Dictionary(REDIS_SYNC, name)
+def relist(name: str) -> List:
+    return List(REDIS_SYNC, name)
+
+
+def redict(name: str) -> Dictionary:
+    return Dictionary(REDIS_SYNC, name)
 
 
-def wait_for_redis_sync():
+# Functions to wait for redis
+def wait_for_redis_sync() -> None:
     # Initialize ping response
     ping_response = None
 
@@ -45,7 +55,7 @@ def wait_for_redis_sync():
         time.sleep(1)
 
 
-async def wait_for_redis_async():
+async def wait_for_redis_async() -> None:
     # Initialize ping response
     ping_response = None
 
@@ -60,22 +70,22 @@ async def wait_for_redis_async():
         await asyncio.sleep(1)
 
 
-def broadcast_sync(channel=GLOBAL_CHANNEL, redis=REDIS_SYNC, **parameters):
+def broadcast_sync(channel: str = GLOBAL_CHANNEL, connection: redis.Redis = REDIS_SYNC, **parameters: typing.Any) -> None:
     # Publish to channel
-    redis.publish(channel, json.dumps(parameters))
+    connection.publish(channel, json.dumps(parameters))
 
 
-async def broadcast_async(channel=GLOBAL_CHANNEL, redis=REDIS_ASYNC, **parameters):
+async def broadcast_async(channel: str = GLOBAL_CHANNEL, connection: redis.Redis = REDIS_ASYNC, **parameters: typing.Any) -> None:
     # Publish to channel
-    await redis.publish(channel, json.dumps(parameters))
+    await connection.publish(channel, json.dumps(parameters))
 
 
-def receive_sync(channel=GLOBAL_CHANNEL, redis=REDIS_SYNC, count=0):
+def receive_sync(channel: str = GLOBAL_CHANNEL, connection: redis.Redis = REDIS_SYNC, count: int = 0) -> munch.Munch:
     # Count messages
     received = 0
 
     # Create Pub / Sub subscriber
-    with redis.pubsub() as subscriber:
+    with connection.pubsub() as subscriber:
         # Subscribe to channel
         subscriber.subscribe(channel)
 
@@ -102,12 +112,12 @@ def receive_sync(channel=GLOBAL_CHANNEL, redis=REDIS_SYNC, count=0):
             received += 1
 
 
-async def receive_async(channel=GLOBAL_CHANNEL, redis=REDIS_ASYNC, count=0):
+async def receive_async(channel: str = GLOBAL_CHANNEL, connection: redis.Redis = REDIS_ASYNC, count: int = 0) -> munch.Munch:
     # Count messages
     received = 0
 
     # Create Pub / Sub subscriber
-    async with redis.pubsub() as subscriber:
+    async with connection.pubsub() as subscriber:
         # Subscribe to channel
         await subscriber.subscribe(channel)