Major cleanups and improvements (#12)

Makefile

@@ -10,7 +10,7 @@ DOCKER ?= $(shell which docker)
 PYTHON ?= $(shell which python3)
 
 # Python virtual environment paths
-VENV_PATH := .venv
+VENV_PATH := $(abspath .venv)
 
 # Python executable paths
 PIP := $(VENV_PATH)/bin/pip
@@ -20,10 +20,10 @@ PYLINT := $(VENV_PATH)/bin/pylint
 PYTHON := $(VENV_PATH)/bin/python
 
 # All paths
-IMAGE_PATH := image
-BUNDLES_PATH := bundles
-EXAMPLES_PATH := examples
-RESOURCES_PATH := resources
+IMAGE_PATH := $(abspath image)
+BUNDLES_PATH := $(abspath bundles)
+EXAMPLES_PATH := $(abspath examples)
+RESOURCES_PATH := $(abspath resources)
 
 # Additional resources
 TESTS_PATH := $(RESOURCES_PATH)/tests
@@ -33,7 +33,7 @@ SCRIPTS_PATH := $(RESOURCES_PATH)/scripts
 BACKEND_PATH := $(IMAGE_PATH)/src/backend
 FRONTEND_PATH := $(IMAGE_PATH)/src/frontend
 ENTRYPOINT_PATH := $(IMAGE_PATH)/src/entrypoint.py
-REQUIREMENTS_PATH := $(IMAGE_PATH)/requirements.txt
+REQUIREMENTS_PATH := $(IMAGE_PATH)/resources/requirements.txt
 
 # Bundle paths
 HEADLESS_BUNDLE_PATH := $(BUNDLES_PATH)/headless
@@ -66,11 +66,11 @@ checks: format lint typecheck
 
 lint: $(PYLINT) $(PYTHON_SOURCES)
 	@# Lint all of the sources
-	$(PYLINT) -d C0301 -d C0114 -d C0115 -d C0116 $(PYTHON_SOURCES)
+	cd $(BACKEND_PATH); $(PYLINT) -d C0301 -d C0114 -d C0115 -d C0116 -d W0401 $(PYTHON_SOURCES)
 
 typecheck: $(MYPY) $(PYTHON_SOURCES)
 	@# Typecheck all of the sources
-	$(MYPY) --strict --explicit-package-bases --no-implicit-reexport $(PYTHON_SOURCES)
+	cd $(BACKEND_PATH); $(MYPY) --cache-dir=/dev/null --explicit-package-bases --no-implicit-reexport $(PYTHON_SOURCES)
 
 format: $(YAPF) $(PYTHON_SOURCES)
 	@# Format the python sources using yapf
@@ -103,7 +103,7 @@ $(VENV_PATH): $(REQUIREMENTS_PATH)
 	python3 -m venv $(VENV_PATH)
 
 	@# Install some dependencies
-	$(PIP) install -r $(REQUIREMENTS_PATH) jinja2 yapf mypy pylint
+	$(PIP) install -r $(REQUIREMENTS_PATH) jinja2 yapf mypy pylint munch-stubs
 
 $(YAPF): $(VENV_PATH)
 $(MYPY): $(VENV_PATH)

README.md

@@ -37,10 +37,10 @@ git commit -m "Initial commit"
 
 Webhood is based on popular projects and strives to keep the application architecture simple efficient.
 
-1. Web server duties are handled by [NGINX](https://nginx.org/). NGINX serves as a static file server and as a reverse-proxy for the backend API. NGINX also handles TLS.
-2. Python backend is powered by [Starlette](https://www.starlette.io/) - an open-source WSGI framework that is the basis for many open-source projects. In our case, Starlette is extended by the [`utilities/starlette.py`](https://github.com/NadavTasher/Webhood/blob/master/image/src/backend/utilities/starlette.py) file.
-3. Database duties are handled by [Redis](https://redis.io/). A Redis server can be easily accessed using the [rednest](https://pypi.org/rednest/) library.
-4. Frontend duties are handled by a couple of utility JavaScript and CSS files residing in [src/frontend](https://github.com/NadavTasher/Webhood/tree/master/image/src/frontend). You can see an example of the frontend capabilities in the [Headless Test Page](https://github.com/NadavTasher/Webhood/blob/master/bundles/headless/test-page.html).
+1. Python backend is powered by [Starlette](https://www.starlette.io/) - an open-source WSGI framework. See [usage](https://github.com/NadavTasher/Webhood/blob/master/image/src/backend/webhood/router.py).
+1. Web server duties are handled by [Gunicorn](https://gunicorn.org/) - an open-source WSGI server. See [usage](https://github.com/NadavTasher/Webhood/blob/master/image/resources/entrypoint.conf).
+2. Database duties are handled by [Redis](https://redis.io/) using the [rednest](https://pypi.org/rednest/) library.
+3. Frontend duties are handled by custom JS and CSS files in [src/frontend](https://github.com/NadavTasher/Webhood/tree/master/image/src/frontend). An example can be seen [here](https://github.com/NadavTasher/Webhood/blob/master/bundles/headless/test-page.html).
 
 ## Examples
 
@@ -54,7 +54,7 @@ You can easily spin up one of the example applications found [here](https://gith
 
 By default, the color scheme is defined by the system configuration.
 
-This can be disabled by excluding the `/stylesheets/colors.css` file from your page, and create a custom color scheme like so:
+This behaviour can be disabled - exclude the `/stylesheets/colors.css` file from your page, and create a custom color scheme:
 
 ```css
 :root {
@@ -341,7 +341,7 @@ import hashlib
 
 from runtypes import Optional, Text, ByteString
 
-from utilities.starlette import PlainTextResponse, router
+from webhood.router import PlainTextResponse, router
 
 
 @router.get("/api/code")
@@ -418,11 +418,11 @@ import hashlib
 
 from runtypes import Text
 
-from utilities.starlette import router
+from webhood.router import router
 
 
 @router.socket("/socket/notifications")
-async def notifications_socket(websocket, id: Text):
+async def notifications_socket(websocket, id: Text) -> None:
 	# Run additional validations here...
 
 	# Accept the client
@@ -451,8 +451,8 @@ Note that for database backups to take place, a Docker volume must be mounted on
 ```python
 import hashlib
 
-from utilities.redis import wait_for_redis_sync, redict
-from utilities.starlette import router
+from webhood.router import router
+from webhood.database import wait_for_redis_sync, redict
 
 # Wait for redis to ping back before operating on database
 wait_for_redis_sync()
@@ -482,8 +482,8 @@ The [`utilities/redis.py`](https://github.com/NadavTasher/Webhood/blob/master/im
 ```python
 import hashlib
 
-from utilities.redis import wait_for_redis_sync, broadcast_sync, receive_async, redict
-from utilities.starlette import router
+from webhood.router import router
+from webhood.database import wait_for_redis_sync, broadcast_sync, receive_async, redict
 
 # Wait for redis to ping back before operating on database
 wait_for_redis_sync()
@@ -526,31 +526,14 @@ async def notify_clicks(websocket):
 app = router.initialize()
 ```
 
-### Configuration
-
-Configurations can be easily extended, using several `conf.d` directories inside of the container.
-
-NGINX configuration can be extended through `/etc/nginx/conf.d/`.
-
-Entrypoint configuration can be extended using `/etc/entrypoint/conf.d/`.
-
-## Quirks
-
 ### Container entrypoint
 
-The entrypoint of the container (the default built image), is `entrypoint.py`.
-
-This entrypoint is a simple Python process watcher. It stops when any one of it's direct children exits.
-
-It can be configured to spawn new services and can be patched to change the default configuration using it's `conf.d` directory.
-
-A sample configuration can be found [here](https://github.com/NadavTasher/Webhood/blob/master/image/configurations/entrypoint.conf).
-
-### Creating an in-mem application
+The entrypoint of the image, is [`entrypoint.py`](https://github.com/NadavTasher/Webhood/blob/master/image/src/entrypoint.conf).
 
-If you do not want to use the pre-bundled `rednest` library to create a Redis persistent storage key-value store, you might want to use a regular `dict()` as a way to temporarly store globals.
+This entrypoint is a simple Python process watcher.
+It stops when one of it's direct children exits, and terminates the others.
 
-To make this possible, you might need to extend the `entrypoint` configuration to tell `gunicorn` to only spawn a single worker. That way all of the requests will be handled by the same process with the same globals.
+A sample configuration can be found [here](https://github.com/NadavTasher/Webhood/blob/master/image/resources/entrypoint.conf).
 
 ## Contributing
 

bundles/buildless/docker-compose.yaml

@@ -3,7 +3,7 @@ version: "3"
 # Configure applcation service
 services: 
     application:
-        image: webhood/3.8
+        image: webhood/3.10
         restart: unless-stopped
         ports: 
             - 80:80
@@ -14,9 +14,6 @@ services:
             - ./src/backend/worker.py:/application/backend/worker.py:ro
             - ./src/frontend/index.html:/application/frontend/index.html:ro
             - ./src/frontend/application:/application/frontend/application:ro
-            # Configuration mounts
-            - ./configurations/nginx.conf:/etc/nginx/conf.d/nginx.conf:ro
-            - ./configurations/entrypoint.conf:/etc/entrypoint/conf.d/entrypoint.conf:ro
             # Data volume path
             - data:/data
         environment:

bundles/buildless/src/backend/app.py

@@ -1,12 +1,15 @@
+# mypy: disable-error-code=valid-type
+# pylint: disable=no-member, unused-wildcard-import
+
 import logging
 
 # Import utilities
 from runtypes import *
 from guardify import *
 
 # Import the router
-from utilities.redis import wait_for_redis_sync, broadcast_sync, broadcast_async, receive_sync, receive_async, redict
-from utilities.starlette import WebSocket, router
+from webhood.router import router
+from webhood.database import wait_for_redis_sync, broadcast_async, receive_async, redict
 
 # Wait for redis to ping back before operating on database
 wait_for_redis_sync()
@@ -39,7 +42,7 @@ async def relay_request(message: str, sender: Optional[Email] = None) -> None:
 
 
 @router.socket("/socket/relay")
-async def relay_socket(websocket: WebSocket) -> None:
+async def relay_socket(websocket) -> None:
     # Accept the websocket
     await websocket.accept()
 

bundles/independent/Makefile

@@ -27,4 +27,7 @@ push: image
 endif
 
 image: $(IMAGE_SOURCES)
-	@docker build $(IMAGE_PATH) -t $(IMAGE_NAME):$(IMAGE_TAG)
+	@docker build $(IMAGE_PATH) -t $(IMAGE_NAME):$(IMAGE_TAG)
+
+develop: image
+	@docker compose up

bundles/independent/application/Dockerfile

@@ -1,9 +1,5 @@
 # Select the base image
-FROM webhood/3.8
-
-# Copy configurations
-COPY configurations/nginx.conf /etc/nginx/conf.d/nginx.conf
-COPY configurations/entrypoint.conf /etc/entrypoint/conf.d/entrypoint.conf
+FROM webhood/3.10
 
 # Copy sources
 COPY src /application

bundles/independent/application/src/backend/app.py

@@ -1,12 +1,15 @@
+# mypy: disable-error-code=valid-type
+# pylint: disable=no-member, unused-wildcard-import
+
 import logging
 
 # Import utilities
 from runtypes import *
 from guardify import *
 
 # Import the router
-from utilities.redis import wait_for_redis_sync, broadcast_sync, broadcast_async, receive_sync, receive_async, redict
-from utilities.starlette import WebSocket, router
+from webhood.router import router
+from webhood.database import wait_for_redis_sync, broadcast_async, receive_async, redict
 
 # Wait for redis to ping back before operating on database
 wait_for_redis_sync()
@@ -39,7 +42,7 @@ async def relay_request(message: str, sender: Optional[Email] = None) -> None:
 
 
 @router.socket("/socket/relay")
-async def relay_socket(websocket: WebSocket) -> None:
+async def relay_socket(websocket) -> None:
     # Accept the websocket
     await websocket.accept()
 

bundles/independent/docker-compose.yaml

@@ -10,7 +10,7 @@ services:
             - 443:443
         volumes:
             # Data volume path
-            - data:/opt
+            - data:/data
         environment:
             # Address to redis instance
             - REDIS=redis://redis/

image/Dockerfile

@@ -4,30 +4,30 @@ ARG PYTHON_VERSION=3.10
 # Select the base image
 FROM python:${PYTHON_VERSION}-slim-bookworm
 
+# Expose the HTTP and HTTPs ports
+EXPOSE 80 443
+
+# Mark /data as a volume directory
+VOLUME /data
+
+# Specify stop signal
+STOPSIGNAL SIGINT
+
 # Copy the requirements file
-COPY requirements.txt /tmp/requirements.txt
+COPY resources/requirements.txt /tmp/requirements.txt
 
 # Upgrade pip and install dependencies
 RUN pip install -U -r /tmp/requirements.txt pip ipython
 
 # Copy default configurations
-COPY configurations/entrypoint.conf /etc/entrypoint/entrypoint.conf
+COPY resources/entrypoint.conf /etc/entrypoint.conf
 
 # Copy the sources
 COPY src /application
 
-# Expose the HTTP and HTTPs ports
-EXPOSE 80 443
-
-# Mark /data as a volume directory
-VOLUME /data
-
 # Change working directory to application
 WORKDIR /application/backend
 
-# Specify stop signal
-STOPSIGNAL SIGINT
-
 # Set the entrypoint to the entrypoint script
 CMD [ "python", "/application/entrypoint.py" ]
 

image/configurations/entrypoint.conf → image/resources/entrypoint.conf

@@ -1,14 +1,14 @@
 [worker]
+signal=SIGINT
 command=python worker.py
 directory=/application/backend
 
 [http]
+signal=SIGTERM
 command=gunicorn --workers 4 --forwarded-allow-ips * --worker-class uvicorn.workers.UvicornWorker --access-logfile /dev/stdout --bind 0.0.0.0:80 app:app
 directory=/application/backend
 
 [https]
+signal=SIGTERM
 command=gunicorn --workers 4 --forwarded-allow-ips * --worker-class uvicorn.workers.UvicornWorker --access-logfile /dev/stdout --bind 0.0.0.0:443 --certfile /etc/ssl/private/server.crt --keyfile /etc/ssl/private/server.key app:app
-directory=/application/backend
-
-[include]
-files = /etc/entrypoint/conf.d/*.conf
+directory=/application/backend