From 9705e8439c41a95f46986a51f445c2feded4e494 Mon Sep 17 00:00:00 2001 From: Benno Date: Mon, 6 Jan 2025 22:02:26 +0100 Subject: [PATCH] Add Postgres container and use it. Run Teamspeak recorder in separate process as Django manage.py command in run.sh script. --- .gitignore | 4 +- Dockerfile | 2 - Spybot2/settings.py | 2 +- docker-compose-local.yml | 17 +++++++- infrastructure/docker-compose.yml | 12 ++++++ pyproject.toml | 2 +- run.sh | 5 +++ spybot/apps.py | 3 -- spybot/management/commands/recorder.py | 14 +++++++ spybot/recorder/recorder.py | 5 --- uv.lock | 58 ++++++++++++++++++++------ 11 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 spybot/management/commands/recorder.py diff --git a/.gitignore b/.gitignore index b71317f..dfd8cfb 100644 --- a/.gitignore +++ b/.gitignore @@ -142,4 +142,6 @@ GitHub.sublime-settings # frontend node_modules/ -frontend/output \ No newline at end of file +frontend/output + +secrets/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0a68ffe..fc16c9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,6 @@ ENV PYTHONUNBUFFERED=1 EXPOSE 8000 -# Install mysqlclient debian package dependencies -RUN apt-get update && apt-get install -y --no-install-recommends python3-dev default-libmysqlclient-dev build-essential pkg-config && rm -rf /var/lib/apt/lists/* RUN pip install uv COPY pyproject.toml pyproject.toml RUN uv sync diff --git a/Spybot2/settings.py b/Spybot2/settings.py index f30db77..760d020 100644 --- a/Spybot2/settings.py +++ b/Spybot2/settings.py @@ -50,7 +50,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env.bool('DEBUG', False) -ALLOWED_HOSTS = [SERVER_IP, TS_IP, 'localhost', '127.0.0.1', 'spybot.localhost.direct'] +ALLOWED_HOSTS = [SERVER_IP, TS_IP, 'localhost', '127.0.0.1', 'spybot.localhost.direct', '192.168.59.100'] CSRF_TRUSTED_ORIGINS = [f"https://{SERVER_IP}"] diff --git a/docker-compose-local.yml b/docker-compose-local.yml index aad56bb..61d8bf3 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -5,7 +5,7 @@ services: restart: unless-stopped volumes: - static_files:/spybot_static - - $PWD/.env:/.env + - $PWD/.env:/app/.env expose: - 8000 caddy: @@ -18,13 +18,26 @@ services: # - "443:443" # - "443:443/udp" volumes: - - $PWD/Caddyfile:/etc/caddy/Caddyfile + - $PWD/infrastructure/Caddyfile:/etc/caddy/Caddyfile - caddy_data:/data - caddy_config:/config - static_files:/spybot_static + db: + image: postgres:17.2 + restart: always + environment: + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + volumes: + - pg_data:/var/lib/postgresql/data + secrets: + - db_password volumes: caddy_data: caddy_config: static_files: + pg_data: +secrets: + db_password: + file: secrets/db_password.txt diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 68146c5..4a9fcbb 100755 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -19,9 +19,21 @@ services: - caddy_data:/data - caddy_config:/config - static_files:/spybot_static + db: + image: postgres:17.2 + restart: always + environment: + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + volumes: + - spybot_pg_data:/var/lib/postgresql/data + secrets: + - db_password volumes: caddy_data: caddy_config: static_files: +secrets: + db_password: + file: secrets/db_password.txt diff --git a/pyproject.toml b/pyproject.toml index 57c2954..ea429a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ dependencies = [ "Django ~=5.1", "ts3 ~=2.0.0b3", "django-environ ~=0.11.2", - "mysqlclient ~=2.2", "django-crontab ~=0.7.1", "num2words ~=0.5.12", "requests ~=2.31", @@ -22,6 +21,7 @@ dependencies = [ "django-bootstrap5 ~=24.2", "sentry-sdk>=2.13.0", "gunicorn>=23.0.0", + "psycopg2-binary>=2.9.10", ] diff --git a/run.sh b/run.sh index 2a77faf..1165538 100755 --- a/run.sh +++ b/run.sh @@ -10,5 +10,10 @@ ls -a # run DB migrations if necessary .venv/bin/python manage.py migrate +# start recorder in background and terminate on script exit +.venv/bin/python manage.py recorder & +RECORDER_JOB=$! +trap 'kill $RECORDER_JOB' EXIT HUP TERM INT + # run django app .venv/bin/gunicorn -w 2 --bind 0.0.0.0:8000 Spybot2.wsgi diff --git a/spybot/apps.py b/spybot/apps.py index 5b546ef..0daa558 100644 --- a/spybot/apps.py +++ b/spybot/apps.py @@ -9,7 +9,4 @@ class SpybotConfig(AppConfig): def ready(self): if os.environ.get('RUN_MAIN'): - from spybot.recorder.recorder import Recorder print("spybot app ready") - rec = Recorder() - rec.start() diff --git a/spybot/management/commands/recorder.py b/spybot/management/commands/recorder.py new file mode 100644 index 0000000..98303cd --- /dev/null +++ b/spybot/management/commands/recorder.py @@ -0,0 +1,14 @@ +from django.core.management import BaseCommand, CommandError + + +class Command(BaseCommand): + def handle(self, *args, **options): + try: + from spybot.recorder.recorder import Recorder + from Spybot2 import settings + if settings.RECORDER_ENABLED: + print('Starting recorder') + rec = Recorder() + rec.run() + except Exception as e: + raise CommandError(f'Error in recorder command: {e}') diff --git a/spybot/recorder/recorder.py b/spybot/recorder/recorder.py index ae6b3a7..1c83f99 100644 --- a/spybot/recorder/recorder.py +++ b/spybot/recorder/recorder.py @@ -15,11 +15,6 @@ def __init__(self): self.ts = TS() self.client = Client(self.ts) - def start(self): - if settings.RECORDER_ENABLED: - thread = Thread(target=self.run, daemon=True) - thread.start() - def run(self): while True: diff --git a/uv.lock b/uv.lock index 68b1749..ef70790 100644 --- a/uv.lock +++ b/uv.lock @@ -344,16 +344,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/db/214290d58ad68c587bd5d6af3d34e56830438733d0d0856c0275fde43652/lxml-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d", size = 3814417 }, ] -[[package]] -name = "mysqlclient" -version = "2.2.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/33/996dc0ba3f03e2399adc91a7de1f61cb14b57ebdb4cc6eca8a78723043cb/mysqlclient-2.2.4.tar.gz", hash = "sha256:33bc9fb3464e7d7c10b1eaf7336c5ff8f2a3d3b88bab432116ad2490beb3bf41", size = 90400 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/0c/338ff73f627db62f2c321bcda61eddb80a384933aa543e3f59821b3c9756/mysqlclient-2.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:329e4eec086a2336fe3541f1ce095d87a6f169d1cc8ba7b04ac68bcb234c9711", size = 203187 }, - { url = "https://files.pythonhosted.org/packages/d3/56/1fe1a8aa7ba090576632c16bd43fb0df78747d2f18c54e769015378667fb/mysqlclient-2.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:e1ebe3f41d152d7cb7c265349fdb7f1eca86ccb0ca24a90036cde48e00ceb2ab", size = 203295 }, -] - [[package]] name = "num2words" version = "0.5.13" @@ -389,6 +379,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/6e/4a52a8923d840107024b844d83502dfa6a1e5399ad31cf9d1a4ddbaaa7e5/paramiko-3.4.1-py3-none-any.whl", hash = "sha256:8e49fd2f82f84acf7ffd57c64311aa2b30e575370dc23bdb375b10262f7eac32", size = 226224 }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/8f/9feb01291d0d7a0a4c6a6bab24094135c2b59c6a81943752f632c75896d6/psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff", size = 3043397 }, + { url = "https://files.pythonhosted.org/packages/15/30/346e4683532011561cd9c8dfeac6a8153dd96452fee0b12666058ab7893c/psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c", size = 3274806 }, + { url = "https://files.pythonhosted.org/packages/66/6e/4efebe76f76aee7ec99166b6c023ff8abdc4e183f7b70913d7c047701b79/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c", size = 2851370 }, + { url = "https://files.pythonhosted.org/packages/7f/fd/ff83313f86b50f7ca089b161b8e0a22bb3c319974096093cd50680433fdb/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb", size = 3080780 }, + { url = "https://files.pythonhosted.org/packages/e6/c4/bfadd202dcda8333a7ccafdc51c541dbdfce7c2c7cda89fa2374455d795f/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341", size = 3264583 }, + { url = "https://files.pythonhosted.org/packages/5d/f1/09f45ac25e704ac954862581f9f9ae21303cc5ded3d0b775532b407f0e90/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a", size = 3019831 }, + { url = "https://files.pythonhosted.org/packages/9e/2e/9beaea078095cc558f215e38f647c7114987d9febfc25cb2beed7c3582a5/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b", size = 2871822 }, + { url = "https://files.pythonhosted.org/packages/01/9e/ef93c5d93f3dc9fc92786ffab39e323b9aed066ba59fdc34cf85e2722271/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7", size = 2820975 }, + { url = "https://files.pythonhosted.org/packages/a5/f0/049e9631e3268fe4c5a387f6fc27e267ebe199acf1bc1bc9cbde4bd6916c/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e", size = 2919320 }, + { url = "https://files.pythonhosted.org/packages/dc/9a/bcb8773b88e45fb5a5ea8339e2104d82c863a3b8558fbb2aadfe66df86b3/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68", size = 2957617 }, + { url = "https://files.pythonhosted.org/packages/e2/6b/144336a9bf08a67d217b3af3246abb1d027095dab726f0687f01f43e8c03/psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392", size = 1024618 }, + { url = "https://files.pythonhosted.org/packages/61/69/3b3d7bd583c6d3cbe5100802efa5beacaacc86e37b653fc708bf3d6853b8/psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4", size = 1163816 }, + { url = "https://files.pythonhosted.org/packages/49/7d/465cc9795cf76f6d329efdafca74693714556ea3891813701ac1fee87545/psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0", size = 3044771 }, + { url = "https://files.pythonhosted.org/packages/8b/31/6d225b7b641a1a2148e3ed65e1aa74fc86ba3fee850545e27be9e1de893d/psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a", size = 3275336 }, + { url = "https://files.pythonhosted.org/packages/30/b7/a68c2b4bff1cbb1728e3ec864b2d92327c77ad52edcd27922535a8366f68/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539", size = 2851637 }, + { url = "https://files.pythonhosted.org/packages/0b/b1/cfedc0e0e6f9ad61f8657fd173b2f831ce261c02a08c0b09c652b127d813/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526", size = 3082097 }, + { url = "https://files.pythonhosted.org/packages/18/ed/0a8e4153c9b769f59c02fb5e7914f20f0b2483a19dae7bf2db54b743d0d0/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1", size = 3264776 }, + { url = "https://files.pythonhosted.org/packages/10/db/d09da68c6a0cdab41566b74e0a6068a425f077169bed0946559b7348ebe9/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e", size = 3020968 }, + { url = "https://files.pythonhosted.org/packages/94/28/4d6f8c255f0dfffb410db2b3f9ac5218d959a66c715c34cac31081e19b95/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f", size = 2872334 }, + { url = "https://files.pythonhosted.org/packages/05/f7/20d7bf796593c4fea95e12119d6cc384ff1f6141a24fbb7df5a668d29d29/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00", size = 2822722 }, + { url = "https://files.pythonhosted.org/packages/4d/e4/0c407ae919ef626dbdb32835a03b6737013c3cc7240169843965cada2bdf/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5", size = 2920132 }, + { url = "https://files.pythonhosted.org/packages/2d/70/aa69c9f69cf09a01da224909ff6ce8b68faeef476f00f7ec377e8f03be70/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47", size = 2959312 }, + { url = "https://files.pythonhosted.org/packages/d3/bd/213e59854fafe87ba47814bf413ace0dcee33a89c8c8c814faca6bc7cf3c/psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64", size = 1025191 }, + { url = "https://files.pythonhosted.org/packages/92/29/06261ea000e2dc1e22907dbbc483a1093665509ea586b29b8986a0e56733/psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0", size = 1164031 }, + { url = "https://files.pythonhosted.org/packages/3e/30/d41d3ba765609c0763505d565c4d12d8f3c79793f0d0f044ff5a28bf395b/psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d", size = 3044699 }, + { url = "https://files.pythonhosted.org/packages/35/44/257ddadec7ef04536ba71af6bc6a75ec05c5343004a7ec93006bee66c0bc/psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb", size = 3275245 }, + { url = "https://files.pythonhosted.org/packages/1b/11/48ea1cd11de67f9efd7262085588790a95d9dfcd9b8a687d46caf7305c1a/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7", size = 2851631 }, + { url = "https://files.pythonhosted.org/packages/62/e0/62ce5ee650e6c86719d621a761fe4bc846ab9eff8c1f12b1ed5741bf1c9b/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d", size = 3082140 }, + { url = "https://files.pythonhosted.org/packages/27/ce/63f946c098611f7be234c0dd7cb1ad68b0b5744d34f68062bb3c5aa510c8/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73", size = 3264762 }, + { url = "https://files.pythonhosted.org/packages/43/25/c603cd81402e69edf7daa59b1602bd41eb9859e2824b8c0855d748366ac9/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673", size = 3020967 }, + { url = "https://files.pythonhosted.org/packages/5f/d6/8708d8c6fca531057fa170cdde8df870e8b6a9b136e82b361c65e42b841e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f", size = 2872326 }, + { url = "https://files.pythonhosted.org/packages/ce/ac/5b1ea50fc08a9df82de7e1771537557f07c2632231bbab652c7e22597908/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909", size = 2822712 }, + { url = "https://files.pythonhosted.org/packages/c4/fc/504d4503b2abc4570fac3ca56eb8fed5e437bf9c9ef13f36b6621db8ef00/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1", size = 2920155 }, + { url = "https://files.pythonhosted.org/packages/b2/d1/323581e9273ad2c0dbd1902f3fb50c441da86e894b6e25a73c3fda32c57e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567", size = 2959356 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -484,8 +516,8 @@ dependencies = [ { name = "django-environ" }, { name = "fido2" }, { name = "gunicorn" }, - { name = "mysqlclient" }, { name = "num2words" }, + { name = "psycopg2-binary" }, { name = "requests" }, { name = "sentry-sdk" }, { name = "ts3" }, @@ -506,9 +538,9 @@ requires-dist = [ { name = "django-crontab", specifier = "~=0.7.1" }, { name = "django-environ", specifier = "~=0.11.2" }, { name = "fido2", specifier = "~=1.1" }, - { name = "gunicorn" }, - { name = "mysqlclient", specifier = "~=2.2" }, + { name = "gunicorn", specifier = ">=23.0.0" }, { name = "num2words", specifier = "~=0.5.12" }, + { name = "psycopg2-binary", specifier = ">=2.9.10" }, { name = "requests", specifier = "~=2.31" }, { name = "sentry-sdk", specifier = ">=2.13.0" }, { name = "ts3", specifier = "~=2.0.0b3" },