Merge pull request #128 from SmoFlaDru/dev-benno

docker compose and github action deployment via docker

.dockerignore

@@ -0,0 +1,3 @@
+venv
+.venv
+node_modules

.github/workflows/build-push-docker-image.yml

@@ -0,0 +1,62 @@
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches: ['master']
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+jobs:
+  build-push-deploy-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    outputs:
+      image_name: ${{ steps.get-build-name.outputs.image_name }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            # branch event
+            type=ref,event=branch
+            # tag event
+            type=ref,event=tag
+            # commit sha (short)
+            type=sha,format=long
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          platforms: "linux/arm64"
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)."
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.github/workflows/deploy-docker.yml

@@ -0,0 +1,37 @@
+name: Deploy to server via docker image
+
+on:
+  push:
+    branches: [ "master", "workflow_dispatch" ]
+  workflow_dispatch:
+jobs:
+  deploy:
+    name: Deploy to Server
+    needs: build-push-deploy-image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: upload infrastructure files
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.SSH_SECONDARY_HOST }}
+          username: ${{ secrets.SSH_SECONDARY_USERNAME }}
+          password: ${{ secrets.SSH_SECONDARY_PASSWORD }}
+          port: ${{ secrets.SSH_SECONDARY_PORT }}
+          source: infrastructure/*
+          target: .
+      - name: execute remote ssh commands
+        uses: appleboy/ssh-action@master
+        env:
+          SECRET: ${{ secrets.ENV_FILE }}
+          COMMIT_SHA: ${{ github.sha }}
+        with:
+          host: ${{ secrets.SSH_SECONDARY_HOST }}
+          username: ${{ secrets.SSH_SECONDARY_USERNAME }}
+          password: ${{ secrets.SSH_SECONDARY_PASSWORD }}
+          port: ${{ secrets.SSH_SECONDARY_PORT }}
+          envs: SECRET,COMMIT_SHA
+          script: |
+            cd Spybot2
+            echo "$SECRET" > .env
+            docker-compose -f docker-compose-deploy.yml up

.github/workflows/deploy.yml

@@ -1,26 +1,26 @@
-name: Deploy to staging
-
-on:
-  push:
-    branches: [ "master" ]
-  workflow_dispatch:
-jobs:
-  deploy:
-    name: Deploy to Server
-    runs-on: ubuntu-latest
-    steps:
-      - name: execute remote ssh commands
-        uses: appleboy/ssh-action@master
-        env:
-          SECRET: ${{ secrets.ENV_FILE }}
-        with:
-          host: ${{ secrets.LINUX_HOST }}
-          username: ${{ secrets.LINUX_USERNAME }}
-          password: ${{ secrets.LINUX_PASSWORD }}
-          port: ${{ secrets.LINUX_PORT }}
-          envs: SECRET
-          script: |
-            cd Spybot2
-            echo "$SECRET" > .env
-            chmod +x deploy.sh
-            ./deploy.sh
+#name: Deploy to staging
+#
+#on:
+#  push:
+#    branches: [ "master" ]
+#  workflow_dispatch:
+#jobs:
+#  deploy:
+#    name: Deploy to Server
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: execute remote ssh commands
+#        uses: appleboy/ssh-action@master
+#        env:
+#          SECRET: ${{ secrets.ENV_FILE }}
+#        with:
+#          host: ${{ secrets.LINUX_HOST }}
+#          username: ${{ secrets.LINUX_USERNAME }}
+#          password: ${{ secrets.LINUX_PASSWORD }}
+#          port: ${{ secrets.LINUX_PORT }}
+#          envs: SECRET
+#          script: |
+#            cd Spybot2
+#            echo "$SECRET" > .env
+#            chmod +x deploy.sh
+#            ./deploy.sh

Dockerfile

@@ -1,11 +1,26 @@
-FROM python:3.12.5-bookworm
+FROM node:23-slim AS frontend-build
+
+COPY frontend frontend
+WORKDIR frontend
+RUN npm install
+RUN npm run package
+
+FROM python:3.12-slim-bookworm
 
 WORKDIR /app
 
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+EXPOSE 8000
 
+# Install mysqlclient debian package dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends python3-dev default-libmysqlclient-dev build-essential pkg-config && rm -rf /var/lib/apt/lists/*
 RUN pip install uv
 COPY pyproject.toml pyproject.toml
 RUN uv sync
 COPY . .
+
+COPY --from=frontend-build frontend/output frontend/output
+
+CMD ["sh", "run.sh"]

docker-compose-local.yml

@@ -0,0 +1,30 @@
+services:
+  spybot:
+    build:
+      dockerfile: Dockerfile
+    restart: unless-stopped
+    volumes:
+      - static_files:/spybot_static
+      - $PWD/.env:/.env
+    expose:
+      - 8000
+  caddy:
+    image: 'caddy:2.8-alpine'
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+    ports:
+      - "80:80"
+#      - "443:443"
+#      - "443:443/udp"
+    volumes:
+      - $PWD/Caddyfile:/etc/caddy/Caddyfile
+      - caddy_data:/data
+      - caddy_config:/config
+      - static_files:/spybot_static
+volumes:
+  caddy_data:
+  caddy_config:
+  static_files:
+
+

infrastructure/Caddyfile

@@ -0,0 +1,16 @@
+{
+	auto_https off
+}
+:80 {
+    bind 0.0.0.0
+    # Serve requests to /static/* from a static files
+    handle_path /static/* {
+        root * /spybot_static/
+        file_server
+    }
+
+    # Reverse proxy all other requests to Django application
+    handle {
+        reverse_proxy spybot:8000
+    }
+}

infrastructure/docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+  spybot:
+    image: 'ghcr.io/smofladru/spybot2:sha-${COMMIT_SHA}'
+    restart: unless-stopped
+    volumes:
+      - static_files:/spybot_static
+      - $PWD/.env:/.env
+    expose:
+      - 8000
+  caddy:
+    image: 'caddy:2.8-alpine'
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+    ports:
+      - "80:80"
+    volumes:
+      - $PWD/Caddyfile:/etc/caddy/Caddyfile
+      - caddy_data:/data
+      - caddy_config:/config
+      - static_files:/spybot_static
+volumes:
+  caddy_data:
+  caddy_config:
+  static_files:
+
+

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "user-agents ~=2.2",
     "django-bootstrap5 ~=24.2",
     "sentry-sdk>=2.13.0",
+    "gunicorn>=23.0.0",
 ]
 
 

run.sh

@@ -1,33 +1,14 @@
 #!/bin/bash
 
-# install frontend dependencies
-cd frontend
-npm install
-npm run package
-cd ..
-
+ls -a
 # activate venv
-source venv/bin/activate
-
-# install project python deps
-pip3.11 install uv
-uv sync
+#.venv/bin/activate
 
 # copy static files to directory for http server
-uv run manage.py collectstatic --noinput
-
-# create django superuser if necessary.
-# Set the env vars DJANGO_SUPERUSER_{PASSWORD,USERNAME,EMAIL} when running this script
-# shellcheck disable=SC2046
-env $(grep -E '^DJANGO_SUPERUSER_(PASSWORD|USERNAME|EMAIL)' .env | xargs) python manage.py createsuperuser --noinput
+.venv/bin/python manage.py collectstatic --noinput
 
 # run DB migrations if necessary
-uv run manage.py migrate
-
-# start cronjobs with crontab using django-crontab
-uv run manage.py crontab add
-# run twice in case jobs have been removed. See: https://github.com/kraiz/django-crontab/blob/master/django_crontab/crontab.py#L209
-uv run manage.py crontab add
+.venv/bin/python manage.py migrate
 
 # run django app
-uv run manage.py runserver 127.0.0.1:8000
+.venv/bin/gunicorn -w 2 --bind 0.0.0.0:8000 Spybot2.wsgi

spybot/visualization.py

@@ -25,7 +25,7 @@ def daily_activity(days: int):
                 WHERE
                     startTime > DATE_SUB(CURDATE(), INTERVAL %(days)s DAY)
                     AND endTime IS NOT NULL
-                    AND channel.name NOT IN ('bei\\\sBedarf\\\sanstupsen', 'AFK')
+                    AND channel.name NOT IN ('bei\\sBedarf\\sanstupsen', 'AFK')
                 GROUP BY date
                 ORDER BY date
             ),
@@ -38,7 +38,7 @@ def daily_activity(days: int):
                 WHERE
                     startTime > DATE_SUB(CURDATE(), INTERVAL %(days)s DAY)
                     AND endTime IS NOT NULL
-                    AND channel.name IN ('bei\\\sBedarf\\\sanstupsen', 'AFK')
+                    AND channel.name IN ('be\\sBedarf\\sanstupsen', 'AFK')
                 GROUP BY date
                 ORDER BY date
             )
@@ -320,7 +320,7 @@ def user_month_activity(merged_user_id: int):
             INNER JOIN TSUser user ON TSUserActivity.tsUserID = user.id
             WHERE startTime > MAKEDATE(2016,1)
                 AND endTime IS NOT NULL
-                AND channel.name NOT IN ('bei\\\sBedarf\\\sanstupsen', 'AFK')
+                AND channel.name NOT IN ('bei\\sBedarf\\sanstupsen', 'AFK')
                 AND user.merged_user_id = %s
             GROUP BY year, month
             ORDER BY year, month),

stunnel.conf

@@ -0,0 +1,12 @@
+
+output  = /Users/bennokrauss/PycharmProjects/Spybot2/stunnel.log
+cert    = stunnel_ca.pem
+client  = no
+sslVersion = all
+pid =
+foreground = yes
+
+
+[http]
+accept  = 0.0.0.0:8888
+connect = 127.0.0.1:8000