Merge branch 'master' of https://github.com/qgis/QGIS-Django into dow…

…nload_stats_per_country

.dockerignore

@@ -0,0 +1 @@
+dockerize/postgres_data

.github/workflows/test.yaml

@@ -16,15 +16,15 @@ jobs:
     steps:
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
 
       - name: Checkout
         uses: actions/checkout@master
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('REQUIREMENTS-dev.txt') }}
@@ -45,26 +45,24 @@ jobs:
       run:
         working-directory: dockerize
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Generate the .env file
+        run: cp .env.template .env
 
-      - name: Run docker-compose build
-        run: docker-compose build
 
       - name: Run the containers
         run: docker-compose up -d db devweb
+
 
-      - name: Run Coverage test
-        run: |
-          cat << EOF | docker-compose exec -T devweb bash
-          pip install coverage
-          python manage.py makemigrations
-          python manage.py migrate
-          coverage run manage.py test
-          coverage xml
-          EOF
+      - name: Wait for the containers to start
+        run: sleep 15 
 
-      - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v2
-        with:
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+      - name: Run test
+        run: |
+          docker-compose exec -T devweb bash -c '
+            set -e  # Exit immediately if any command fails
+            python manage.py makemigrations &&
+            python manage.py migrate &&
+            python manage.py test
+          '

.gitignore

@@ -43,6 +43,8 @@ static_media
 dockerize/static
 dockerize/media
 dockerize/logs
+dockerize/certbot-etc
+dockerize/webroot
 .editorconfig
 
 # test cache
@@ -51,3 +53,5 @@ qgis-app/api/tests/*/
 
 # whoosh_index
 qgis-app/whoosh_index/
+docker-compose.override.yml
+.env

INSTALL.md

@@ -193,8 +193,11 @@ $ make dbrestore
 ```
 
 ---
+
 ## Deploy Update with Rancher
 
+**TO BE DEPRECATED**
+
 - Go to [https://rancher.qgis.org](https://rancher.qgis.org)
 - Click the home icon in the top left corner,  and choose the environment: `plugins.qgis.org`
 - You will be redirected to User Stacks page, choose the plugins stack by clicking on `plugins` link

REQUIREMENTS.txt

@@ -1,17 +1,21 @@
-Django==2.2.25
+Django==3.2.11
+
 # Currently broken with 'no module named defaults' error
 #Feedjack==0.9.18
 # So use George's fork rather
 # git+https://github.com/Erve1879/feedjack.git
-# George's is also broken: use my fork (django 1.8 ready)
-git+https://github.com/elpaso/feedjack.git
+# George's is also broken: use elpaso fork (django 1.8 ready)
+# git+https://github.com/elpaso/feedjack.git
+# His is also broken, use dimasciput (django 3.2 ready)
+git+https://github.com/dimasciput/feedjack.git
+
 Markdown==2.3.1
 #PIL==1.1.7
 Pillow
 Pygments==2.7.4
 
 # Updates for Django 2 & Python 3.7
-git+https://github.com/Xpirix/whoosh.git@a306553
+git+https://github.com/Xpirix/whoosh.git@main
 pickle5==0.0.12
 django-haystack==3.2.1
 
@@ -21,16 +25,18 @@ argparse==1.2.1
 django-annoying==0.7.7
 django-auth-ldap==1.2.6
 django-autoslug==1.7.1
-django-debug-toolbar==1.11.1
+django-debug-toolbar==3.2.4
 django-endless-pagination==2.0
 django-extensions==1.2.0
 django-generic-aggregation==0.3.2
 #django-olwidget==0.61.0 unmaintained, use this fork
 git+https://github.com/Christophe31/olwidget.git
 django-pagination==1.0.7
+
 # Unmaintained!
 #django-ratings==0.3.7
-git+https://github.com/enikesha/django-ratings.git
+git+https://github.com/gelo-zhukov/django-ratings.git
+
 django-simple-ratings==0.3.2
 # SIMPLEMENU
 git+https://github.com/elpaso/django-simplemenu.git
@@ -57,6 +63,9 @@ django-bootstrap-pagination==1.7.1
 django-sortable-listview==0.43
 django-user-map
 djangorestframework==3.12.2
+pyjwt==1.7.1
+djangorestframework-simplejwt==4.4 
 django-rest-auth==0.9.5
 drf-yasg
 geoip2==4.5.0
+django-matomo==0.1.6

dockerize/.env.template

@@ -0,0 +1,34 @@
+# RabbitMQ host
+RABBITMQ_HOST=rabbitmq
+
+# Database variables
+DATABASE_NAME=gis
+DATABASE_USERNAME=docker
+DATABASE_PASSWORD=docker
+DATABASE_HOST=db
+
+# Django settings
+DJANGO_SETTINGS_MODULE=settings_docker
+DEBUG=False
+
+# Docker volumes
+QGISPLUGINS_STATIC_VOLUME=static-data
+QGISPLUGINS_MEDIA_VOLUME=media-data
+QGISPLUGINS_BACKUP_VOLUME=backups-data
+
+# Email variables
+EMAIL_BACKEND='django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST=''
+EMAIL_PORT=587
+EMAIL_USE_TLS=True
+EMAIL_HOST_USER=''
+EMAIL_HOST_PASSWORD=''
+
+# URL
+DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/'
+
+# ENV: debug or prod
+QGISPLUGINS_ENV=debug
+
+# Ldap
+ENABLE_LDAP=True

dockerize/Makefile

@@ -13,35 +13,70 @@ build:
 	@echo "------------------------------------------------------------------"
 	@echo "Building in production and development mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) build
+	@docker compose -p $(PROJECT_ID) build
+
+build-dev:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Building in development mode only"
+	@echo "------------------------------------------------------------------"
+	@docker compose -p $(PROJECT_ID) build devweb
 
 db:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running db in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) up -d db
+	@docker compose -p $(PROJECT_ID) up -d db
+
+metabase: db
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running metabase in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker compose -p $(PROJECT_ID) up -d metabase
 
 web: db
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) up -d web
+	@docker compose -p $(PROJECT_ID) up -d --scale uwsgi=2 web worker beat
+
+dbbackups: db
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running dbbackups in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker compose -p $(PROJECT_ID) up -d dbbackups
+
+certbot: web
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running cerbot in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker compose -p $(PROJECT_ID) up -d certbot
+
+devweb-test: db
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running in TESTING mode"
+	@echo "------------------------------------------------------------------"
+	@docker compose up --no-deps -d devweb
 
 devweb: db
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running in DEVELOPMENT mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) up --no-deps -d devweb
+	@docker compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat
 
 devweb-runserver: devweb
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running in DEVELOPMENT mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080
+	@docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080
 
 migrate:
 	@echo
@@ -54,26 +89,26 @@ migrate:
 	@# We add the '-' prefix to the next line as the migration may fail
 	@# but we want to continue anyway.
 	@#We need to migrate accounts first as it has a reference to user model
-	-@docker-compose -p $(PROJECT_ID) exec web python manage.py migrate auth
-	@docker-compose -p $(PROJECT_ID) exec web python manage.py migrate
+	-@docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth
+	@docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate
 
 update-migrations:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running update migrations in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) exec web python manage.py makemigrations
+	@docker compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations
 
 collectstatic:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Collecting static in production mode"
 	@echo "------------------------------------------------------------------"
-	#@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput
+	@docker compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput
 	#We need to run collect static in the same context as the running
 	# uwsgi container it seems so I use docker exec here
 	# no -it flag so we can run over remote shell
-	@docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput
+	# @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput
 
 reload:
 	@echo
@@ -89,7 +124,7 @@ kill:
 	@echo "------------------------------------------------------------------"
 	@echo "Killing in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) kill
+	@docker compose -p $(PROJECT_ID) kill
 
 rm: rm-only
 
@@ -98,38 +133,59 @@ rm-only: kill
 	@echo "------------------------------------------------------------------"
 	@echo "Removing production instance!!! "
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) rm
+	@docker compose -p $(PROJECT_ID) rm
+
+maillogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing smtp logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker compose exec smtp tail -f /var/log/mail.log
+
+mailerrorlogs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing smtp error logs in production mode"
+	@echo "------------------------------------------------------------------"
+	@docker compose exec smtp tail -f /var/log/mail.err
 
 dbrestore:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Restore dump from backups/latest.dmp in production mode"
 	@echo "------------------------------------------------------------------"
 	@# - prefix causes command to continue even if it fails
-	@echo "stopping web container"
-	@docker-compose -p $(PROJECT_ID) stop web
-	@echo "dropping gis"
-	@docker exec -t $(PROJECT_ID)-db su - postgres -c "dropdb gis"
-	@echo "creating gis"
-	@docker exec -t $(PROJECT_ID)-db su - postgres -c "createdb -O docker -T template_postgis gis"
-	@echo "restoring gis"
-	@# Because we pipe from one docker command to another and we are going
-	@# to execute this Make command from a remote server at times, we need to using use interactive mode (-i)
-	@# in the first command and not use terminal (-t) in the second. Please do not change these!
-	@docker exec -t  $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis"
-	@docker-compose -p $(PROJECT_ID) start web
-	@echo "starting web container"
+	@echo "stopping uwsgi container"
+	@docker compose -p $(PROJECT_ID) stop uwsgi
+	@echo "Dropping the gis and metabase databases"
+	-@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force gis"
+	-@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force metabase"
+	@echo "Creating the gis and metabase databases"
+	-@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 gis"
+	-@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 metabase"
+	@echo "Restore database from backups/latest-gis.dmp"
+	-@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-gis.dmp -d gis"
+	-@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-metabase.dmp -d metabase"
+	@echo "starting uwsgi container"
+	@docker compose -p $(PROJECT_ID) up -d uwsgi
+
+wait-db:
+	@docker compose -p $(PROJECT_ID) exec db su - postgres -c "until pg_isready; do sleep 5; done"
+
+create-test-db:
+	@docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -c 'create database test_db;'"
+	@docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -d test_db -c 'create extension postgis;'"
 
 dbseed:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Seed db with JSON data from /fixtures/*.json"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json'
+	@docker compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json'
 
 rebuild_index:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Rebuild search index in PRODUCTION mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -p $(PROJECT_ID) exec web bash -c 'python manage.py rebuild_index'
+	@docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index'