diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7ab91dc3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dockerize/postgres_data diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ca4d9fe9..de4f1d6f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,17 +50,18 @@ jobs: with: node-version: 20 - - name: Run docker-compose build - run: docker-compose build + - name: Generate the .env file + run: cp .env.template .env + - name: Run the containers run: docker-compose up -d db devweb - + - name: Wait for the containers to start run: sleep 15 - - name: Run test + - name: Run Django tests run: | docker-compose exec -T devweb bash -c ' set -e # Exit immediately if any command fails @@ -70,7 +71,6 @@ jobs: python manage.py migrate && python manage.py test ' - - name: Start Django server run: | docker-compose exec -T devweb bash -c "python manage.py loaddata fixtures/*.json" diff --git a/.gitignore b/.gitignore index b44def3a..89323100 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,8 @@ static_media dockerize/static dockerize/media dockerize/logs +dockerize/certbot-etc +dockerize/webroot .editorconfig # test cache @@ -54,3 +56,5 @@ qgis-app/whoosh_index/ # playwright fixture !playwright/ci-test/tests/fixtures/qgis-logo.zip +docker-compose.override.yml +.env diff --git a/INSTALL.md b/INSTALL.md index 1a66d939..7cbcfe23 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -43,6 +43,8 @@ $ make rebuild_index ``` This command ensures that the search index accurately reflects the current state of the database, resolving the presence of 'None' in the search results. Automatic synchronization is currently managed in settings.py: `HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor"`. +For more information about make commands, please see the full docs [here](./dockerize/README.md). + --- ### Setup git-hooks and local linting @@ -196,6 +198,8 @@ $ make dbrestore ## Deploy Update with Rancher +**TO BE DEPRECATED** + - Go to [https://rancher.qgis.org](https://rancher.qgis.org) - Click the home icon in the top left corner, and choose the environment: `plugins.qgis.org` - You will be redirected to User Stacks page, choose the plugins stack by clicking on `plugins` link diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 2e741ec8..c8375f10 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -1,17 +1,21 @@ -Django==2.2.25 +Django==3.2.11 + # Currently broken with 'no module named defaults' error #Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git -# George's is also broken: use my fork (django 1.8 ready) -git+https://github.com/elpaso/feedjack.git +# George's is also broken: use elpaso fork (django 1.8 ready) +# git+https://github.com/elpaso/feedjack.git +# His is also broken, use dimasciput (django 3.2 ready) +git+https://github.com/dimasciput/feedjack.git + Markdown==2.3.1 #PIL==1.1.7 Pillow Pygments==2.7.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 @@ -21,16 +25,18 @@ argparse==1.2.1 django-annoying==0.7.7 django-auth-ldap==1.2.6 django-autoslug==1.7.1 -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 django-endless-pagination==2.0 django-extensions==1.2.0 django-generic-aggregation==0.3.2 #django-olwidget==0.61.0 unmaintained, use this fork git+https://github.com/Christophe31/olwidget.git django-pagination==1.0.7 + # Unmaintained! #django-ratings==0.3.7 -git+https://github.com/enikesha/django-ratings.git +git+https://github.com/gelo-zhukov/django-ratings.git + django-simple-ratings==0.3.2 # SIMPLEMENU git+https://github.com/elpaso/django-simplemenu.git @@ -61,4 +67,5 @@ pyjwt==1.7.1 djangorestframework-simplejwt==4.4 django-rest-auth==0.9.5 drf-yasg -django-matomo==0.1.6 \ No newline at end of file +geoip2==4.5.0 +django-matomo==0.1.6 diff --git a/dockerize/.env.template b/dockerize/.env.template new file mode 100644 index 00000000..8cbf8dc8 --- /dev/null +++ b/dockerize/.env.template @@ -0,0 +1,37 @@ +# RabbitMQ host +RABBITMQ_HOST=rabbitmq + +# Database variables +DATABASE_NAME=gis +DATABASE_USERNAME=docker +DATABASE_PASSWORD=docker +DATABASE_HOST=db + +# Django settings +DJANGO_SETTINGS_MODULE=settings_docker +DEBUG=False + +# Docker volumes +QGISPLUGINS_STATIC_VOLUME=static-data +QGISPLUGINS_MEDIA_VOLUME=media-data +QGISPLUGINS_BACKUP_VOLUME=backups-data + +# Email variables +EMAIL_BACKEND='django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST='' +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_HOST_USER='' +EMAIL_HOST_PASSWORD='' + +# URL +DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/' + +# ENV: debug or prod +QGISPLUGINS_ENV=debug + +# Ldap +ENABLE_LDAP=False + +# Download stats URL +METABASE_DOWNLOAD_STATS_URL='https://plugins.qgis.org/metabase/public/dashboard/' diff --git a/dockerize/Makefile b/dockerize/Makefile index 21838313..c58599c3 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -13,35 +13,43 @@ build: @echo "------------------------------------------------------------------" @echo "Building in production and development mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) build + @docker compose -p $(PROJECT_ID) build + db: @echo @echo "------------------------------------------------------------------" @echo "Running db in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d db + @docker compose -p $(PROJECT_ID) up -d db + +metabase: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running metabase in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up -d metabase web: db @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d web + @docker compose -p $(PROJECT_ID) up -d --scale uwsgi=2 web worker beat -devweb: db +dbbackups: db @echo @echo "------------------------------------------------------------------" - @echo "Running in DEVELOPMENT mode" + @echo "Running dbbackups in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker compose -p $(PROJECT_ID) up -d dbbackups -devweb-runserver: devweb +certbot: web @echo @echo "------------------------------------------------------------------" - @echo "Running in DEVELOPMENT mode" + @echo "Running cerbot in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 + @docker compose -p $(PROJECT_ID) up -d certbot migrate: @echo @@ -54,42 +62,43 @@ migrate: @# We add the '-' prefix to the next line as the migration may fail @# but we want to continue anyway. @#We need to migrate accounts first as it has a reference to user model - -@docker-compose -p $(PROJECT_ID) exec web python manage.py migrate auth - @docker-compose -p $(PROJECT_ID) exec web python manage.py migrate + -@docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth + @docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate update-migrations: @echo @echo "------------------------------------------------------------------" @echo "Running update migrations in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web python manage.py makemigrations + @docker compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations collectstatic: @echo @echo "------------------------------------------------------------------" @echo "Collecting static in production mode" @echo "------------------------------------------------------------------" - #@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput - #We need to run collect static in the same context as the running - # uwsgi container it seems so I use docker exec here - # no -it flag so we can run over remote shell - @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput + @docker compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput -reload: +start: @echo @echo "------------------------------------------------------------------" - @echo "Reload django project in production mode" + @echo "Starting a specific container(s) in production mode. Use web if you want to start all." @echo "------------------------------------------------------------------" - # no -it flag so we can run over remote shell - @docker exec $(PROJECT_ID)-web uwsgi --reload /tmp/django.pid + @docker compose -p $(PROJECT_ID) up -d $(c) +restart: + @echo + @echo "------------------------------------------------------------------" + @echo "Restarting all or a specific container(s) in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) restart $(c) kill: @echo @echo "------------------------------------------------------------------" - @echo "Killing in production mode" + @echo "Killing all or a specific container(s) in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) kill + @docker compose -p $(PROJECT_ID) kill $(c) rm: rm-only @@ -98,7 +107,7 @@ rm-only: kill @echo "------------------------------------------------------------------" @echo "Removing production instance!!! " @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) rm + @docker compose -p $(PROJECT_ID) rm dbrestore: @echo @@ -106,30 +115,132 @@ dbrestore: @echo "Restore dump from backups/latest.dmp in production mode" @echo "------------------------------------------------------------------" @# - prefix causes command to continue even if it fails - @echo "stopping web container" - @docker-compose -p $(PROJECT_ID) stop web - @echo "dropping gis" - @docker exec -t $(PROJECT_ID)-db su - postgres -c "dropdb gis" - @echo "creating gis" - @docker exec -t $(PROJECT_ID)-db su - postgres -c "createdb -O docker -T template_postgis gis" - @echo "restoring gis" - @# Because we pipe from one docker command to another and we are going - @# to execute this Make command from a remote server at times, we need to using use interactive mode (-i) - @# in the first command and not use terminal (-t) in the second. Please do not change these! - @docker exec -t $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis" - @docker-compose -p $(PROJECT_ID) start web - @echo "starting web container" + @echo "stopping uwsgi container" + @docker compose -p $(PROJECT_ID) stop uwsgi + @echo "Dropping the gis and metabase databases" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force metabase" + @echo "Creating the gis and metabase databases" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 metabase" + @echo "Restore database from backups/latest-gis.dmp" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-gis.dmp -d gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-metabase.dmp -d metabase" + @echo "starting uwsgi container" + @docker compose -p $(PROJECT_ID) up -d uwsgi + +wait-db: + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "until pg_isready; do sleep 5; done" + +create-test-db: + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -c 'create database test_db;'" + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -d test_db -c 'create extension postgis;'" -dbseed: +rebuild_index: @echo @echo "------------------------------------------------------------------" - @echo "Seed db with JSON data from /fixtures/*.json" + @echo "Rebuild search index in PRODUCTION mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' -rebuild_index: +uwsgi-shell: @echo @echo "------------------------------------------------------------------" - @echo "Rebuild search index in PRODUCTION mode" + @echo "Shelling into the uwsgi container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash + +uwsgi-reload: + @echo + @echo "------------------------------------------------------------------" + @echo "Reload django project in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'uwsgi uwsgi --reload /tmp/django.pid' + +uwsgi-errors: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing errors in the uwsgi container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'tail -f /var/log/uwsgi-errors.log' + +uwsgi-logs: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing access logs in uwsgi container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'tail -f /var/log/uwsgi-requests.log' + +web-shell: + @echo + @echo "------------------------------------------------------------------" + @echo "Shelling into the NGINX/WEB container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec web bash + +web-logs: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing logs in NGINX/WEB container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) logs -f web + +logs: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing all logs or a specific container" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) logs -f $(c) + +shell: + @echo + @echo "------------------------------------------------------------------" + @echo "Shelling into a specific container" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec $(c) bash + +exec: + @echo + @echo "------------------------------------------------------------------" + @echo "Execute a specific docker command" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) $(c) + +# ---------------------------------------------------------------------------- +# D E V E L O P M E N T C O M M A N D S +# ---------------------------------------------------------------------------- + +build-dev: + @echo + @echo "------------------------------------------------------------------" + @echo "Building in development mode only" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) build devweb + +devweb-test: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in TESTING mode" + @echo "------------------------------------------------------------------" + @docker compose up --no-deps -d devweb + +devweb: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in DEVELOPMENT mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat + +devweb-runserver: devweb + @echo + @echo "------------------------------------------------------------------" + @echo "Running in DEVELOPMENT mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 + +dbseed: + @echo + @echo "------------------------------------------------------------------" + @echo "Seed db with JSON data from /fixtures/*.json" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web bash -c 'python manage.py rebuild_index' + @docker compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' \ No newline at end of file diff --git a/dockerize/README.md b/dockerize/README.md new file mode 100644 index 00000000..aa59cea4 --- /dev/null +++ b/dockerize/README.md @@ -0,0 +1,181 @@ +# Docker compose commands Documentation + +## Overview +This doc is designed for managing a Docker-based project with the ID `qgis-plugins`. It includes various commands for building, running, and maintaining both production and development environments. Below is a detailed description of each command available in the Makefile. + +## Commands + +### Production Commands + +- **default**: Alias for the build command. +```sh +make +``` + +- **run**: Builds the project and runs the web, migrate, and collectstatic commands. +```sh +make run +``` + +- **build**: Builds Docker images for both production and development environments. +```sh +make build +``` + +- **db**: Starts the database container in production mode. +```sh +make db +``` + +- **metabase**: Starts the Metabase container after ensuring the database is running. +```sh +make metabase +``` + +- **web**: Starts the web container and scales the `uwsgi` service to 2 instances. +```sh +make web +``` + +- **dbbackups**: Starts the database backups container. +```sh +make dbbackups +``` + +- **certbot**: Starts the Certbot container for managing SSL certificates. +```sh +make certbot +``` + +- **migrate**: Runs database migrations, with the `auth` app being migrated first. +```sh +make migrate +``` + +- **update-migrations**: Creates new migration files based on changes in models. +```sh +make update-migrations +``` + +- **collectstatic**: Collects static files for the Django application. +```sh +make collectstatic +``` + +- **start**: Starts a specific container or all containers. Specify the container with the `c` variable. +```sh +make start c=container_name +``` + +- **restart**: Restarts a specific container or all containers. Specify the container with the `c` variable. +```sh +make restart c=container_name +``` + +- **kill**: Stops a specific container or all containers. Specify the container with the `c` variable. +```sh +make kill c=container_name +``` + +- **rm**: Removes all containers after stopping them. +```sh +make rm +``` + +- **rm-only:** Removes all containers without stopping them first. +```sh +make rm-only +``` + +- **dbrestore:** Restores the database from a backup file. +```sh +make dbrestore +``` + +- **wait-db:** Waits for the database to be ready. +```sh +make wait-db +``` + +- **create-test-db:** Creates a test database with PostGIS extension. +```sh +make create-test-db +``` + +- **rebuild_index:** Rebuilds the search index for the Django application. +```sh +make rebuild_index +``` + +- **uwsgi-shell:** Opens a shell in the `uwsgi` container. +```sh +make uwsgi-shell +``` + +- **uwsgi-reload:** Reloads the Django project in the `uwsgi` container. +```sh +make uwsgi-reload +``` + +- **uwsgi-errors:** Tails the error logs in the `uwsgi` container. +```sh +make uwsgi-errors +``` + +- **uwsgi-logs:** Tails the requests logs in the `uwsgi` container. +```sh +make uwsgi-logs +``` + +- **web-shell:** Opens a shell in the NGINX/web container. +```sh +make web-shell +``` + +- **web-logs:** Tails the logs in the NGINX/web container. +```sh +make web-logs +``` + +- **logs:** Tails logs for a specific container or all containers. Specify the container with the `c` variable. +```sh +make logs c=container_name +``` + +- **shell:** Opens a shell in a specific container. Specify the container with the `c` variable. +```sh +make shell c=container_name +``` + +- **exec:** Executes a specific Docker command. Specify the command with the `c` variable. +```sh +make exec c="command" +``` + +### Development Commands + +- **build-dev:** Builds Docker images for the development environment. +```sh +make build-dev +``` + +- **devweb-test:** Starts the `devweb` container for testing, ensuring the database is running. +```sh +make devweb-test +``` + +- **devweb:** Starts the `devweb` container for development, along with RabbitMQ, worker, and beat containers. +```sh +make devweb +``` + +- **devweb-runserver:** Runs the Django development server inside the `devweb` container. +```sh +make devweb-runserver +``` + +- **dbseed:** Seeds the database with initial data from JSON files in the `fixtures` directory. +```sh +make dbseed +``` + diff --git a/dockerize/docker-compose.override.template.yml b/dockerize/docker-compose.override.template.yml new file mode 100644 index 00000000..bb0fdd8a --- /dev/null +++ b/dockerize/docker-compose.override.template.yml @@ -0,0 +1,55 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" + + beat: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + worker: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + uwsgi: + container_name: qgis-plugins-uwsgi + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod + + db: + volumes: + - ./postgres_data:/var/lib/postgresql + - ./backups:/backups + + web: + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + - ./static:/home/web/static:ro + - ./media:/home/web/media:ro + ports: + - "62201:8080" diff --git a/dockerize/docker-compose.override.test.yml b/dockerize/docker-compose.override.test.yml new file mode 100644 index 00000000..e10fe9a1 --- /dev/null +++ b/dockerize/docker-compose.override.test.yml @@ -0,0 +1,19 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 80578f9f..387fd96d 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,121 +1,201 @@ -version: "3.8" volumes: - django-statics-data: {} - django-media-data: {} + postgres_data: + rabbitmq: + celerybeat-schedule: + static-data: + media-data: + backups-data: services: + db: container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 + image: kartoza/postgis:16-3.4 environment: - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - PASSWORD_AUTHENTICATION=${PASSWORD_AUTHENTICATION:-md5} + - DATADIR=/opt/postgres/data + - DEFAULT_ENCODING=UTF8 + - DEFAULT_COLLATION=en_GB.utf8 + - DEFAULT_CTYPE=en_GB.utf8 volumes: - - ./backups:/backups + - postgres_data:/opt/postgres/data + - ${QGISPLUGINS_BACKUP_VOLUME}:/backups restart: unless-stopped + networks: + internal: - web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker + uwsgi: &uwsgi-common + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod hostname: uwsgi + expose: + - "8080" environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq + - DATABASE_NAME=${DATABASE_NAME:-gis} + - DATABASE_USERNAME=${DATABASE_USERNAME:-docker} + - DATABASE_PASSWORD=${DATABASE_PASSWORD:-docker} + - DATABASE_HOST=${DATABASE_HOST:-db} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-settings_docker} + - VIRTUAL_HOST=${VIRTUAL_HOST:-plugins.kartoza.com} + - VIRTUAL_PORT=${VIRTUAL_PORT:-8080} + - DEBUG=${DEBUG:-False} + - ENABLE_LDAP=${ENABLE_LDAP:-False} + - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} + - BROKER_URL=amqp://rabbitmq:5672 + - METABASE_DOWNLOAD_STATS_URL=${METABASE_DOWNLOAD_STATS_URL:-/metabase} + - EMAIL_BACKEND=${EMAIL_BACKEND} + - EMAIL_HOST=${EMAIL_HOST} + - EMAIL_PORT=${EMAIL_PORT} + - EMAIL_USE_TLS=${EMAIL_USE_TLS} + - EMAIL_HOST_USER=${EMAIL_HOST_USER:-automation} + - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD} + - DEFAULT_PLUGINS_SITE=${DEFAULT_PLUGINS_SITE:-https://plugins.qgis.org/} + volumes: - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker + - ./docker/uwsgi.conf:/uwsgi.conf + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw + - celerybeat-schedule:/home/web/celerybeat-schedule:rw + + depends_on: + - db + - rabbitmq restart: unless-stopped user: root - command: uwsgi --ini /uwsgi.conf + networks: + internal: + # This is the entry point for a development server. + # Run with --no-deps to run attached to the services + # from prod environment if wanted devweb: - # Note you cannot scale if you use container_name + <<: *uwsgi-common container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev volumes: - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw ports: # for django test server - "62202:8080" # for ssh - "62203:22" + networks: + internal: rabbitmq: - image: library/rabbitmq:3.6 + image: rabbitmq:3.7-alpine hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit + volumes: + - rabbitmq:/var/lib/rabbitmq restart: unless-stopped + networks: + internal: + + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + networks: + internal: worker: - # Note you cannot scale if you use container_name + <<: *uwsgi-common container_name: qgis-plugins-worker - build: docker - hostname: uwsgi + depends_on: + - db + - rabbitmq + - beat working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq + entrypoint: [] + command: celery -A plugins worker -l INFO + networks: + internal: - nginx: + web: # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx + container_name: qgis-plugins-web image: nginx - hostname: nginx - volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - - django-statics-data:/home/web/static:ro - - django-media-data:/home/web/media:ro - - ./logs:/var/log/nginx - links: - - web:uwsgi + hostname: web + entrypoint: + - /etc/nginx/sites-available/docker-entrypoint.sh ports: - - "62201:8080" + - "80:80" + - "443:443" + volumes: + - ./sites-enabled:/etc/nginx/sites-available/:ro + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro + - ./webroot:/var/www/webroot + - ./certbot-etc:/etc/letsencrypt + depends_on: + - uwsgi + - metabase + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + restart: unless-stopped + command: + - ${QGISPLUGINS_ENV} + networks: + internal: + + dbbackups: + image: kartoza/pg-backup:16-3.4 + hostname: pg-backups + volumes: + - ${QGISPLUGINS_BACKUP_VOLUME}:/backups + depends_on: + - db + environment: + # take care to let the project name below match that + # declared in the top of the makefile + - DUMPPREFIX=${DUMPPREFIX:-QGIS_PLUGINS} + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_HOST=${DATABASE_HOST:-db} + - PGDATABASE=${DATABASE_NAME:-gis} restart: unless-stopped + networks: + internal: + + metabase: + image: metabase/metabase:latest + environment: + - MB_DB_TYPE=postgres + - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} + depends_on: + - db + expose: + - "3000" + networks: + internal: + + certbot: + image: certbot/certbot + container_name: certbot + volumes: + - ./webroot:/var/www/webroot + - ./certbot-etc:/etc/letsencrypt + depends_on: + - web + command: certonly --webroot --webroot-path=/var/www/webroot --email admin@qgis.org --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org + networks: + internal: + +networks: + internal: \ No newline at end of file diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 3bd21f73..b635df70 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -1,22 +1,50 @@ -#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -# Note this base image is based on debian -FROM kartoza/django-base:3.7 -MAINTAINER Dimas Ciputra +# For more information, please refer to https://aka.ms/vscode-docker-python +FROM python:3.12-slim as prod -#RUN ln -s /bin/true /sbin/initctl -RUN apt-get clean all +EXPOSE 8000 -# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 -RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 -RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev -ADD REQUIREMENTS.txt /REQUIREMENTS.txt -RUN pip install -r /REQUIREMENTS.txt -RUN pip install uwsgi freezegun==1.3.1 +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 +RUN apt-get update && apt-get install -y \ + git python3-dev libxml2-dev \ + libsasl2-dev libldap2-dev libssl-dev \ + libxslt1-dev zlib1g-dev \ + build-essential \ + libffi-dev gdal-bin\ + libjpeg-dev libpq-dev \ + liblcms2-dev libblas-dev libatlas-base-dev \ + libmaxminddb0 libmaxminddb-dev mmdb-bin + +# GeoIp mmdb +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + +RUN rm -rf /uwsgi.conf +ADD dockerize/docker/uwsgi.conf /uwsgi.conf +ADD qgis-app /home/web/django_project +ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt + +RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt + +RUN mkdir -p /var/log/uwsgi + +WORKDIR /home/web/django_project +CMD ["uwsgi", "--ini", "/uwsgi.conf"] + + +FROM prod as dev + +# This section taken on 2 July 2015 from # https://docs.docker.com/examples/running_ssh_service/ # Sudo is needed by pycharm when it tries to pip install packages -RUN apt-get install -y openssh-server sudo +RUN apt-get update && apt-get install -y openssh-server sudo RUN mkdir /var/run/sshd RUN echo 'root:docker' | chpasswd RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config @@ -27,15 +55,16 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile -RUN rm -rf /uwsgi.conf -ADD uwsgi.conf /uwsgi.conf +# Install freezegun for feedback test +RUN pip install freezegun -# Open port 8080 as we will be running our uwsgi socket on that -EXPOSE 8080 +# -------------------------------------------------------- +# Open ports as needed +# -------------------------------------------------------- +# Open port 8080 as we will be running our django dev server on +EXPOSE 8080 # Open port 22 as we will be using a remote interpreter from pycharm EXPOSE 22 -RUN mkdir -p /var/log/uwsgi -WORKDIR /home/web/django_project CMD ["/usr/sbin/sshd", "-D"] diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index 4de38450..3d738ad9 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -1,59 +1,62 @@ -django==2.2.25 -django-auth-ldap -python-ldap -django-taggit==2.0.0 +django~=4.2 +django-auth-ldap~=4.6 +python-ldap~=3.4 +django-taggit~=5.0 django-tinymce==3.4.0 -psycopg2 +psycopg2-binary~=2.9 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git +# Updates for Django 4 +git+https://github.com/Xpirix/django-ratings.git@modernize +django-taggit-autosuggest~=0.4 +django-annoying~=0.10 + # Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize -django-taggit-autosuggest -django-annoying -# Updates for Django 2 -git+https://github.com/elpaso/rpc4django.git@modernize -Pillow +# git+https://github.com/elpaso/rpc4django.git@modernize +rpc4django~=0.6 +Pillow~=10.1 django-taggit-templatetags -# Updates for Django 2 -git+https://github.com/elpaso/django-simplemenu.git@modernize -django-bootstrap-pagination -django-sortable-listview -sorl-thumbnail -django-extensions -django-debug-toolbar==1.11.1 - -# Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 -pickle5==0.0.12 -django-haystack==3.2.1 +# Updates for Django 4 +git+https://github.com/Xpirix/django-simplemenu.git@modernize +django-bootstrap-pagination-forked~=1.7 +django-sortable-listview~=0.43 +sorl-thumbnail~=12.10 +django-extensions~=3.2 +django-debug-toolbar~=4.2 +whoosh~=2.7 +django-haystack~=3.2 # Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git # George's is also broken: use my fork (django 1.8 ready) # git+https://github.com/elpaso/feedjack.git -# His is also broken, use mine (django 2.2 ready) -git+https://github.com/dimasciput/feedjack.git -feedparser==5.2.1 -celery==4.3.1 +# His is also broken, use dimasciput (django 2.2 ready) +# git+https://github.com/dimasciput/feedjack.git +# For django 4, use Xpirix (django 4.2 ready) +git+https://github.com/Xpirix/feedjack.git +feedparser~=6.0 +celery~=5.3 # pin due to issues with a breaking change # https://github.com/celery/celery/issues/7783 importlib_metadata<5 -requests==2.23.0 - -markdown==3.2.1 +requests~=2.31 -djangorestframework==3.11.2 -pyjwt==1.7.1 -djangorestframework-simplejwt==4.4 +markdown~=3.5 +djangorestframework~=3.14 +pyjwt~=2.8 +djangorestframework-simplejwt~=5.3 sorl-thumbnail-serializer-field==0.2.1 django-rest-auth==0.9.5 -drf-yasg==1.17.1 +drf-yasg~=1.21 django-rest-multiple-models==2.1.3 django-preferences==1.0.0 PyWavefront==1.3.3 +geoip2==4.5.0 django-matomo==0.1.6 +uwsgi~=2.0 +freezegun~=1.4 diff --git a/dockerize/docker/uwsgi.conf b/dockerize/docker/uwsgi.conf index 50a80d0d..7efc0305 100644 --- a/dockerize/docker/uwsgi.conf +++ b/dockerize/docker/uwsgi.conf @@ -7,15 +7,16 @@ module = wsgi master = true pidfile=/tmp/django.pid socket = 0.0.0.0:8080 -workers = 4 +workers = 8 cheaper = 2 env = DJANGO_SETTINGS_MODULE=settings_docker # disabled so we run in the foreground for docker #daemonize = /tmp/django.log req-logger = file:/var/log/uwsgi-requests.log logger = file:/var/log/uwsgi-errors.log -reload-os-env +# reload-os-env #uid = 1000 #gid = 1000 memory-report = true -harakiri = 50 +harakiri = 100 +listen = 127 diff --git a/dockerize/nginx.conf b/dockerize/nginx.conf deleted file mode 100644 index 4a728e85..00000000 --- a/dockerize/nginx.conf +++ /dev/null @@ -1,99 +0,0 @@ -# Define connection details for connecting to django running in -# a docker container. -upstream uwsgi { - server uwsgi:8080; -} -server { - # OTF gzip compression - gzip on; - gzip_min_length 860; - gzip_comp_level 5; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; - gzip_disable “MSIE [1-6].(?!.*SV1)”; - - access_log /var/log/nginx/access.log; - - client_max_body_size 20M; - error_log /var/log/nginx/error.log; - - # the port your site will be served on - listen 80; - # the domain name it will serve for - server_name ""; - - if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCB -ot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCo -llector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|Go -zaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ N -inja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan| -Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGe -nSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pyc -url|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|s -itecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|t -urnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II| -WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { - return 403; - } - charset utf-8; - - # Drop any non django related requests - # Its probably someone nefarious probing for vulnerabilities... - location ~ (\.php|\.asp|myadmin) { - return 404; - } - - # Django media - location /media { - # your Django project's media files - amend as required - alias /home/web/media; - expires 21d; # cache for 71 days - } - location /static { - # your Django project's static files - amend as required - alias /home/web/static; - expires 21d; # cache for 21 days - } - location /plugins/plugins.xml { - if ($request_uri !~ "&package_name(.*)") { - rewrite ^/plugins/plugins.xml /web/static/cached_xmls/plugins_$arg_qgis.xml break; - root /home; - expires 600s; - } - } - location /archive { - # Changed from http_host to host because of error messages when - # bots hit urls like this: - # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', - # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ - # for more details. - #proxy_set_header Host $http_host; - proxy_set_header Host $host; - autoindex on; - # your Django project's static files - amend as required - alias /home/web/archive; - expires 21d; # cache for 6h - } - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi; - # the uwsgi_params file you installed needs to be passed with each - # request. - # the uwsgi_params need to be passed with each uwsgi request - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - } -} diff --git a/dockerize/production/Dockerfile b/dockerize/production/Dockerfile index 7a57d864..3631f46a 100644 --- a/dockerize/production/Dockerfile +++ b/dockerize/production/Dockerfile @@ -5,6 +5,10 @@ MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all + +# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 +RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list + RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev ARG BRANCH_TAG=develop @@ -13,12 +17,23 @@ RUN mkdir -p /usr/src; mkdir -p /home/web && \ rm -rf /home/web/django_project && \ ln -s /usr/src/plugins/qgis-app /home/web/django_project +# Install C library for geoip2 +RUN apt-get install -y libmaxminddb0 libmaxminddb-dev mmdb-bin + RUN cd /usr/src/plugins/dockerize/docker && \ + pip install --upgrade pip && \ pip install -r REQUIREMENTS.txt && \ pip install uwsgi && \ rm -rf /uwsgi.conf && \ ln -s ${PWD}/uwsgi.conf /uwsgi.conf +# GeoIp mmdb +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + # Open port 8080 as we will be running our uwsgi socket on that EXPOSE 8080 diff --git a/dockerize/scripts/renew-ssl.sh b/dockerize/scripts/renew-ssl.sh new file mode 100644 index 00000000..7004ead6 --- /dev/null +++ b/dockerize/scripts/renew-ssl.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + + +# Run daily on crontab e.g. +# Your cron job will be run at: (5 times displayed) +# +# 2021-11-08 11:10:00 UTC +# 2021-11-09 11:10:00 UTC +# 2021-11-10 11:10:00 UTC +# 2021-11-11 11:10:00 UTC +# 2021-11-12 11:10:00 UTC +# ...etc + +#25 11 * * * /bin/bash /home/web/QGIS-Django/dockerize/scripts/renew_ssl.sh > /tmp/ssl-renewal-logs.txt + + +docker compose -f /home/web/QGIS-Django/dockerize/docker-compose.yml -p qgis-plugins run certbot renew \ No newline at end of file diff --git a/dockerize/sites-enabled/default.conf b/dockerize/sites-enabled/default.conf deleted file mode 100644 index e870a2a6..00000000 --- a/dockerize/sites-enabled/default.conf +++ /dev/null @@ -1,77 +0,0 @@ -# Define connection details for connecting to django running in -# a docker container. -upstream uwsgi { - server uwsgi:8080; -} -server { - # OTF gzip compression - gzip on; - gzip_min_length 860; - gzip_comp_level 5; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; - gzip_disable “MSIE [1-6].(?!.*SV1)”; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - # the port your site will be served on - listen 8080; - # the domain name it will serve for - server_name ""; - charset utf-8; - - # Drop any non django related requests - # Its probably someone nefarious probing for vulnerabilities... - location ~ (\.php|.aspx|.asp|myadmin) { - return 404; - } - # max upload size, adjust to taste - client_max_body_size 15M; - # Django media - location /media { - # your Django project's media files - amend as required - alias /home/web/media; - expires 21d; # cache for 71 days - } - location /static { - # your Django project's static files - amend as required - alias /home/web/static; - expires 21d; # cache for 21 days - } - location /archive { - # Changed from http_host to host because of error messages when - # bots hit urls like this: - # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', - # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ - # for more details. - #proxy_set_header Host $http_host; - proxy_set_header Host $host; - autoindex on; - # your Django project's static files - amend as required - alias /home/web/archive; - expires 21d; # cache for 6h - } - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi; - # the uwsgi_params file you installed needs to be passed with each - # request. - # the uwsgi_params need to be passed with each uwsgi request - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - } -} diff --git a/dockerize/sites-enabled/dev.conf b/dockerize/sites-enabled/dev.conf new file mode 100644 index 00000000..ef93e13c --- /dev/null +++ b/dockerize/sites-enabled/dev.conf @@ -0,0 +1,31 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + charset utf-8; + + # Send all requests to the Django server. + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://uwsgi; + } +} diff --git a/dockerize/sites-enabled/docker-entrypoint.sh b/dockerize/sites-enabled/docker-entrypoint.sh new file mode 100755 index 00000000..fbc27084 --- /dev/null +++ b/dockerize/sites-enabled/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Clean up sites-enabled +echo "Clean sites-enabled" +rm -rf /etc/nginx/conf.d/*.conf +mkdir -p /etc/nginx/conf.d + +if [ $# -eq 1 ]; then + case $1 in + # Debug mode, enable dev.conf + [Dd][Ee][Bb][Uu][Gg]) + echo "Run in debug mode" + CONF_FILE=dev.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + # Production mode, run using uwsgi + [Pp][Rr][Oo][Dd]) + echo "Run in prod mode" + CONF_FILE=prod.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + # Production SSL mode, run using uwsgi + [Pp][Rr][Oo][Dd][-][Ss][Ss][Ll]) + echo "Run in prod SSL mode" + CONF_FILE=prod-ssl.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + esac +fi + +# Run as bash entrypoint +exec "$@" diff --git a/dockerize/sites-enabled/prod-ssl.conf b/dockerize/sites-enabled/prod-ssl.conf new file mode 100644 index 00000000..589715f1 --- /dev/null +++ b/dockerize/sites-enabled/prod-ssl.conf @@ -0,0 +1,219 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} + +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name plugins.qgis.org; + + # Redirect all HTTP traffic to HTTPS + return 301 https://$server_name$request_uri; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 21 days + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + + if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { + return 403; + } + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + location ~ /.well-known/acme-challenge { + # set to webroot path + root /var/www/webroot; + default_type "text/plain"; + allow all; + } +} + + +server { + # SSL Cert + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name plugins.qgis.org; + + server_tokens off; + + ssl_certificate /etc/letsencrypt/live/plugins.qgis.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/plugins.qgis.org/privkey.pem; + + ssl_buffer_size 8k; + + # ssl_dhparam /etc/ssl/certs/dhparam-2048.pem; + + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + ssl_prefer_server_ciphers on; + + ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5; + + ssl_ecdh_curve secp384r1; + ssl_session_tickets off; + + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8; + + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 21 days + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + + if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { + return 403; + } + + } + + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + root /var/www/webroot; +} \ No newline at end of file diff --git a/dockerize/sites-enabled/prod.conf b/dockerize/sites-enabled/prod.conf new file mode 100644 index 00000000..086d6903 --- /dev/null +++ b/dockerize/sites-enabled/prod.conf @@ -0,0 +1,103 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} + +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 21 days + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + + if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { + return 403; + } + + } + + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + location ~ /.well-known/acme-challenge { + # set to webroot path + root /var/www/webroot; + default_type "text/plain"; + allow all; + } +} diff --git a/qgis-app/REQUIREMENTS_plugins.txt b/qgis-app/REQUIREMENTS_plugins.txt index fa9c03ac..ebb39373 100644 --- a/qgis-app/REQUIREMENTS_plugins.txt +++ b/qgis-app/REQUIREMENTS_plugins.txt @@ -1,4 +1,4 @@ -django==2.2.25 +django==3.2.11 django-auth-ldap python-ldap django-taggit==2.0.0 @@ -6,8 +6,10 @@ django-tinymce==3.4.0 psycopg2 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git -# Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize + +# Updates for Django 3 +git+https://github.com/gelo-zhukov/django-ratings.git + django-taggit-autosuggest django-annoying # Updates for Django 2 @@ -20,9 +22,9 @@ django-bootstrap-pagination django-sortable-listview sorl-thumbnail django-extensions -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 diff --git a/qgis-app/api/tests/test_views.py b/qgis-app/api/tests/test_views.py index 9f0ff30b..acb2cb30 100644 --- a/qgis-app/api/tests/test_views.py +++ b/qgis-app/api/tests/test_views.py @@ -216,7 +216,7 @@ def test_download_resource_should_be_a_file_in_a_zip(self): url = reverse("resource-download", kwargs={"uuid": self.style.uuid}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=style_zero.zip" ) with io.BytesIO(response.content) as file: diff --git a/qgis-app/base/forms/processing_forms.py b/qgis-app/base/forms/processing_forms.py index 771eb3eb..cdf85a7f 100644 --- a/qgis-app/base/forms/processing_forms.py +++ b/qgis-app/base/forms/processing_forms.py @@ -1,6 +1,6 @@ from base.validator import filesize_validator from django import forms -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class ResourceBaseReviewForm(forms.Form): diff --git a/qgis-app/base/models/processing_models.py b/qgis-app/base/models/processing_models.py index 0d6590c5..9b53c625 100644 --- a/qgis-app/base/models/processing_models.py +++ b/qgis-app/base/models/processing_models.py @@ -7,7 +7,7 @@ from django.contrib.auth.models import User from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class UnapprovedManager(models.Manager): diff --git a/qgis-app/base/validator.py b/qgis-app/base/validator.py index 534c4c08..4a491e9e 100644 --- a/qgis-app/base/validator.py +++ b/qgis-app/base/validator.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ RESOURCE_MAX_SIZE = getattr(settings, "RESOURCE_MAX_SIZE", 1000000) # 1MB ERROR_FILESIZE_TOO_BIG = ValidationError( diff --git a/qgis-app/base/views/processing_view.py b/qgis-app/base/views/processing_view.py index bddb4fb5..10087884 100644 --- a/qgis-app/base/views/processing_view.py +++ b/qgis-app/base/views/processing_view.py @@ -17,7 +17,7 @@ from django.urls import reverse, reverse_lazy from django.utils.decorators import method_decorator from django.utils.text import slugify -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.generic import ( CreateView, diff --git a/qgis-app/custom_haystack_urls.py b/qgis-app/custom_haystack_urls.py index a0ab8c23..c14fa7f5 100755 --- a/qgis-app/custom_haystack_urls.py +++ b/qgis-app/custom_haystack_urls.py @@ -1,6 +1,6 @@ # Custom haystack search to match partial strings -from django.conf.urls import include, url +from django.urls import re_path as url from haystack.query import SearchQuerySet from haystack.views import SearchView diff --git a/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py b/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py new file mode 100644 index 00000000..0fc4f397 --- /dev/null +++ b/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('geopackages', '0008_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/geopackages/models.py b/qgis-app/geopackages/models.py index 0cfc14e2..3475f08a 100644 --- a/qgis-app/geopackages/models.py +++ b/qgis-app/geopackages/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ GEOPACKAGES_STORAGE_PATH = getattr( settings, "GEOPACKAGE_STORAGE_PATH", "geopackages/%Y" diff --git a/qgis-app/homepage.py b/qgis-app/homepage.py index 7bf3d7c2..a7e9e51f 100644 --- a/qgis-app/homepage.py +++ b/qgis-app/homepage.py @@ -1,7 +1,7 @@ from django.contrib.flatpages.models import FlatPage from django.shortcuts import render from django.template import RequestContext -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin # from feedjack.models import Post diff --git a/qgis-app/layerdefinitions/file_handler.py b/qgis-app/layerdefinitions/file_handler.py index ddacc02d..ffc196c1 100644 --- a/qgis-app/layerdefinitions/file_handler.py +++ b/qgis-app/layerdefinitions/file_handler.py @@ -6,7 +6,7 @@ import xml.etree.ElementTree as ET from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def parse_qlr(xmlfile): diff --git a/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py b/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py new file mode 100644 index 00000000..97576ae6 --- /dev/null +++ b/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('layerdefinitions', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/layerdefinitions/models.py b/qgis-app/layerdefinitions/models.py index 1c6be6e4..2383b5be 100644 --- a/qgis-app/layerdefinitions/models.py +++ b/qgis-app/layerdefinitions/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ LAYERDEFINITIONS_STORAGE_PATH = getattr( settings, "LAYERDEFINITION_STORAGE_PATH", "layerdefinitions" diff --git a/qgis-app/layerdefinitions/tests/test_views.py b/qgis-app/layerdefinitions/tests/test_views.py index f46119b2..44f8dd28 100644 --- a/qgis-app/layerdefinitions/tests/test_views.py +++ b/qgis-app/layerdefinitions/tests/test_views.py @@ -193,7 +193,7 @@ def test_download_should_return_zipfile_with_custom_license(self): url = reverse("layerdefinition_download", kwargs={"pk": qlr.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=test-qlr-file.zip", ) diff --git a/qgis-app/models/migrations/0007_alter_review_reviewer.py b/qgis-app/models/migrations/0007_alter_review_reviewer.py new file mode 100644 index 00000000..ad03f8c2 --- /dev/null +++ b/qgis-app/models/migrations/0007_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('models', '0006_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/models/models.py b/qgis-app/models/models.py index 1786bd9e..4aa2809f 100644 --- a/qgis-app/models/models.py +++ b/qgis-app/models/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ MODELS_STORAGE_PATH = getattr(settings, "MODELS_STORAGE_PATH", "models/%Y") diff --git a/qgis-app/models/validator.py b/qgis-app/models/validator.py index e4d41829..6241633d 100644 --- a/qgis-app/models/validator.py +++ b/qgis-app/models/validator.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ MODEL_MAX_SIZE = getattr(settings, "MODEL_MAX_SIZE", 1000000) # 1MB diff --git a/qgis-app/plugins/api.py b/qgis-app/plugins/api.py index 62ff6b8e..72615d68 100644 --- a/qgis-app/plugins/api.py +++ b/qgis-app/plugins/api.py @@ -12,7 +12,7 @@ # Transaction from django.db import IntegrityError, connection -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import * from plugins.validator import validator from plugins.views import plugin_notify diff --git a/qgis-app/plugins/celery.py b/qgis-app/plugins/celery.py index 3ec72655..cdc9b450 100644 --- a/qgis-app/plugins/celery.py +++ b/qgis-app/plugins/celery.py @@ -3,6 +3,10 @@ import os from celery import Celery +import logging + +logger = logging.getLogger('plugins') + # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_docker") diff --git a/qgis-app/plugins/forms.py b/qgis-app/plugins/forms.py index 40c8ab5a..9f98f03f 100644 --- a/qgis-app/plugins/forms.py +++ b/qgis-app/plugins/forms.py @@ -5,7 +5,7 @@ from django.contrib.auth.models import User from django.forms import CharField, ModelForm, ValidationError from django.utils.safestring import mark_safe -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin, PluginOutstandingToken, PluginVersion, PluginVersionFeedback from plugins.validator import validator from taggit.forms import * diff --git a/qgis-app/plugins/management/commands/generate_plugins_xml.py b/qgis-app/plugins/management/commands/generate_plugins_xml.py index 4efd70e0..1795dcf9 100644 --- a/qgis-app/plugins/management/commands/generate_plugins_xml.py +++ b/qgis-app/plugins/management/commands/generate_plugins_xml.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.core.management.base import BaseCommand from plugins.tasks.generate_plugins_xml import generate_plugins_xml - +from django.conf import settings class Command(BaseCommand): @@ -12,7 +12,7 @@ def add_arguments(self, parser): "-s", "--site", dest="site", - default="http://plugins.qgis.org", + default=settings.DEFAULT_PLUGINS_SITE, help="Site url to get the source of plugins", ) diff --git a/qgis-app/plugins/middleware.py b/qgis-app/plugins/middleware.py index 17adb828..5d564ba9 100644 --- a/qgis-app/plugins/middleware.py +++ b/qgis-app/plugins/middleware.py @@ -14,7 +14,7 @@ def middleware(request): if auth_basic and not str(auth_basic).startswith('Bearer'): import base64 - username, dummy, password = base64.decodestring( + username, dummy, password = base64.decodebytes( auth_basic[6:].encode("utf8") ).partition(b":") username = username.decode("utf8") diff --git a/qgis-app/plugins/migrations/0004_merge_20231123_0018.py b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py new file mode 100644 index 00000000..8e18926d --- /dev/null +++ b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.11 on 2023-11-23 00:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0002_plugins_feedback'), + ('plugins', '0003_plugin_allow_update_name'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/migrations/0005_auto_20231214_2317.py b/qgis-app/plugins/migrations/0005_auto_20231214_2317.py new file mode 100644 index 00000000..8cd7045b --- /dev/null +++ b/qgis-app/plugins/migrations/0005_auto_20231214_2317.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.25 on 2023-12-14 23:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0004_merge_20231122_0223'), + ] + + operations = [ + migrations.AddField( + model_name='pluginversiondownload', + name='country_code', + field=models.CharField(default='N/D', max_length=3), + ), + migrations.AddField( + model_name='pluginversiondownload', + name='country_name', + field=models.CharField(default='N/D', max_length=100), + ), + ] diff --git a/qgis-app/plugins/migrations/0009_merge_20240321_0207.py b/qgis-app/plugins/migrations/0009_merge_20240321_0207.py new file mode 100644 index 00000000..45cd685a --- /dev/null +++ b/qgis-app/plugins/migrations/0009_merge_20240321_0207.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.11 on 2024-03-21 02:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0004_merge_20231123_0018'), + ('plugins', '0008_merge_20240206_0448'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/migrations/0010_merge_20240517_0729.py b/qgis-app/plugins/migrations/0010_merge_20240517_0729.py new file mode 100644 index 00000000..54355764 --- /dev/null +++ b/qgis-app/plugins/migrations/0010_merge_20240517_0729.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.13 on 2024-05-17 07:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0005_auto_20231214_2317'), + ('plugins', '0009_merge_20240321_0207'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/models.py b/qgis-app/plugins/models.py index 9f9cc209..90245e85 100644 --- a/qgis-app/plugins/models.py +++ b/qgis-app/plugins/models.py @@ -8,7 +8,7 @@ from django.contrib.auth.models import User from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.utils import timezone from djangoratings.fields import AnonymousRatingField from taggit_autosuggest.managers import TaggableManager @@ -943,6 +943,8 @@ class PluginVersionDownload(models.Model): download_date = models.DateField( default=timezone.now ) + country_code = models.CharField(max_length=3, default='N/D') + country_name = models.CharField(max_length=100, default='N/D') download_count = models.IntegerField( default=0 ) diff --git a/qgis-app/plugins/tasks/__init__.py b/qgis-app/plugins/tasks/__init__.py index 7987374c..a0df9010 100644 --- a/qgis-app/plugins/tasks/__init__.py +++ b/qgis-app/plugins/tasks/__init__.py @@ -1 +1,2 @@ -from plugins.tasks.generate_plugins_xml import * +from plugins.tasks.generate_plugins_xml import * # noqa +from plugins.tasks.update_feedjack import * # noqa diff --git a/qgis-app/plugins/tasks/generate_plugins_xml.py b/qgis-app/plugins/tasks/generate_plugins_xml.py index b1e60cef..a7415b60 100644 --- a/qgis-app/plugins/tasks/generate_plugins_xml.py +++ b/qgis-app/plugins/tasks/generate_plugins_xml.py @@ -2,10 +2,15 @@ import requests from celery import shared_task +from celery.utils.log import get_task_logger +from preferences import preferences from django.conf import settings from preferences import preferences +logger = get_task_logger(__name__) + + @shared_task def generate_plugins_xml(site=""): """ @@ -13,6 +18,8 @@ def generate_plugins_xml(site=""): :param site: site domain where the plugins will be fetched, default to http://plugins.qgis.org """ + logger.info('generate_plugins_xml : {}'.format(site)) + if not site: if settings.DEFAULT_PLUGINS_SITE: site = settings.DEFAULT_PLUGINS_SITE diff --git a/qgis-app/plugins/tasks/update_feedjack.py b/qgis-app/plugins/tasks/update_feedjack.py new file mode 100644 index 00000000..e12d14bd --- /dev/null +++ b/qgis-app/plugins/tasks/update_feedjack.py @@ -0,0 +1,10 @@ +from celery import shared_task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@shared_task +def update_feedjack(): + import subprocess + subprocess.call(['python', 'manage.py', 'feedjackupdate']) diff --git a/qgis-app/plugins/templates/plugins/form_snippet.html b/qgis-app/plugins/templates/plugins/form_snippet.html index f33ec0ce..ae794508 100755 --- a/qgis-app/plugins/templates/plugins/form_snippet.html +++ b/qgis-app/plugins/templates/plugins/form_snippet.html @@ -2,7 +2,7 @@
{% for field in form %}
- {% ifequal field.field.widget|klass 'CheckboxInput' %} + {% if field.field.widget|klass == 'CheckboxInput' %}
{% endif %} {{ field }} - {% endifequal %} + {% endif %}
{{ field.help_text }}
{% endfor %} diff --git a/qgis-app/plugins/templates/plugins/pagination.html b/qgis-app/plugins/templates/plugins/pagination.html index fc7e65c7..99bec111 100644 --- a/qgis-app/plugins/templates/plugins/pagination.html +++ b/qgis-app/plugins/templates/plugins/pagination.html @@ -8,11 +8,11 @@ {% endif %} {% for page in pages %} {% if page %} - {% ifequal page page_obj.number %} + {% if page == page_obj.number %} {{ page }} {% else %} {{ page }} - {% endifequal %} + {% endif %} {% else %} ... {% endif %} diff --git a/qgis-app/plugins/templates/plugins/plugin_detail.html b/qgis-app/plugins/templates/plugins/plugin_detail.html index a2ffcc37..726360f9 100644 --- a/qgis-app/plugins/templates/plugins/plugin_detail.html +++ b/qgis-app/plugins/templates/plugins/plugin_detail.html @@ -204,6 +204,7 @@

{{ object.name }}
  • {% trans "Versions" %}
    • {% if user.is_staff or user in object.editors %}
  • {% trans "Manage" %}
    • +
  • {% trans "Stats" %}
    • {% endif %} @@ -362,6 +363,15 @@

    {{ object.name }} +
    + +
    {% endif %} {# end admin #} diff --git a/qgis-app/plugins/tests/test_change_maintainer.py b/qgis-app/plugins/tests/test_change_maintainer.py index a9357f6f..515b5c46 100644 --- a/qgis-app/plugins/tests/test_change_maintainer.py +++ b/qgis-app/plugins/tests/test_change_maintainer.py @@ -50,7 +50,7 @@ def setUp(self): self.plugin = Plugin.objects.get(name='Test Plugin') self.plugin.save() - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_change_maintainer(self): """ diff --git a/qgis-app/plugins/tests/test_download.py b/qgis-app/plugins/tests/test_download.py index f18f0874..34f204bd 100644 --- a/qgis-app/plugins/tests/test_download.py +++ b/qgis-app/plugins/tests/test_download.py @@ -1,11 +1,11 @@ -from django.test import TestCase, RequestFactory +from django.test import Client, TestCase, RequestFactory from django.contrib.auth.models import User from django.utils import timezone from django.core.files.uploadedfile import SimpleUploadedFile from plugins.models import Plugin, PluginVersion, PluginVersionDownload from plugins.views import version_download - +from django.urls import reverse class TestVersionDownloadView(TestCase): def setUp(self): @@ -50,3 +50,19 @@ def test_version_download(self): self.assertEqual(self.version.downloads, 1) self.assertEqual(self.plugin.downloads, 1) self.assertEqual(download_record.download_count, 1) + + def test_version_download_per_country(self): + download_url = reverse('version_download', args=[self.plugin.package_name, self.version.version]) + c = Client(REMOTE_ADDR='180.247.213.170') + response = c.get(download_url) + + self.version.refresh_from_db() + self.plugin.refresh_from_db() + download_record = PluginVersionDownload.objects.get( + plugin_version=self.version, + download_date=timezone.now().date() + ) + + self.assertEqual(response.status_code, 200) + self.assertTrue(download_record.country_code == 'ID') + self.assertTrue(download_record.country_name == 'Indonesia') \ No newline at end of file diff --git a/qgis-app/plugins/tests/test_plugin_update.py b/qgis-app/plugins/tests/test_plugin_update.py index 97e842e2..d1e4c34b 100644 --- a/qgis-app/plugins/tests/test_plugin_update.py +++ b/qgis-app/plugins/tests/test_plugin_update.py @@ -51,7 +51,7 @@ def setUp(self): self.plugin = Plugin.objects.get(name='Test Plugin') - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_new_version(self): """ @@ -103,7 +103,7 @@ def test_plugin_new_version(self): settings.EMAIL_HOST_USER ) - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_version_update(self): """ diff --git a/qgis-app/plugins/tests/test_plugin_upload.py b/qgis-app/plugins/tests/test_plugin_upload.py index 3539701a..9b3d2814 100644 --- a/qgis-app/plugins/tests/test_plugin_upload.py +++ b/qgis-app/plugins/tests/test_plugin_upload.py @@ -34,7 +34,7 @@ def setUp(self): email='test@example.com' ) - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_upload_form(self): # Log in the test user diff --git a/qgis-app/plugins/tests/test_rename_plugin.py b/qgis-app/plugins/tests/test_rename_plugin.py index d1eb890b..78ff7d69 100644 --- a/qgis-app/plugins/tests/test_rename_plugin.py +++ b/qgis-app/plugins/tests/test_rename_plugin.py @@ -51,7 +51,7 @@ def setUp(self): self.plugin.name = "New name Test Plugin" self.plugin.save() - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_rename(self): """ diff --git a/qgis-app/plugins/tests/test_validator.py b/qgis-app/plugins/tests/test_validator.py index 7a0e7e4e..87c5a884 100644 --- a/qgis-app/plugins/tests/test_validator.py +++ b/qgis-app/plugins/tests/test_validator.py @@ -114,16 +114,13 @@ def test_invalid_metadata_web_does_not_exist(self): @mock.patch("requests.get", side_effect=requests.exceptions.SSLError()) def test_check_url_link_ssl_error(self, mock_request): - url = "http://example.com/" - self.assertIsNone(_check_url_link(url, "forbidden_url", "metadata attribute")) + urls = [{'url': "http://example.com/", 'forbidden_url': "forbidden_url", 'metadata_attr': "metadata attribute"}] + self.assertIsNone(_check_url_link(urls)) @mock.patch("requests.get", side_effect=requests.exceptions.HTTPError()) def test_check_url_link_does_not_exist(self, mock_request): - url = "http://example.com/" - self.assertRaises( - ValidationError, - _check_url_link(url, "forbidden_url", "metadata attribute"), - ) + urls = [{'url': "http://example.com/", 'forbidden_url': "forbidden_url", 'metadata_attr': "metadata attribute"}] + self.assertIsNone(_check_url_link(urls)) class TestValidatorForbiddenFileFolder(TestCase): @@ -157,7 +154,10 @@ def test_zipfile_with_MACOSX(self, mock_namelist): mock_namelist.return_value = ["__MACOSX/"] with self.assertRaisesMessage( Exception, - ("For security reasons, zip file cannot contain " "'__MACOSX' directory"), + ( + "For security reasons, zip file cannot contain '__MACOSX' directory. " + "However, there is one present at the root of the archive." + ), ): validator(self.package) @@ -167,8 +167,20 @@ def test_zipfile_with_pycache(self, mock_namelist): with self.assertRaisesMessage( Exception, ( - "For security reasons, zip file cannot contain " - "'__pycache__' directory" + "For security reasons, zip file cannot contain '__pycache__' directory. " + "However, there is one present at the root of the archive." + ), + ): + validator(self.package) + + @mock.patch("zipfile.ZipFile.namelist") + def test_zipfile_with_pycache_in_children(self, mock_namelist): + mock_namelist.return_value = ["path/to/__pycache__/"] + with self.assertRaisesMessage( + Exception, + ( + "For security reasons, zip file cannot contain '__pycache__' directory. " + "However, it has been found at 'path/to/__pycache__/' ." ), ): validator(self.package) @@ -178,7 +190,10 @@ def test_zipfile_with_git(self, mock_namelist): mock_namelist.return_value = [".git"] with self.assertRaisesMessage( Exception, - ("For security reasons, zip file cannot contain " "'.git' directory"), + ( + "For security reasons, zip file cannot contain '.git' directory. " + "However, there is one present at the root of the archive." + ), ): validator(self.package) @@ -191,7 +206,8 @@ def test_zipfile_with_gitignore(self, mock_namelist): exception = cm.exception self.assertNotEqual( exception.message, - "For security reasons, zip file cannot contain '.git' directory", + "For security reasons, zip file cannot contain '.git' directory. ", + "However, there is one present at the root of the archive." ) diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ index c694809e..f5ba6212 100644 Binary files a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ and b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ differ diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ index f838d489..613f0f1f 100644 Binary files a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ and b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ differ diff --git a/qgis-app/plugins/tests/tests.py b/qgis-app/plugins/tests/tests.py index 3748f41b..f51d798f 100644 --- a/qgis-app/plugins/tests/tests.py +++ b/qgis-app/plugins/tests/tests.py @@ -13,7 +13,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = { diff --git a/qgis-app/plugins/urls.py b/qgis-app/plugins/urls.py index dd021892..3d1c5da3 100644 --- a/qgis-app/plugins/urls.py +++ b/qgis-app/plugins/urls.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from django.conf.urls import include, url +from django.urls import re_path as url from django.contrib.auth.decorators import login_required, user_passes_test -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin, PluginVersion from plugins.views import * from rpc4django.views import serve_rpc_request diff --git a/qgis-app/plugins/utils.py b/qgis-app/plugins/utils.py index e180e22b..521a5797 100644 --- a/qgis-app/plugins/utils.py +++ b/qgis-app/plugins/utils.py @@ -1,5 +1,6 @@ import requests import re +from django.http import HttpRequest def extract_version(tag): @@ -47,3 +48,11 @@ def get_qgis_versions(): if version not in all_versions: all_versions.append(version) return all_versions + + +def parse_remote_addr(request: HttpRequest) -> str: + """Extract client IP from request.""" + x_forwarded_for = request.headers.get("X-Forwarded-For", "") + if x_forwarded_for: + return x_forwarded_for.split(",")[0] + return request.META.get("REMOTE_ADDR", "") \ No newline at end of file diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index e1482bd4..1f075f2e 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -15,7 +15,7 @@ from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ PLUGIN_MAX_UPLOAD_SIZE = getattr(settings, "PLUGIN_MAX_UPLOAD_SIZE", 25000000) # 25 mb PLUGIN_REQUIRED_METADATA = getattr( @@ -61,16 +61,16 @@ def _read_from_init(initcontent, initname): i = 0 lines = initcontent.split("\n") while i < len(lines): - if re.search("def\s+([^\(]+)", lines[i]): - k = re.search("def\s+([^\(]+)", lines[i]).groups()[0] + if re.search(r"def\s+([^\(]+)", lines[i]): + k = re.search(r"def\s+([^\(]+)", lines[i]).groups()[0] i += 1 while i < len(lines) and lines[i] != "": - if re.search("return\s+[\"']?([^\"']+)[\"']?", lines[i]): + if re.search(r"return\s+[\"']?([^\"']+)[\"']?", lines[i]): metadata.append( ( k, re.search( - "return\s+[\"']?([^\"']+)[\"']?", lines[i] + r"return\s+[\"']?([^\"']+)[\"']?", lines[i] ).groups()[0], ) ) @@ -86,65 +86,68 @@ def _check_required_metadata(metadata): """ Checks if required metadata are in place, raise ValidationError if not found """ - for md in PLUGIN_REQUIRED_METADATA: - if md not in dict(metadata) or not dict(metadata)[md]: - raise ValidationError( - _( - 'Cannot find metadata %s in metadata source %s.
    For further informations about metadata, please see: metadata documentation' - ) - % (md, dict(metadata).get("metadata_source")) - ) + missing_fields = [field for field in PLUGIN_REQUIRED_METADATA if field not in [item[0] for item in metadata]] + if len(missing_fields) > 0: + missing_fields_str = ', '.join(missing_fields) + raise ValidationError( + _( + f'Cannot find metadata {missing_fields_str} in metadata source {dict(metadata).get("metadata_source")}.
    For further informations about metadata, please see: metadata documentation' + ) + ) -def _check_url_link(url: str, forbidden_url: str, metadata_attr: str) -> None: +def _check_url_link(urls): """ - Checks if the url link is valid. + Checks if all the url link is valid. """ - error_check = ValidationError( - _("Please provide valid url link for %s in metadata.") % metadata_attr + def error_check(url: str, forbidden_url: str)->bool: + # Check against forbidden_url + if url == forbidden_url: + return True + + # Check if parsed URL is valid + try: + parsed_url = urlparse(url) + return not all([parsed_url.scheme, parsed_url.netloc]) + except Exception as e: + # Log the exception or handle it as per your requirement + print(f"Error occurred: {e}") + return True + + def error_check_if_exist(url: str)->bool: + # Check if url is exist + try: + # https://stackoverflow.com/a/41950438/10268058 + # add the headers parameter to make the request appears like coming + # from browser, otherwise some websites will return 403 + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/56.0.2924.76 Safari/537.36" + } + req = requests.head(url, headers=headers) + except requests.exceptions.SSLError: + req = requests.head(url, verify=False) + except Exception: + return True + return req.status_code >= 400 + + url_error = [item for item in [url_item['metadata_attr'] for url_item in urls if error_check(url_item['url'], url_item['forbidden_url'])]] + if len(url_error) > 0: + url_error_str = ", ".join(url_error) + raise ValidationError( + _(f"Please provide valid url link for the following key(s) in the metadata source: {url_error_str}. ") ) - error_check_if_exist = ValidationError( + exist_url_error = [item for item in [url_item['metadata_attr'] for url_item in urls if error_check_if_exist(url_item['url'])]] + if len(exist_url_error) > 0: + exist_url_error_str = ", ".join(exist_url_error) + raise ValidationError( _( - "Please provide valid url link for %s in metadata. " - "This website cannot be reached." - ) - % metadata_attr + f"Please provide valid url link for the following key(s) in the metadata source: {exist_url_error_str}. " + "The website(s) cannot be reached." + ) ) - # check against forbidden_url - is_forbidden_url = url == forbidden_url - if is_forbidden_url: - raise error_check - - # check if parsed url is valid - # https://stackoverflow.com/a/38020041 - try: - parsed_url = urlparse(url) # e.g https://plugins.qgis.org/ - if not ( - all([parsed_url.scheme, parsed_url.netloc]) # e.g http - ): # e.g www.qgis.org - raise error_check - except Exception: - raise error_check - - # Check if url is exist - try: - # https://stackoverflow.com/a/41950438/10268058 - # add the headers parameter to make the request appears like coming - # from browser, otherwise some websites will return 403 - headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/56.0.2924.76 Safari/537.36" - } - req = requests.head(url, headers=headers) - except requests.exceptions.SSLError: - req = requests.head(url, verify=False) - except Exception: - raise error_check_if_exist - if req.status_code >= 400: - raise error_check_if_exist - def validator(package): """ @@ -189,11 +192,19 @@ def validator(package): _("For security reasons, zip file cannot contain .pyc file") ) for forbidden_dir in ["__MACOSX", ".git", "__pycache__"]: - if forbidden_dir in zname.split("/"): + dir_name_list = zname.split("/") + if forbidden_dir in dir_name_list: + if forbidden_dir == dir_name_list[0]: + raise ValidationError( + _( + "For security reasons, zip file " + "cannot contain '%s' directory. However, there is one present at the root of the archive." % (forbidden_dir,) + ) + ) raise ValidationError( _( "For security reasons, zip file " - "cannot contain '%s' directory" % (forbidden_dir,) + "cannot contain '%s' directory. However, it has been found at '%s' ." % (forbidden_dir, zname) ) ) bad_file = zip.testzip() @@ -255,7 +266,7 @@ def validator(package): try: parser = configparser.ConfigParser() parser.optionxform = str - parser.readfp(StringIO(codecs.decode(zip.read(metadataname), "utf8"))) + parser.read_file(StringIO(codecs.decode(zip.read(metadataname), "utf8"))) if not parser.has_section("general"): raise ValidationError( _("Cannot find a section named 'general' in %s") % metadataname @@ -329,11 +340,14 @@ def validator(package): ) % (min_qgs_version, ",".join(e.messages)) ) - # check url_link - _check_url_link(dict(metadata).get("tracker"), "http://bugs", "Bug tracker") - _check_url_link(dict(metadata).get("repository"), "http://repo", "Repository") - _check_url_link(dict(metadata).get("homepage"), "http://homepage", "Home page") + urls_to_check = [ + {'url': dict(metadata).get("tracker"), 'forbidden_url': "http://bugs", 'metadata_attr': "tracker"}, + {'url': dict(metadata).get("repository"), 'forbidden_url': "http://repo", 'metadata_attr': "repository"}, + {'url': dict(metadata).get("homepage"), 'forbidden_url': "http://homepage", 'metadata_attr': "homepage"}, + ] + + _check_url_link(urls_to_check) # Checks for LICENCE file presence diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index d1897117..f6987f9c 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -21,7 +21,7 @@ from django.utils.timezone import now from django.utils.decorators import method_decorator from django.utils.encoding import DjangoUnicodeDecodeError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt, csrf_protect from django.views.decorators.http import require_POST @@ -34,6 +34,8 @@ from plugins.forms import * from plugins.models import Plugin, PluginOutstandingToken, PluginVersion, PluginVersionDownload, vjust from plugins.validator import PLUGIN_REQUIRED_METADATA +from django.contrib.gis.geoip2 import GeoIP2 +from plugins.utils import parse_remote_addr from rest_framework_simplejwt.token_blacklist.models import OutstandingToken from rest_framework_simplejwt.tokens import RefreshToken, api_settings @@ -543,8 +545,10 @@ def get_context_data(self, **kwargs): "%s metadata is missing, this metadata entry is required. Please add %s to metadata.txt." ) % (md, md) messages.error(self.request, msg, fail_silently=True) + stats_url = f"{settings.METABASE_DOWNLOAD_STATS_URL}?package_name={plugin.package_name}#hide_parameters=package_name" context.update( { + "stats_url": stats_url, "rating": plugin.rating.get_rating(), "votes": plugin.rating.votes, } @@ -1364,7 +1368,11 @@ def version_approve(request, package_name, version): return HttpResponseRedirect(version.get_absolute_url()) version.approved = True version.save() - msg = _('The plugin version "%s" is now approved' % version) + msg = _( + "The plugin version '%s' is now approved. " + "Please note that there may be a delay of up to 15 minutes " + "between the approval of the plugin and its actual availability in the XML." + ) % version messages.success(request, msg, fail_silently=True) plugin_approve_notify(version.plugin, msg, request.user) try: @@ -1519,8 +1527,22 @@ def version_download(request, package_name, version): plugin.downloads = plugin.downloads + 1 plugin.save(keep_date=True) + remote_addr = parse_remote_addr(request) + g = GeoIP2() + + if remote_addr: + try: + country_data = g.country(remote_addr) + country_code = country_data['country_code'] + country_name = country_data['country_name'] + except Exception as e: # AddressNotFoundErrors: + country_code = 'N/D' + country_name = 'N/D' + download_record, created = PluginVersionDownload.objects.get_or_create( plugin_version = version, + country_code = country_code, + country_name = country_name, download_date = now().date(), defaults = {'download_count': 1} ) diff --git a/qgis-app/qgis_context_processor.py b/qgis-app/qgis_context_processor.py index 11ff0481..8b8de12c 100644 --- a/qgis-app/qgis_context_processor.py +++ b/qgis-app/qgis_context_processor.py @@ -2,12 +2,14 @@ from django.contrib.sites.models import Site from django.core.exceptions import ImproperlyConfigured +def is_ajax(request): + return request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest' def additions(request): """Insert some additional information into the template context from the settings and set the base template according to qs. """ - if request.is_ajax() or request.GET.get("ajax"): + if is_ajax(request=request) or request.GET.get("ajax"): base_template = "ajax_base.html" is_naked = True else: diff --git a/qgis-app/settings.py b/qgis-app/settings.py index c3e7a79b..16d3cb2d 100644 --- a/qgis-app/settings.py +++ b/qgis-app/settings.py @@ -253,7 +253,7 @@ DEBUG_TOOLBAR_CONFIG = {"INTERCEPT_REDIRECTS": False} -THUMBNAIL_ENGINE = "sorl.thumbnail.engines.convert_engine.Engine" +# THUMBNAIL_ENGINE = "sorl.thumbnail.engines.convert_engine.Engine" USER_MAP = { "project_name": "QGIS", @@ -335,6 +335,7 @@ CELERY_BROKER_URL = BROKER_URL CELERY_RESULT_BACKEND = CELERY_BROKER_URL +GEOIP_PATH='/var/opt/maxmind/' # Token access and refresh validity SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': timedelta(days=15), diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index ee93d537..9d8fcacf 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -1,3 +1,6 @@ +from celery.schedules import crontab + +from settings import * import ast import os @@ -26,7 +29,7 @@ # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = os.environ.get("STATIC_ROOT", "/home/web/static") +STATIC_ROOT = os.environ.get("STATIC_ROOT", "/home/web/static/") # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" @@ -106,7 +109,7 @@ PAGINATION_DEFAULT_PAGINATION_HUB = 30 LOGIN_REDIRECT_URL = "/" SERVE_STATIC_MEDIA = DEBUG -DEFAULT_PLUGINS_SITE = os.environ.get("DEFAULT_PLUGINS_SITE", "") +DEFAULT_PLUGINS_SITE = os.environ.get("DEFAULT_PLUGINS_SITE", "https://plugins.qgis.org/") # See fig.yml file for postfix container definition # @@ -131,6 +134,26 @@ "TEST_REQUEST_DEFAULT_FORMAT": "json", } +GEOIP_PATH='/var/opt/maxmind/' +METABASE_DOWNLOAD_STATS_URL = os.environ.get( + "METABASE_DOWNLOAD_STATS_URL", + "/metabase" +) +CELERY_RESULT_BACKEND = 'rpc://' +CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672') +CELERY_BEAT_SCHEDULE = { + 'generate_plugins_xml': { + 'task': 'plugins.tasks.generate_plugins_xml.generate_plugins_xml', + 'schedule': crontab(minute='*/10'), # Execute every 10 minutes. + 'kwargs': { + 'site': DEFAULT_PLUGINS_SITE + } + }, + 'update_feedjack': { + 'task': 'plugins.tasks.update_feedjack.update_feedjack', + 'schedule': crontab(minute='*/30'), # Execute every 30 minutes. + } +} # Set plugin token access and refresh validity to a very long duration SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': timedelta(days=365*1000), @@ -139,3 +162,6 @@ MATOMO_SITE_ID="1" MATOMO_URL="//matomo.qgis.org/" + +# Default primary key type +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' diff --git a/qgis-app/styles/file_handler.py b/qgis-app/styles/file_handler.py index e35e855f..42b3c2de 100644 --- a/qgis-app/styles/file_handler.py +++ b/qgis-app/styles/file_handler.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def _check_name_type_attribute(element): diff --git a/qgis-app/styles/forms.py b/qgis-app/styles/forms.py index 3fe24047..a23655ff 100644 --- a/qgis-app/styles/forms.py +++ b/qgis-app/styles/forms.py @@ -1,6 +1,6 @@ from django import forms from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from styles.file_handler import validator from styles.models import Style diff --git a/qgis-app/styles/migrations/0015_alter_review_reviewer.py b/qgis-app/styles/migrations/0015_alter_review_reviewer.py new file mode 100644 index 00000000..8c6b0558 --- /dev/null +++ b/qgis-app/styles/migrations/0015_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('styles', '0014_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/styles/models.py b/qgis-app/styles/models.py index f1e4f753..52e62f51 100644 --- a/qgis-app/styles/models.py +++ b/qgis-app/styles/models.py @@ -3,7 +3,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ STYLES_STORAGE_PATH = getattr(settings, "PLUGINS_STORAGE_PATH", "styles/%Y") diff --git a/qgis-app/styles/views.py b/qgis-app/styles/views.py index 84cfb874..db156da4 100644 --- a/qgis-app/styles/views.py +++ b/qgis-app/styles/views.py @@ -19,7 +19,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.urls import reverse, reverse_lazy from django.utils.crypto import get_random_string -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from styles.file_handler import read_xml_style from styles.forms import UpdateForm, UploadForm diff --git a/qgis-app/templates/base.html b/qgis-app/templates/base.html index a702adc1..c8014abb 100644 --- a/qgis-app/templates/base.html +++ b/qgis-app/templates/base.html @@ -59,7 +59,7 @@ {% get_namedmenu Navigation as menu %} -
    +
    {% endif %} -
    +
    {% block content %} {% endblock %}
    @@ -179,6 +179,32 @@
    Silver Sponsors
    + {% tracking_code %} diff --git a/qgis-app/templates/base/includes/detail_object.html b/qgis-app/templates/base/includes/detail_object.html index e1ae3a66..1ad9e4ab 100644 --- a/qgis-app/templates/base/includes/detail_object.html +++ b/qgis-app/templates/base/includes/detail_object.html @@ -7,8 +7,12 @@
    {{ object_detail.description|md_to_html }}
    Uploaded Date
    {{ object_detail.upload_date|date:"d F Y" }}
    +{% if object_detail.review_set.last.review_date and object_detail.approved%}
    Approved Date
    {{ object_detail.review_set.last.review_date|date:"d F Y"}}
    +{% endif %} +
    Modified Date
    +
    {{ object_detail.modified_date|date:"d F Y" }}
    Creator
    {{ object_detail.get_creator_name|title }}
    {% if is_qlr %} diff --git a/qgis-app/templates/base/list.html b/qgis-app/templates/base/list.html index 6c5292b1..529ac62d 100644 --- a/qgis-app/templates/base/list.html +++ b/qgis-app/templates/base/list.html @@ -79,6 +79,7 @@

    {% if title %}{{title}}{% else %}{% trans "All" %} {{ resource_name }}s{% en {% anchor_sort_arrow 'Creator' 'creator' order_by queries %} {% anchor_sort_arrow 'Upload Date' 'upload_date' order_by queries %} + {% anchor_sort_arrow 'Modified Date' 'modified_date' order_by queries %} @@ -105,6 +106,7 @@

    {% if title %}{{title}}{% else %}{% trans "All" %} {{ resource_name }}s{% en {{ object.download_count }} {{ object.get_creator_name|title }} {{ object.upload_date|date:"d F Y" }} + {{ object.modified_date|date:"d F Y" }} {% if user.is_staff or user == style.creator %}   diff --git a/qgis-app/templates/base/list_galery.html b/qgis-app/templates/base/list_galery.html index 9c2f552f..5f230973 100644 --- a/qgis-app/templates/base/list_galery.html +++ b/qgis-app/templates/base/list_galery.html @@ -28,12 +28,12 @@ display: flex; align-items: center; justify-content: center; - height: 420px; + height: 380px; background-color: white; } img.image-demo { height: auto; - max-height: 100%; + max-height: 380px; width: auto; max-width: 100%; } @@ -78,6 +78,7 @@

    {% if title %}{{title}}{% else %}{% trans "All" %} {{ resource_name }}s{% en + {% if object_list.first.style_type %}{% endif %} @@ -106,6 +107,7 @@

    {{ object.get_creator_name|title }} | {{ object.upload_date|date:"d F Y" }} | {{ object.download_count }} downloads +

    Modified Date: {{ object.modified_date|date:"d F Y" }}

    diff --git a/qgis-app/templates/cab/snippet_detail.html b/qgis-app/templates/cab/snippet_detail.html index 38c97359..d47040ae 100644 --- a/qgis-app/templates/cab/snippet_detail.html +++ b/qgis-app/templates/cab/snippet_detail.html @@ -56,9 +56,9 @@

    More like this

    Tools

      {% if user.is_authenticated %} - {% ifequal user.id object.author.id %} + {% if user.id == object.author.id %}
    • Edit this snippet
      • - {% endifequal %} + {% endif %} {% if object|is_bookmarked:request.user %}
    • This snippet is in your bookmarks
      • {% else %} @@ -66,7 +66,7 @@

      Tools

      {% endif %} {% with object|rating_score:request.user as rating_score %} {% if rating_score %} -
    • You rated this snippet {% ifequal rating_score 1 %}useful{% else %}not useful{% endifequal %}.
      • +
    • You rated this snippet {% if rating_score == 1 %}useful{% else %}not useful{% endif %}.
      • {% else %}
    • Rate this snippet: useful or not useful
      • {% endif %} diff --git a/qgis-app/templates/search/search.html b/qgis-app/templates/search/search.html index 01f9f7e4..44c51b89 100644 --- a/qgis-app/templates/search/search.html +++ b/qgis-app/templates/search/search.html @@ -46,7 +46,7 @@

      Search

    {% if query %} -

    Search results

    +

    Search results for "{{query}}" ({{paginator.count}} items found)

    {% for result in page.object_list %}

    @@ -58,9 +58,27 @@

    Search results

    {% if page.has_previous or page.has_next %} {% endif %} {% else %} diff --git a/qgis-app/templates/snippet_base.html b/qgis-app/templates/snippet_base.html index d1a359fa..0e4c29b5 100644 --- a/qgis-app/templates/snippet_base.html +++ b/qgis-app/templates/snippet_base.html @@ -36,7 +36,7 @@

    QGIS Snippets

    {% if user.is_authenticated %} Logged in as {{ user.username }} (Add a snippet) (Log out) {% else %} - Log in or register + Log in or register {% endif %}
      diff --git a/qgis-app/urls.py b/qgis-app/urls.py index 7166f5c1..90acac68 100644 --- a/qgis-app/urls.py +++ b/qgis-app/urls.py @@ -1,6 +1,6 @@ import simplemenu from django.conf import settings -from django.conf.urls import url +from django.urls import re_path as url # Uncomment the next two lines to enable the admin: from django.contrib import admin diff --git a/qgis-app/userexport/urls.py b/qgis-app/userexport/urls.py index 84d6a35c..45ac6d08 100755 --- a/qgis-app/userexport/urls.py +++ b/qgis-app/userexport/urls.py @@ -1,5 +1,5 @@ # -* coding:utf-8 *- # -from django.conf.urls import * +from django.urls import re_path as url from userexport.views import * urlpatterns = [ diff --git a/qgis-app/users/tests.py b/qgis-app/users/tests.py index 3748f41b..f51d798f 100644 --- a/qgis-app/users/tests.py +++ b/qgis-app/users/tests.py @@ -13,7 +13,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = { diff --git a/qgis-app/users/urls.py b/qgis-app/users/urls.py index dbcb3317..dd51a6d9 100644 --- a/qgis-app/users/urls.py +++ b/qgis-app/users/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import include, url +from django.urls import re_path as url from django.views.generic.base import TemplateView from .views import * diff --git a/qgis-app/users/views.py b/qgis-app/users/views.py index 46234176..25d08f3e 100644 --- a/qgis-app/users/views.py +++ b/qgis-app/users/views.py @@ -1,21 +1,23 @@ # python logging support to django logging middleware -import logging - -from annoying.functions import get_object_or_None -from django.conf import settings +from django.http import HttpResponse +from django.http import HttpResponseRedirect from django.contrib import messages -from django.contrib.sites.models import Site -from django.core.mail import send_mail -from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render_to_response +from django.shortcuts import render, get_object_or_404 +from annoying.functions import get_object_or_None from django.template import RequestContext +from django.utils.translation import gettext_lazy as _ +from django.core.mail import send_mail from django.template.loader import render_to_string -from django.utils.translation import ugettext_lazy as _ -from olwidget.fields import EditableLayerField, MapField -from olwidget.widgets import EditableLayer, InfoLayer, InfoMap, Map +from django.contrib.sites.models import Site +from django.conf import settings from users.forms import * from users.models import * +from olwidget.fields import MapField, EditableLayerField +from olwidget.widgets import Map, EditableLayer, InfoLayer, InfoMap +import os +# python logging support to django logging middleware +import logging def usersMap(theRequest): @@ -32,16 +34,15 @@ def usersMap(theRequest): myMap = InfoMap(users) - return render_to_response( + return render( + theRequest, "view_users.html", { - "myMap": myMap, - "myUserCount": myUserCount, - "myRandomUser": myRandomUser, + 'myMap' : myMap, + 'myUserCount' : myUserCount, + 'myRandomUser' : myRandomUser, }, - context_instance=RequestContext(theRequest), - ) - + context_instance=RequestContext(theRequest)) def createUser(theRequest): @@ -52,41 +53,29 @@ def createUser(theRequest): myForm.save() return HttpResponseRedirect("/community-map/view_users.html") else: - return render_to_response( - "create_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + context_instance=RequestContext(theRequest)) else: myForm = QgisUserForm() - return render_to_response( - "create_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + context_instance=RequestContext(theRequest)) def updateUser(theRequest, theId): - myUserCount = QgisUser.objects.all().count() - myUser = get_object_or_404(QgisUser, guid=theId) - if theRequest.method == "POST": - myForm = QgisUserForm(theRequest.POST, theRequest.FILES, instance=myUser) - if myForm.is_valid(): - myForm.save() - return HttpResponseRedirect("/community-map/view_users.html") - return render_to_response( - "update_user_form.html", - {"myUser": myUser, "myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) - else: - myForm = QgisUserForm(instance=myUser) - return render_to_response( - "update_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) - + myUserCount = QgisUser.objects.all().count() + myUser = get_object_or_404(QgisUser,guid=theId) + if theRequest.method == 'POST': + myForm = QgisUserForm(theRequest.POST, theRequest.FILES, instance=myUser) + if myForm.is_valid(): + myForm.save() + return HttpResponseRedirect("/community-map/view_users.html") + return render(theRequest, "update_user_form.html", { + 'myUser': myUser, 'myForm': myForm, 'myUserCount' : myUserCount + }, context_instance=RequestContext(theRequest)) + else: + myForm = QgisUserForm(instance=myUser) + return render(theRequest, "update_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + context_instance=RequestContext(theRequest)) def emailEditAddress(theRequest): @@ -115,22 +104,17 @@ def emailEditAddress(theRequest): send_mail(subject, message, sender, [recipient]) return HttpResponseRedirect("/community-map/edit/email_confirm.html") + else: myUserCount = QgisUser.objects.all().count() msg = _("User is NOT valid.") messages.warning(theRequest, msg, fail_silently=True) logging.info("User or form is NOT valid") - return render_to_response( - "update_user.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "update_user.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + context_instance=RequestContext(theRequest)) else: myUserCount = QgisUser.objects.all().count() myForm = EmailForm() - return render_to_response( - "update_user.html", - {"myForm": myForm}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "update_user.html", {'myForm' : myForm}, + context_instance=RequestContext(theRequest)) diff --git a/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py b/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py new file mode 100644 index 00000000..79efde7f --- /dev/null +++ b/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('wavefronts', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/wavefronts/models.py b/qgis-app/wavefronts/models.py index 7c5c52f4..65b10e85 100644 --- a/qgis-app/wavefronts/models.py +++ b/qgis-app/wavefronts/models.py @@ -6,7 +6,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from preferences.models import Preferences WAVEFRONTS_STORAGE_PATH = getattr(settings, "WAVEFRONTS_STORAGE_PATH", "wavefronts") diff --git a/qgis-app/wavefronts/tests/test_view.py b/qgis-app/wavefronts/tests/test_view.py index 364f4929..380848e6 100644 --- a/qgis-app/wavefronts/tests/test_view.py +++ b/qgis-app/wavefronts/tests/test_view.py @@ -170,7 +170,7 @@ def test_download_should_return_zipfile(self): url = reverse("wavefront_download", kwargs={"pk": wavefront.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=odm-texturing.zip", ) diff --git a/qgis-app/wavefronts/validator.py b/qgis-app/wavefronts/validator.py index 09e8f338..deff6be8 100644 --- a/qgis-app/wavefronts/validator.py +++ b/qgis-app/wavefronts/validator.py @@ -9,7 +9,7 @@ import pywavefront from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from wavefronts.models import WAVEFRONTS_STORAGE_PATH diff --git a/qgis-app/wavefronts/views.py b/qgis-app/wavefronts/views.py index e7c11151..a93458ea 100644 --- a/qgis-app/wavefronts/views.py +++ b/qgis-app/wavefronts/views.py @@ -20,7 +20,7 @@ from django.template.response import TemplateResponse from django.urls import reverse_lazy from django.utils.text import slugify -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from wavefronts.forms import UpdateForm, UploadForm from wavefronts.models import Review, Wavefront from wavefronts.utilities import zipped_all_with_license diff --git a/qgis-app/wsgi.py b/qgis-app/wsgi.py index d56c6515..71f8df99 100644 --- a/qgis-app/wsgi.py +++ b/qgis-app/wsgi.py @@ -4,7 +4,7 @@ It exposes the WSGI callable as a module-level variable named ``application``. For more information on this file, see -https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ """ import os diff --git a/readme.md b/readme.md index ec2ea04c..56f730e2 100644 --- a/readme.md +++ b/readme.md @@ -33,7 +33,7 @@ docker and rancher. ## Token based authentication -Users have the ability to generate a Simple JWT token by providing their credentials, which can then be utilized to access endpoints requiring authentication. +Users can generate a Simple JWT token by providing their credentials, which can then be utilized to access endpoints requiring authentication. Users can create specific tokens for a plugin at `https://plugins.qgis.org//tokens/`. diff --git a/setup.cfg b/setup.cfg index fae25c68..2fa09d19 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ exclude = .git, .venv/, ./.venv/, + venv/, vagrant_assets, qgis-app/*/migrations/, qgis-app/plugins/tests/HelloWorld/,