diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7ab91dc3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dockerize/postgres_data diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index afc17f03..8166e470 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -48,23 +48,29 @@ jobs: - uses: actions/checkout@v2 - name: Run docker-compose build - run: docker-compose build + run: docker-compose build devweb - - name: Run the containers - run: docker-compose up -d db devweb + - name: Run docker-compose services + working-directory: dockerize + run: | + cp docker-compose.override.test.yml docker-compose.override.yml + make devweb-test + make wait-db + make create-test-db - name: Run Coverage test run: | - cat << EOF | docker-compose exec -T devweb bash + cat << EOF | docker-compose -p qgis-plugins exec -T devweb bash pip install coverage python manage.py makemigrations python manage.py migrate + python manage.py collectstatic --noinput --verbosity 0 coverage run manage.py test coverage xml EOF - name: Upload coverage to codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 4267cc6f..073c345f 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ qgis-app/api/tests/*/ # whoosh_index qgis-app/whoosh_index/ +docker-compose.override.yml +.env diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 2e741ec8..a92fb708 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -1,17 +1,21 @@ -Django==2.2.25 +Django==3.2.11 + # Currently broken with 'no module named defaults' error #Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git -# George's is also broken: use my fork (django 1.8 ready) -git+https://github.com/elpaso/feedjack.git +# George's is also broken: use elpaso fork (django 1.8 ready) +# git+https://github.com/elpaso/feedjack.git +# His is also broken, use dimasciput (django 3.2 ready) +git+https://github.com/dimasciput/feedjack.git + Markdown==2.3.1 #PIL==1.1.7 Pillow Pygments==2.7.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 @@ -21,16 +25,18 @@ argparse==1.2.1 django-annoying==0.7.7 django-auth-ldap==1.2.6 django-autoslug==1.7.1 -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 django-endless-pagination==2.0 django-extensions==1.2.0 django-generic-aggregation==0.3.2 #django-olwidget==0.61.0 unmaintained, use this fork git+https://github.com/Christophe31/olwidget.git django-pagination==1.0.7 + # Unmaintained! #django-ratings==0.3.7 -git+https://github.com/enikesha/django-ratings.git +git+https://github.com/gelo-zhukov/django-ratings.git + django-simple-ratings==0.3.2 # SIMPLEMENU git+https://github.com/elpaso/django-simplemenu.git diff --git a/dockerize/.env.template b/dockerize/.env.template new file mode 100644 index 00000000..cd4fbcfe --- /dev/null +++ b/dockerize/.env.template @@ -0,0 +1,6 @@ +RABBITMQ_HOST=rabbitmq +DATABASE_NAME=gis +DATABASE_USERNAME=docker +DATABASE_PASSWORD=docker +DATABASE_HOST=db +DJANGO_SETTINGS_MODULE=settings_docker diff --git a/dockerize/Makefile b/dockerize/Makefile index 21838313..1ad35681 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -15,6 +15,13 @@ build: @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) build +build-dev: + @echo + @echo "------------------------------------------------------------------" + @echo "Building in development mode only" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) build devweb + db: @echo @echo "------------------------------------------------------------------" @@ -22,19 +29,33 @@ db: @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) up -d db +metabase: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running metabase in production mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) up -d metabase + web: db @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d web + @docker-compose -p $(PROJECT_ID) up -d uwsgi web smtp worker beat + +devweb-test: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in TESTING mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb devweb: db @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat devweb-runserver: devweb @echo @@ -54,26 +75,26 @@ migrate: @# We add the '-' prefix to the next line as the migration may fail @# but we want to continue anyway. @#We need to migrate accounts first as it has a reference to user model - -@docker-compose -p $(PROJECT_ID) exec web python manage.py migrate auth - @docker-compose -p $(PROJECT_ID) exec web python manage.py migrate + -@docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth + @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate update-migrations: @echo @echo "------------------------------------------------------------------" @echo "Running update migrations in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web python manage.py makemigrations + @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations collectstatic: @echo @echo "------------------------------------------------------------------" @echo "Collecting static in production mode" @echo "------------------------------------------------------------------" - #@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput + @docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput #We need to run collect static in the same context as the running # uwsgi container it seems so I use docker exec here # no -it flag so we can run over remote shell - @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput + # @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput reload: @echo @@ -100,12 +121,27 @@ rm-only: kill @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) rm +maillogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp logs in production mode" + @echo "------------------------------------------------------------------" + @docker-compose exec smtp tail -f /var/log/mail.log + +mailerrorlogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp error logs in production mode" + @echo "------------------------------------------------------------------" + @docker-compose exec smtp tail -f /var/log/mail.err + dbrestore: @echo @echo "------------------------------------------------------------------" @echo "Restore dump from backups/latest.dmp in production mode" @echo "------------------------------------------------------------------" @# - prefix causes command to continue even if it fails + @# - TODO: This command needs update since postgis has been updated. @echo "stopping web container" @docker-compose -p $(PROJECT_ID) stop web @echo "dropping gis" @@ -120,6 +156,13 @@ dbrestore: @docker-compose -p $(PROJECT_ID) start web @echo "starting web container" +wait-db: + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" + +create-test-db: + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -c 'create database test_db;'" + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -d test_db -c 'create extension postgis;'" + dbseed: @echo @echo "------------------------------------------------------------------" @@ -132,4 +175,4 @@ rebuild_index: @echo "------------------------------------------------------------------" @echo "Rebuild search index in PRODUCTION mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web bash -c 'python manage.py rebuild_index' + @docker-compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' diff --git a/dockerize/docker-compose.override.template.yml b/dockerize/docker-compose.override.template.yml new file mode 100644 index 00000000..bb0fdd8a --- /dev/null +++ b/dockerize/docker-compose.override.template.yml @@ -0,0 +1,55 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" + + beat: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + worker: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + uwsgi: + container_name: qgis-plugins-uwsgi + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod + + db: + volumes: + - ./postgres_data:/var/lib/postgresql + - ./backups:/backups + + web: + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + - ./static:/home/web/static:ro + - ./media:/home/web/media:ro + ports: + - "62201:8080" diff --git a/dockerize/docker-compose.override.test.yml b/dockerize/docker-compose.override.test.yml new file mode 100644 index 00000000..e10fe9a1 --- /dev/null +++ b/dockerize/docker-compose.override.test.yml @@ -0,0 +1,19 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 80578f9f..102bb6cb 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,70 +1,74 @@ -version: "3.8" +version: '3.8' volumes: - django-statics-data: {} - django-media-data: {} + backups-data: + static-data: + media-data: + logs-data: + nginx-conf: + db-backups: + rabbitmq: + celerybeat-schedule: + postgres-data: services: + + smtp: + image: catatnight/postfix:latest + hostname: postfix + environment: + - maildomain=qgis.org + - smtp_user=noreply:docker + restart: unless-stopped + db: container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 + image: kartoza/postgis:14-3.3 environment: - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - PASSWORD_AUTHENTICATION=${PASSWORD_AUTHENTICATION:-md5} volumes: - - ./backups:/backups + - postgres-data:/var/lib/postgresql + - backups-data:/backups restart: unless-stopped - web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker + uwsgi: &uwsgi-common + container_name: qgis-plugins-uwsgi-common + image: kartoza/qgis-plugins-uwsgi hostname: uwsgi environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq + - DATABASE_NAME=${DATABASE_NAME:-gis} + - DATABASE_USERNAME=${DATABASE_USERNAME:-docker} + - DATABASE_PASSWORD=${DATABASE_PASSWORD:-docker} + - DATABASE_HOST=${DATABASE_HOST:-db} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-settings_docker} + - VIRTUAL_HOST=${VIRTUAL_HOST:-plugins.kartoza.com} + - VIRTUAL_PORT=${VIRTUAL_PORT:-8080} + - DEBUG=${DEBUG:-False} + - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} + - BROKER_URL=amqp://rabbitmq:5672 volumes: - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw + - static-data:/home/web/static:rw + - media-data:/home/web/media:rw + - celerybeat-schedule:/home/web/celerybeat-schedule:rw links: - db:db - rabbitmq:rabbitmq - - worker:worker restart: unless-stopped user: root - command: uwsgi --ini /uwsgi.conf + # This is the entry point for a development server. + # Run with --no-deps to run attached to the services + # from prod environment if wanted devweb: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq + <<: *uwsgi-common + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev volumes: - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root ports: # for django test server - "62202:8080" @@ -72,50 +76,71 @@ services: - "62203:22" rabbitmq: - image: library/rabbitmq:3.6 + image: rabbitmq:3.7-alpine hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit + volumes: + - rabbitmq:/var/lib/rabbitmq restart: unless-stopped + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + worker: - # Note you cannot scale if you use container_name + <<: *uwsgi-common container_name: qgis-plugins-worker - build: docker - hostname: uwsgi - working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw links: - - db:db - - rabbitmq:rabbitmq + - db + - rabbitmq + - beat + working_dir: /home/web/django_project + entrypoint: [] + command: celery -A plugins worker -l INFO - nginx: + web: # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx + container_name: qgis-plugins-web image: nginx - hostname: nginx + hostname: web volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - - django-statics-data:/home/web/static:ro - - django-media-data:/home/web/media:ro - - ./logs:/var/log/nginx + - nginx-conf:/etc/nginx/conf.d:ro + - static-data:/home/web/static:ro + - media-data:/home/web/media:ro links: - - web:uwsgi - ports: - - "62201:8080" + - uwsgi:uwsgi + - metabase:metabase + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + restart: unless-stopped + + dbbackups: + image: kartoza/pg-backup:14-3.3 + hostname: pg-backups + volumes: + - db-backups:/backups + links: + - db:db + environment: + # take care to let the project name below match that + # declared in the top of the makefile + - DUMPPREFIX=${DUMPPREFIX:-QGIS_PLUGINS} + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_HOST=${DATABASE_HOST:-db} + - PGDATABASE=${DATABASE_NAME:-gis} restart: unless-stopped + + metabase: + image: metabase/metabase:latest + environment: + - MB_DB_TYPE=postgres + - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} + links: + - db diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 3bd21f73..9e43d497 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -1,6 +1,6 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ # Note this base image is based on debian -FROM kartoza/django-base:3.7 +FROM kartoza/django-base:3.7 as prod MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl @@ -10,13 +10,30 @@ RUN apt-get clean all RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev -ADD REQUIREMENTS.txt /REQUIREMENTS.txt -RUN pip install -r /REQUIREMENTS.txt + +RUN rm -rf /uwsgi.conf +ADD dockerize/docker/uwsgi.conf /uwsgi.conf +ADD qgis-app /home/web/django_project +ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt +RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt RUN pip install uwsgi freezegun==1.3.1 + +# Open port 8080 as we will be running our uwsgi socket on that +EXPOSE 8080 + +RUN mkdir -p /var/log/uwsgi + +WORKDIR /home/web/django_project +CMD ["uwsgi", "--ini", "/uwsgi.conf"] + + +FROM prod as dev + +# This section taken on 2 July 2015 from # https://docs.docker.com/examples/running_ssh_service/ # Sudo is needed by pycharm when it tries to pip install packages -RUN apt-get install -y openssh-server sudo +RUN apt-get update && apt-get install -y openssh-server sudo RUN mkdir /var/run/sshd RUN echo 'root:docker' | chpasswd RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config @@ -27,15 +44,16 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile -RUN rm -rf /uwsgi.conf -ADD uwsgi.conf /uwsgi.conf +# Install freezegun for feedback test +RUN pip install freezegun -# Open port 8080 as we will be running our uwsgi socket on that -EXPOSE 8080 +# -------------------------------------------------------- +# Open ports as needed +# -------------------------------------------------------- +# Open port 8080 as we will be running our django dev server on +EXPOSE 8080 # Open port 22 as we will be using a remote interpreter from pycharm EXPOSE 22 -RUN mkdir -p /var/log/uwsgi -WORKDIR /home/web/django_project CMD ["/usr/sbin/sshd", "-D"] diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index 4de38450..cfe52640 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -1,4 +1,4 @@ -django==2.2.25 +django==3.2.11 django-auth-ldap python-ldap django-taggit==2.0.0 @@ -6,8 +6,10 @@ django-tinymce==3.4.0 psycopg2 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git -# Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize + +# Updates for Django 3 +git+https://github.com/gelo-zhukov/django-ratings.git + django-taggit-autosuggest django-annoying # Updates for Django 2 @@ -20,10 +22,11 @@ django-bootstrap-pagination django-sortable-listview sorl-thumbnail django-extensions -django-debug-toolbar==1.11.1 + +django-debug-toolbar==3.2.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 @@ -32,10 +35,12 @@ django-haystack==3.2.1 # git+https://github.com/Erve1879/feedjack.git # George's is also broken: use my fork (django 1.8 ready) # git+https://github.com/elpaso/feedjack.git -# His is also broken, use mine (django 2.2 ready) +# His is also broken, use dimasciput (django 3.2 ready) git+https://github.com/dimasciput/feedjack.git + feedparser==5.2.1 -celery==4.3.1 +celery==5.2.7 +django-celery-beat==2.3.0 # pin due to issues with a breaking change # https://github.com/celery/celery/issues/7783 diff --git a/dockerize/production/Dockerfile b/dockerize/production/Dockerfile index 7a57d864..9453fffc 100644 --- a/dockerize/production/Dockerfile +++ b/dockerize/production/Dockerfile @@ -5,6 +5,10 @@ MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all + +# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 +RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list + RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev ARG BRANCH_TAG=develop @@ -14,6 +18,7 @@ RUN mkdir -p /usr/src; mkdir -p /home/web && \ ln -s /usr/src/plugins/qgis-app /home/web/django_project RUN cd /usr/src/plugins/dockerize/docker && \ + pip install --upgrade pip && \ pip install -r REQUIREMENTS.txt && \ pip install uwsgi && \ rm -rf /uwsgi.conf && \ diff --git a/dockerize/sites-enabled/default.conf b/dockerize/sites-enabled/default.conf index e870a2a6..8cf7cec7 100644 --- a/dockerize/sites-enabled/default.conf +++ b/dockerize/sites-enabled/default.conf @@ -28,6 +28,8 @@ server { } # max upload size, adjust to taste client_max_body_size 15M; + + # Django media location /media { # your Django project's media files - amend as required @@ -52,6 +54,15 @@ server { alias /home/web/archive; expires 21d; # cache for 6h } + + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + # Finally, send all non-media requests to the Django server. location / { uwsgi_pass uwsgi; @@ -74,4 +85,10 @@ server { uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; } + + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } } diff --git a/qgis-app/REQUIREMENTS_plugins.txt b/qgis-app/REQUIREMENTS_plugins.txt index fa9c03ac..ebb39373 100644 --- a/qgis-app/REQUIREMENTS_plugins.txt +++ b/qgis-app/REQUIREMENTS_plugins.txt @@ -1,4 +1,4 @@ -django==2.2.25 +django==3.2.11 django-auth-ldap python-ldap django-taggit==2.0.0 @@ -6,8 +6,10 @@ django-tinymce==3.4.0 psycopg2 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git -# Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize + +# Updates for Django 3 +git+https://github.com/gelo-zhukov/django-ratings.git + django-taggit-autosuggest django-annoying # Updates for Django 2 @@ -20,9 +22,9 @@ django-bootstrap-pagination django-sortable-listview sorl-thumbnail django-extensions -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 diff --git a/qgis-app/plugins/celery.py b/qgis-app/plugins/celery.py index 3ec72655..cdc9b450 100644 --- a/qgis-app/plugins/celery.py +++ b/qgis-app/plugins/celery.py @@ -3,6 +3,10 @@ import os from celery import Celery +import logging + +logger = logging.getLogger('plugins') + # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_docker") diff --git a/qgis-app/plugins/migrations/0004_merge_20231123_0018.py b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py new file mode 100644 index 00000000..8e18926d --- /dev/null +++ b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.11 on 2023-11-23 00:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0002_plugins_feedback'), + ('plugins', '0003_plugin_allow_update_name'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/tasks/__init__.py b/qgis-app/plugins/tasks/__init__.py index 7987374c..a0df9010 100644 --- a/qgis-app/plugins/tasks/__init__.py +++ b/qgis-app/plugins/tasks/__init__.py @@ -1 +1,2 @@ -from plugins.tasks.generate_plugins_xml import * +from plugins.tasks.generate_plugins_xml import * # noqa +from plugins.tasks.update_feedjack import * # noqa diff --git a/qgis-app/plugins/tasks/generate_plugins_xml.py b/qgis-app/plugins/tasks/generate_plugins_xml.py index b1e60cef..a7415b60 100644 --- a/qgis-app/plugins/tasks/generate_plugins_xml.py +++ b/qgis-app/plugins/tasks/generate_plugins_xml.py @@ -2,10 +2,15 @@ import requests from celery import shared_task +from celery.utils.log import get_task_logger +from preferences import preferences from django.conf import settings from preferences import preferences +logger = get_task_logger(__name__) + + @shared_task def generate_plugins_xml(site=""): """ @@ -13,6 +18,8 @@ def generate_plugins_xml(site=""): :param site: site domain where the plugins will be fetched, default to http://plugins.qgis.org """ + logger.info('generate_plugins_xml : {}'.format(site)) + if not site: if settings.DEFAULT_PLUGINS_SITE: site = settings.DEFAULT_PLUGINS_SITE diff --git a/qgis-app/plugins/tasks/update_feedjack.py b/qgis-app/plugins/tasks/update_feedjack.py new file mode 100644 index 00000000..e12d14bd --- /dev/null +++ b/qgis-app/plugins/tasks/update_feedjack.py @@ -0,0 +1,10 @@ +from celery import shared_task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@shared_task +def update_feedjack(): + import subprocess + subprocess.call(['python', 'manage.py', 'feedjackupdate']) diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ index c694809e..f5ba6212 100644 Binary files a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ and b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ differ diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ index f838d489..613f0f1f 100644 Binary files a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ and b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ differ diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index bb151b7f..b54836d0 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -1,3 +1,6 @@ +from celery.schedules import crontab + +from settings import * import ast import os @@ -126,6 +129,21 @@ "TEST_REQUEST_DEFAULT_FORMAT": "json", } +CELERY_RESULT_BACKEND = 'rpc://' +CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672') +CELERY_BEAT_SCHEDULE = { + 'generate_plugins_xml': { + 'task': 'plugins.tasks.generate_plugins_xml.generate_plugins_xml', + 'schedule': crontab(minute='*/10'), # Execute every 10 minutes. + 'kwargs': { + 'site': 'https://plugins.qgis.org/' + } + }, + 'update_feedjack': { + 'task': 'plugins.tasks.update_feedjack.update_feedjack', + 'schedule': crontab(minute='*/30'), # Execute every 30 minutes. + } +} # Set plugin token access and refresh validity to a very long duration SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': timedelta(days=365*1000), diff --git a/qgis-app/users/views.py b/qgis-app/users/views.py index 46234176..05774ac4 100644 --- a/qgis-app/users/views.py +++ b/qgis-app/users/views.py @@ -1,21 +1,23 @@ # python logging support to django logging middleware -import logging - -from annoying.functions import get_object_or_None -from django.conf import settings +from django.http import HttpResponse +from django.http import HttpResponseRedirect from django.contrib import messages -from django.contrib.sites.models import Site -from django.core.mail import send_mail -from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render_to_response +from django.shortcuts import render, get_object_or_404 +from annoying.functions import get_object_or_None from django.template import RequestContext -from django.template.loader import render_to_string from django.utils.translation import ugettext_lazy as _ -from olwidget.fields import EditableLayerField, MapField -from olwidget.widgets import EditableLayer, InfoLayer, InfoMap, Map +from django.core.mail import send_mail +from django.template.loader import render_to_string +from django.contrib.sites.models import Site +from django.conf import settings from users.forms import * from users.models import * +from olwidget.fields import MapField, EditableLayerField +from olwidget.widgets import Map, EditableLayer, InfoLayer, InfoMap +import os +# python logging support to django logging middleware +import logging def usersMap(theRequest): @@ -32,16 +34,15 @@ def usersMap(theRequest): myMap = InfoMap(users) - return render_to_response( + return render( + theRequest, "view_users.html", { - "myMap": myMap, - "myUserCount": myUserCount, - "myRandomUser": myRandomUser, + 'myMap' : myMap, + 'myUserCount' : myUserCount, + 'myRandomUser' : myRandomUser, }, - context_instance=RequestContext(theRequest), - ) - + context_instance=RequestContext(theRequest)) def createUser(theRequest): @@ -52,41 +53,29 @@ def createUser(theRequest): myForm.save() return HttpResponseRedirect("/community-map/view_users.html") else: - return render_to_response( - "create_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + context_instance=RequestContext(theRequest)) else: myForm = QgisUserForm() - return render_to_response( - "create_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + context_instance=RequestContext(theRequest)) def updateUser(theRequest, theId): - myUserCount = QgisUser.objects.all().count() - myUser = get_object_or_404(QgisUser, guid=theId) - if theRequest.method == "POST": - myForm = QgisUserForm(theRequest.POST, theRequest.FILES, instance=myUser) - if myForm.is_valid(): - myForm.save() - return HttpResponseRedirect("/community-map/view_users.html") - return render_to_response( - "update_user_form.html", - {"myUser": myUser, "myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) - else: - myForm = QgisUserForm(instance=myUser) - return render_to_response( - "update_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) - + myUserCount = QgisUser.objects.all().count() + myUser = get_object_or_404(QgisUser,guid=theId) + if theRequest.method == 'POST': + myForm = QgisUserForm(theRequest.POST, theRequest.FILES, instance=myUser) + if myForm.is_valid(): + myForm.save() + return HttpResponseRedirect("/community-map/view_users.html") + return render(theRequest, "update_user_form.html", { + 'myUser': myUser, 'myForm': myForm, 'myUserCount' : myUserCount + }, context_instance=RequestContext(theRequest)) + else: + myForm = QgisUserForm(instance=myUser) + return render(theRequest, "update_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + context_instance=RequestContext(theRequest)) def emailEditAddress(theRequest): @@ -115,22 +104,17 @@ def emailEditAddress(theRequest): send_mail(subject, message, sender, [recipient]) return HttpResponseRedirect("/community-map/edit/email_confirm.html") + else: myUserCount = QgisUser.objects.all().count() msg = _("User is NOT valid.") messages.warning(theRequest, msg, fail_silently=True) logging.info("User or form is NOT valid") - return render_to_response( - "update_user.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "update_user.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + context_instance=RequestContext(theRequest)) else: myUserCount = QgisUser.objects.all().count() myForm = EmailForm() - return render_to_response( - "update_user.html", - {"myForm": myForm}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "update_user.html", {'myForm' : myForm}, + context_instance=RequestContext(theRequest)) diff --git a/setup.cfg b/setup.cfg index fae25c68..2fa09d19 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ exclude = .git, .venv/, ./.venv/, + venv/, vagrant_assets, qgis-app/*/migrations/, qgis-app/plugins/tests/HelloWorld/,