Skip to content

Commit

Permalink
Merge pull request qgis#403 from Xpirix/fix_uwsgi_load_balancer
Browse files Browse the repository at this point in the history
Use internal network to fix uwsgi load balancer
  • Loading branch information
Xpirix authored May 15, 2024
2 parents 07412e8 + d4102d4 commit 47ddc04
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 122 deletions.
46 changes: 35 additions & 11 deletions dockerize/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ services:
- postgres_data:/opt/postgres/data
- ${QGISPLUGINS_BACKUP_VOLUME}:/backups
restart: unless-stopped
networks:
internal:

uwsgi: &uwsgi-common
build:
Expand Down Expand Up @@ -57,11 +59,14 @@ services:
- ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw
- ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw
- celerybeat-schedule:/home/web/celerybeat-schedule:rw
links:
- db:db
- rabbitmq:rabbitmq

depends_on:
- db
- rabbitmq
restart: unless-stopped
user: root
networks:
internal:

# This is the entry point for a development server.
# Run with --no-deps to run attached to the services
Expand All @@ -82,31 +87,39 @@ services:
- "62202:8080"
# for ssh
- "62203:22"
networks:
internal:

rabbitmq:
image: rabbitmq:3.7-alpine
hostname: rabbitmq
volumes:
- rabbitmq:/var/lib/rabbitmq
restart: unless-stopped
networks:
internal:

beat:
<<: *uwsgi-common
container_name: qgis-plugins-beat
working_dir: /home/web/django_project
entrypoint: [ ]
command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO
networks:
internal:

worker:
<<: *uwsgi-common
container_name: qgis-plugins-worker
links:
depends_on:
- db
- rabbitmq
- beat
working_dir: /home/web/django_project
entrypoint: []
command: celery -A plugins worker -l INFO
networks:
internal:

web:
# Note you cannot scale if you use container_name
Expand All @@ -124,9 +137,9 @@ services:
- ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro
- ./webroot:/var/www/webroot
- ./certbot-etc:/etc/letsencrypt
links:
- uwsgi:uwsgi
- metabase:metabase
depends_on:
- uwsgi
- metabase
logging:
driver: "json-file"
options:
Expand All @@ -135,14 +148,16 @@ services:
restart: unless-stopped
command:
- ${QGISPLUGINS_ENV}
networks:
internal:

dbbackups:
image: kartoza/pg-backup:16-3.4
hostname: pg-backups
volumes:
- ${QGISPLUGINS_BACKUP_VOLUME}:/backups
links:
- db:db
depends_on:
- db
environment:
# take care to let the project name below match that
# declared in the top of the makefile
Expand All @@ -153,16 +168,20 @@ services:
- POSTGRES_HOST=${DATABASE_HOST:-db}
- PGDATABASE=${DATABASE_NAME:-gis}
restart: unless-stopped
networks:
internal:

metabase:
image: metabase/metabase:latest
environment:
- MB_DB_TYPE=postgres
- MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker}
links:
depends_on:
- db
expose:
- "3000"
networks:
internal:

certbot:
image: certbot/certbot
Expand All @@ -172,4 +191,9 @@ services:
- ./certbot-etc:/etc/letsencrypt
depends_on:
- web
command: certonly --webroot --webroot-path=/var/www/webroot --email [email protected] --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org
command: certonly --webroot --webroot-path=/var/www/webroot --email [email protected] --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org
networks:
internal:

networks:
internal:
4 changes: 2 additions & 2 deletions dockerize/docker/uwsgi.conf
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ module = wsgi
master = true
pidfile=/tmp/django.pid
socket = 0.0.0.0:8080
workers = 4
workers = 8
cheaper = 2
env = DJANGO_SETTINGS_MODULE=settings_docker
# disabled so we run in the foreground for docker
Expand All @@ -18,5 +18,5 @@ logger = file:/var/log/uwsgi-errors.log
#uid = 1000
#gid = 1000
memory-report = true
harakiri = 50
harakiri = 100
listen = 127
99 changes: 0 additions & 99 deletions dockerize/nginx.conf

This file was deleted.

20 changes: 13 additions & 7 deletions dockerize/sites-enabled/prod-ssl.conf
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ server {
listen 80;
# the domain name it will serve for
server_name plugins.qgis.org;

# Redirect all HTTP traffic to HTTPS
return 301 https://$server_name$request_uri;

Expand Down Expand Up @@ -58,19 +58,17 @@ server {
# 'REQUEST_URI': '/phpmyadmin/scripts/setup.php',
# See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/
# for more details.
#proxy_set_header Host $http_host;
proxy_set_header Host $host;
autoindex on;
# your Django project's static files - amend as required
alias /home/web/archive;
expires 21d; # cache for 6h
expires 21d; # cache for 21 days
}
# Finally, send all non-media requests to the Django server.
location / {
uwsgi_pass uwsgi;
# the uwsgi_params file you installed needs to be passed with each
# request.
# the uwsgi_params need to be passed with each uwsgi request
uwsgi_param QUERY_STRING $query_string;
uwsgi_param REQUEST_METHOD $request_method;
uwsgi_param CONTENT_TYPE $content_type;
Expand All @@ -86,6 +84,10 @@ server {
uwsgi_param REMOTE_PORT $remote_port;
uwsgi_param SERVER_PORT $server_port;
uwsgi_param SERVER_NAME $server_name;

if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) {
return 403;
}
}

location /metabase/ {
Expand Down Expand Up @@ -174,19 +176,17 @@ server {
# 'REQUEST_URI': '/phpmyadmin/scripts/setup.php',
# See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/
# for more details.
#proxy_set_header Host $http_host;
proxy_set_header Host $host;
autoindex on;
# your Django project's static files - amend as required
alias /home/web/archive;
expires 21d; # cache for 6h
expires 21d; # cache for 21 days
}
# Finally, send all non-media requests to the Django server.
location / {
uwsgi_pass uwsgi;
# the uwsgi_params file you installed needs to be passed with each
# request.
# the uwsgi_params need to be passed with each uwsgi request
uwsgi_param QUERY_STRING $query_string;
uwsgi_param REQUEST_METHOD $request_method;
uwsgi_param CONTENT_TYPE $content_type;
Expand All @@ -202,8 +202,14 @@ server {
uwsgi_param REMOTE_PORT $remote_port;
uwsgi_param SERVER_PORT $server_port;
uwsgi_param SERVER_NAME $server_name;

if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) {
return 403;
}

}


location /metabase/ {
# set to webroot path
proxy_pass http://metabase:3000/;
Expand Down
Loading

0 comments on commit 47ddc04

Please sign in to comment.