diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 3b7c89db..225f9784 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -23,6 +23,8 @@ services: - postgres_data:/opt/postgres/data - ${QGISPLUGINS_BACKUP_VOLUME}:/backups restart: unless-stopped + networks: + internal: uwsgi: &uwsgi-common build: @@ -57,11 +59,14 @@ services: - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw - celerybeat-schedule:/home/web/celerybeat-schedule:rw - links: - - db:db - - rabbitmq:rabbitmq + + depends_on: + - db + - rabbitmq restart: unless-stopped user: root + networks: + internal: # This is the entry point for a development server. # Run with --no-deps to run attached to the services @@ -82,6 +87,8 @@ services: - "62202:8080" # for ssh - "62203:22" + networks: + internal: rabbitmq: image: rabbitmq:3.7-alpine @@ -89,6 +96,8 @@ services: volumes: - rabbitmq:/var/lib/rabbitmq restart: unless-stopped + networks: + internal: beat: <<: *uwsgi-common @@ -96,17 +105,21 @@ services: working_dir: /home/web/django_project entrypoint: [ ] command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + networks: + internal: worker: <<: *uwsgi-common container_name: qgis-plugins-worker - links: + depends_on: - db - rabbitmq - beat working_dir: /home/web/django_project entrypoint: [] command: celery -A plugins worker -l INFO + networks: + internal: web: # Note you cannot scale if you use container_name @@ -124,9 +137,9 @@ services: - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro - ./webroot:/var/www/webroot - ./certbot-etc:/etc/letsencrypt - links: - - uwsgi:uwsgi - - metabase:metabase + depends_on: + - uwsgi + - metabase logging: driver: "json-file" options: @@ -135,14 +148,16 @@ services: restart: unless-stopped command: - ${QGISPLUGINS_ENV} + networks: + internal: dbbackups: image: kartoza/pg-backup:16-3.4 hostname: pg-backups volumes: - ${QGISPLUGINS_BACKUP_VOLUME}:/backups - links: - - db:db + depends_on: + - db environment: # take care to let the project name below match that # declared in the top of the makefile @@ -153,16 +168,20 @@ services: - POSTGRES_HOST=${DATABASE_HOST:-db} - PGDATABASE=${DATABASE_NAME:-gis} restart: unless-stopped + networks: + internal: metabase: image: metabase/metabase:latest environment: - MB_DB_TYPE=postgres - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} - links: + depends_on: - db expose: - "3000" + networks: + internal: certbot: image: certbot/certbot @@ -172,4 +191,9 @@ services: - ./certbot-etc:/etc/letsencrypt depends_on: - web - command: certonly --webroot --webroot-path=/var/www/webroot --email admin@qgis.org --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org \ No newline at end of file + command: certonly --webroot --webroot-path=/var/www/webroot --email admin@qgis.org --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org + networks: + internal: + +networks: + internal: \ No newline at end of file diff --git a/dockerize/docker/uwsgi.conf b/dockerize/docker/uwsgi.conf index c27283a1..7efc0305 100644 --- a/dockerize/docker/uwsgi.conf +++ b/dockerize/docker/uwsgi.conf @@ -7,7 +7,7 @@ module = wsgi master = true pidfile=/tmp/django.pid socket = 0.0.0.0:8080 -workers = 4 +workers = 8 cheaper = 2 env = DJANGO_SETTINGS_MODULE=settings_docker # disabled so we run in the foreground for docker @@ -18,5 +18,5 @@ logger = file:/var/log/uwsgi-errors.log #uid = 1000 #gid = 1000 memory-report = true -harakiri = 50 +harakiri = 100 listen = 127 diff --git a/dockerize/nginx.conf b/dockerize/nginx.conf deleted file mode 100644 index 4a728e85..00000000 --- a/dockerize/nginx.conf +++ /dev/null @@ -1,99 +0,0 @@ -# Define connection details for connecting to django running in -# a docker container. -upstream uwsgi { - server uwsgi:8080; -} -server { - # OTF gzip compression - gzip on; - gzip_min_length 860; - gzip_comp_level 5; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; - gzip_disable “MSIE [1-6].(?!.*SV1)”; - - access_log /var/log/nginx/access.log; - - client_max_body_size 20M; - error_log /var/log/nginx/error.log; - - # the port your site will be served on - listen 80; - # the domain name it will serve for - server_name ""; - - if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCB -ot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCo -llector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|Go -zaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ N -inja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan| -Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGe -nSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pyc -url|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|s -itecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|t -urnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II| -WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { - return 403; - } - charset utf-8; - - # Drop any non django related requests - # Its probably someone nefarious probing for vulnerabilities... - location ~ (\.php|\.asp|myadmin) { - return 404; - } - - # Django media - location /media { - # your Django project's media files - amend as required - alias /home/web/media; - expires 21d; # cache for 71 days - } - location /static { - # your Django project's static files - amend as required - alias /home/web/static; - expires 21d; # cache for 21 days - } - location /plugins/plugins.xml { - if ($request_uri !~ "&package_name(.*)") { - rewrite ^/plugins/plugins.xml /web/static/cached_xmls/plugins_$arg_qgis.xml break; - root /home; - expires 600s; - } - } - location /archive { - # Changed from http_host to host because of error messages when - # bots hit urls like this: - # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', - # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ - # for more details. - #proxy_set_header Host $http_host; - proxy_set_header Host $host; - autoindex on; - # your Django project's static files - amend as required - alias /home/web/archive; - expires 21d; # cache for 6h - } - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi; - # the uwsgi_params file you installed needs to be passed with each - # request. - # the uwsgi_params need to be passed with each uwsgi request - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - } -} diff --git a/dockerize/sites-enabled/prod-ssl.conf b/dockerize/sites-enabled/prod-ssl.conf index 313b9618..589715f1 100644 --- a/dockerize/sites-enabled/prod-ssl.conf +++ b/dockerize/sites-enabled/prod-ssl.conf @@ -22,7 +22,7 @@ server { listen 80; # the domain name it will serve for server_name plugins.qgis.org; - + # Redirect all HTTP traffic to HTTPS return 301 https://$server_name$request_uri; @@ -58,19 +58,17 @@ server { # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ # for more details. - #proxy_set_header Host $http_host; proxy_set_header Host $host; autoindex on; # your Django project's static files - amend as required alias /home/web/archive; - expires 21d; # cache for 6h + expires 21d; # cache for 21 days } # Finally, send all non-media requests to the Django server. location / { uwsgi_pass uwsgi; # the uwsgi_params file you installed needs to be passed with each # request. - # the uwsgi_params need to be passed with each uwsgi request uwsgi_param QUERY_STRING $query_string; uwsgi_param REQUEST_METHOD $request_method; uwsgi_param CONTENT_TYPE $content_type; @@ -86,6 +84,10 @@ server { uwsgi_param REMOTE_PORT $remote_port; uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; + + if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { + return 403; + } } location /metabase/ { @@ -174,19 +176,17 @@ server { # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ # for more details. - #proxy_set_header Host $http_host; proxy_set_header Host $host; autoindex on; # your Django project's static files - amend as required alias /home/web/archive; - expires 21d; # cache for 6h + expires 21d; # cache for 21 days } # Finally, send all non-media requests to the Django server. location / { uwsgi_pass uwsgi; # the uwsgi_params file you installed needs to be passed with each # request. - # the uwsgi_params need to be passed with each uwsgi request uwsgi_param QUERY_STRING $query_string; uwsgi_param REQUEST_METHOD $request_method; uwsgi_param CONTENT_TYPE $content_type; @@ -202,8 +202,14 @@ server { uwsgi_param REMOTE_PORT $remote_port; uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; + + if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { + return 403; + } + } + location /metabase/ { # set to webroot path proxy_pass http://metabase:3000/; diff --git a/dockerize/sites-enabled/prod.conf b/dockerize/sites-enabled/prod.conf index 84f9d8aa..086d6903 100644 --- a/dockerize/sites-enabled/prod.conf +++ b/dockerize/sites-enabled/prod.conf @@ -55,19 +55,17 @@ server { # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ # for more details. - #proxy_set_header Host $http_host; proxy_set_header Host $host; autoindex on; # your Django project's static files - amend as required alias /home/web/archive; - expires 21d; # cache for 6h + expires 21d; # cache for 21 days } # Finally, send all non-media requests to the Django server. location / { uwsgi_pass uwsgi; # the uwsgi_params file you installed needs to be passed with each # request. - # the uwsgi_params need to be passed with each uwsgi request uwsgi_param QUERY_STRING $query_string; uwsgi_param REQUEST_METHOD $request_method; uwsgi_param CONTENT_TYPE $content_type; @@ -83,8 +81,14 @@ server { uwsgi_param REMOTE_PORT $remote_port; uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; + + if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|HMView|HomePageBot|HTTP\:\:Lite|HubSpot|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Screaming|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|URI\:\:Fetch|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) { + return 403; + } + } + location /metabase/ { # set to webroot path proxy_pass http://metabase:3000/;