Upgrades For Docker And Systemd

README.md

@@ -35,3 +35,26 @@ For further help on how to use `create_instance`, run `./manage.py create_instan
 To run on Windows you must install [Cygwin](https://www.cygwin.com) (including `rsync` and `openssh`) and start a Unix shell like `bash` to run the install scripts.
 
 On Windows the OTM source code is shared with the virtual machine via a one-way `rsync` from Windows host to Ubuntu guest. When you update source code files you must run `vagrant rsync` from your Windows Unix shell. See also [vagrant rsync-auto](http://docs.vagrantup.com/v2/cli/rsync-auto.html).
+
+
+## Running with Docker
+
+Everything can be run using the `docker-compose.yml`, but with a few changes.
+
+### restore a database backup
+
+Download a database dump from the S3 backups. Then, use `psql` to import the database dump into a the running postgres container.
+
+Example:
+
+```
+psql -h localhost -U otm -d otm <~/20210308.dump
+```
+
+There are a few changes that need to be made:
+* The volumes for nginx need to be updated. Both nginx and otm-core need to share a local volume for the static and media directories
+* The otm-core and otm-tiler hostname in the nginx configs should be updated to localhost
+* Running this requires localstack to be setup
+* Each repository under otm-vagrant (otm-core, otm-ecoservice, otm-tiler) needs to have the Dockerfile built with otm-[name]:1.0
+* Create a symlink for otm-core called otm_core
+* Create a symlink for otm-tiler called otm_tiler

configs/etc/nginx/conf.d/default.conf

@@ -0,0 +1 @@
+# blank default.conf

configs/etc/nginx/includes/upstreams/app

@@ -1,3 +1,3 @@
 upstream app {
-  server localhost:12000;
+  server otm-core:12000;
 }

configs/etc/nginx/includes/upstreams/tiler

@@ -1,3 +1,3 @@
 upstream tile {
-  server localhost:4000;
+  server otm-tiler:4000;
 }

configs/etc/nginx/nginx.conf

@@ -0,0 +1,95 @@
+user www-data;
+worker_processes 4;
+pid /run/nginx.pid;
+
+events {
+	worker_connections 768;
+	# multi_accept on;
+}
+
+http {
+
+	##
+	# Basic Settings
+	##
+
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+	# server_tokens off;
+
+	# server_names_hash_bucket_size 64;
+	# server_name_in_redirect off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# Logging Settings
+	##
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	##
+	# Gzip Settings
+	##
+
+	gzip on;
+	gzip_disable "msie6";
+
+	# gzip_vary on;
+	# gzip_proxied any;
+	# gzip_comp_level 6;
+	# gzip_buffers 16 8k;
+	# gzip_http_version 1.1;
+	# gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+	##
+	# nginx-naxsi config
+	##
+	# Uncomment it if you installed nginx-naxsi
+	##
+
+	#include /etc/nginx/naxsi_core.rules;
+
+	##
+	# nginx-passenger config
+	##
+	# Uncomment it if you installed nginx-passenger
+	##
+
+	#passenger_root /usr;
+	#passenger_ruby /usr/bin/ruby;
+
+	##
+	# Virtual Host Configs
+	##
+
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+}
+
+
+#mail {
+#	# See sample authentication script at:
+#	# http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
+# 
+#	# auth_http localhost/auth.php;
+#	# pop3_capabilities "TOP" "USER";
+#	# imap_capabilities "IMAP4rev1" "UIDPLUS";
+# 
+#	server {
+#		listen     localhost:110;
+#		protocol   pop3;
+#		proxy      on;
+#	}
+# 
+#	server {
+#		listen     localhost:143;
+#		protocol   imap;
+#		proxy      on;
+#	}
+#}

configs/etc/nginx/sites-available/otm.conf

@@ -1,9 +1,10 @@
 include includes/upstreams/*;
 
 server {
-  listen 80 default_server;
+  listen 8080 default_server;
 
-  client_max_body_size 20M;
+  #client_max_body_size 20M;
+  client_max_body_size 1M;
 
   include includes/locations/*;
 }

configs/etc/nginx/sites-enabled/otm

@@ -0,0 +1,9 @@
+include includes/upstreams/*;
+
+server {
+  listen 8080 default_server;
+
+  client_max_body_size 20M;
+
+  include includes/locations/*;
+}

docker-compose.yml

@@ -0,0 +1,112 @@
+version: '3'
+services:
+    nginx:
+        image: nginx:latest
+        restart: always
+        volumes:
+            - ./configs/etc/nginx/sites-available:/etc/nginx/sites-available
+            - ./configs/etc/nginx/sites-enabled:/etc/nginx/sites-enabled
+            - ./configs/etc/nginx/nginx.conf:/etc/nginx/nginx.conf
+            - ./configs/etc/nginx/conf.d/default.conf:/etc/nginx/conf.d/default.conf
+            - ./configs/etc/nginx/includes:/etc/nginx/includes
+            - ${HOME}/tools/docker/volumes/otm/static:/usr/local/otm/static
+            - ${HOME}/tools/docker/volumes/otm/media:/usr/local/otm/media
+        ports:
+            - '8080:8080'
+
+    database:
+        image: kartoza/postgis:9.6-2.4
+        environment:
+            - POSTGRES_USER=otm
+            - POSTGRES_PASS=otm
+            - POSTGRES_DBNAME=otm
+            - ALLOW_IP_RANGE=0.0.0.0/0
+        volumes:
+            - pg_data:/var/lib/postgresql
+        ports:
+            - '5432:5432'
+
+    redis:
+        image: redis:latest
+        ports:
+            - '6379:6379'
+
+    localstack:
+        image: localstack/localstack
+        environment:
+            - SERVICES=s3
+            - DEBUG=1
+            - DATA_DIR=/usr/data
+            - PORT_WEB_UI=8081
+            - AWS_ACCESS_KEY_ID=local_test
+            - AWS_SECRET_ACCESS_KEY=local_test
+            - AWS_DEFAULT_REGION=us-east-1
+        ports:
+            - '4563-4599:4563-4599'
+            - '8081:8081'
+        volumes:
+            - ${HOME}/tools/localstack/data:/usr/data
+
+    otm-core:
+       image: otm-core:1.0
+       # command: gunicorn -w 2 -b "0.0.0.0:12000" opentreemap.wsgi:application
+       command: python manage.py runserver 0.0.0.0:12000
+       #command: tail -f /dev/null
+       restart: always
+       depends_on:
+           - database
+       environment:
+           - ESRI_CLIENT_ID=
+           - ESRI_CLIENT_SECRET=
+           - GOOGLE_MAPS_KEY=
+       working_dir: /usr/local/otm/app/opentreemap
+       ports:
+           - '12000:12000'
+           # - '3000:3000'
+       volumes:
+           - ${HOME}/tools/docker/volumes/otm/static:/usr/local/otm/app/static
+           - ${HOME}/tools/docker/volumes/otm/media:/usr/local/otm/app/media
+
+#   otm-celery:
+#       image: otm-core:1.0
+#       #command: celery -A "opentreemap.celery.app" -b "redis://redis:6379" worker -l debug 
+#       command: tail -f /dev/null
+#       working_dir: /usr/local/otm/app/opentreemap
+#       volumes:
+#           - ./otm_core/opentreemap:/usr/local/otm/app/opentreemap
+#           - ./docker/volumes/static:/usr/local/otm/static
+#           - ./docker/volumes/media:/usr/local/otm/media
+
+    otm-tiler:
+        image: otm-tiler:1.0
+        command: node --inspect=0.0.0.0:9229 server.js
+        #command: tail -f /dev/null
+        working_dir: /usr/local/tiler
+        environment:
+            - OTM_CACHE_HOST=redis
+            - OTM_DB_HOST=database
+        ports:
+            - '4000:4000'
+            - '9229:9229'
+        volumes:
+            - ./otm_tiler/:/usr/local/tiler
+
+    otm-ecoservice:
+        image: otm-ecoservice:1.0
+        command: ./ecoservice/ecobenefits
+        #command: tail -f /dev/null
+        working_dir: /usr/local/ecoservice
+        restart: always
+        depends_on:
+            - database
+        environment:
+            - OTM_ECO_HOST=0.0.0.0
+            - OTM_DB_HOST=database
+        ports:
+            - '13000:13000'
+
+volumes:
+    pg_data:
+        driver: local
+    node_modules:
+        driver: local

nginx/configs/etc/init/celery.conf

@@ -0,0 +1,11 @@
+description     "celery"
+
+start on (vagrant-mounted)
+stop on shutdown
+
+setuid otm
+
+respawn
+chdir /usr/local/otm/app/opentreemap
+
+exec celery -A "opentreemap.celery.app" -b "redis://localhost:6379" worker -l debug

nginx/configs/etc/init/ecoservice.conf

@@ -0,0 +1,13 @@
+description "Eco benefit service"
+
+start on vagrant-mounted
+stop on shutdown
+
+respawn
+respawn limit 50 5
+
+kill timeout 5
+
+chdir /usr/local/ecoservice/ecoservice
+
+exec ./ecobenefits 2>&1 1>/var/log/ecoservice

nginx/configs/etc/init/otm-unicorn.conf

@@ -0,0 +1,17 @@
+description "Gunicorn Upstart Script for otm"
+
+start on vagrant-mounted
+stop on shutdown
+
+respawn
+respawn limit 50 5
+
+kill timeout 5
+
+env ESRI_CLIENT_ID=
+env ESRI_CLIENT_SECRET=
+env GOOGLE_MAPS_KEY=
+
+chdir /usr/local/otm/app/opentreemap
+
+exec gunicorn -w 2 -b "0.0.0.0:12000" opentreemap.wsgi:application

nginx/configs/etc/init/tiler.conf

@@ -0,0 +1,12 @@
+description "OTM2 Map Tile Server"
+
+start on vagrant-mounted
+stop on shutdown
+
+respawn
+respawn limit 50 5
+
+chdir /usr/local/tiler
+
+env HOME="/home/otm"
+exec node server.js

nginx/configs/etc/nginx/includes/locations/app

@@ -0,0 +1,17 @@
+location /static/ {
+  alias /usr/local/otm/static/;
+}
+
+location /media/ {
+alias /usr/local/otm/media/;
+}
+
+location / {
+  proxy_set_header Host $http_host;
+  proxy_redirect off;
+
+  if (!-f $request_filename) {
+    proxy_pass http://app;
+    break;
+  }
+}

nginx/configs/etc/nginx/includes/locations/tiler

@@ -0,0 +1,5 @@
+location /tile/ {
+    proxy_redirect off;
+
+    proxy_pass http://tile/;
+}

nginx/configs/etc/nginx/includes/upstreams/app

@@ -0,0 +1,3 @@
+upstream app {
+  server otm-core:12000;
+}

nginx/configs/etc/nginx/includes/upstreams/tiler

@@ -0,0 +1,3 @@
+upstream tile {
+  server otm-tiler:4000;
+}

nginx/configs/etc/nginx/sites-available/otm.conf

@@ -0,0 +1,9 @@
+include includes/upstreams/*;
+
+server {
+  listen 8080 default_server;
+
+  client_max_body_size 20M;
+
+  include includes/locations/*;
+}

nginx/configs/usr/local/otm/app/opentreemap/opentreemap/settings/local_settings.py

@@ -0,0 +1,21 @@
+EXTRA_UNMANAGED_APPS = ('django_extensions',)
+
+STATIC_ROOT = '/usr/local/otm/static'
+MEDIA_ROOT = '/usr/local/otm/media'
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.contrib.gis.db.backends.postgis',
+        'NAME': 'otm',
+        'USER': 'otm',
+        'PASSWORD': 'otm',
+        'HOST': 'localhost',
+        'PORT': '5432'
+    }
+}
+
+CELERY_BROKER_URL = 'redis://localhost:6379/'
+CELERY_RESULT_BACKEND = 'redis://localhost:6379/'
+
+EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
+EMAIL_FILE_PATH = '/usr/local/otm/emails'