diff --git a/README.md b/README.md index 5deebde..6e608da 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,26 @@ For further help on how to use `create_instance`, run `./manage.py create_instan To run on Windows you must install [Cygwin](https://www.cygwin.com) (including `rsync` and `openssh`) and start a Unix shell like `bash` to run the install scripts. On Windows the OTM source code is shared with the virtual machine via a one-way `rsync` from Windows host to Ubuntu guest. When you update source code files you must run `vagrant rsync` from your Windows Unix shell. See also [vagrant rsync-auto](http://docs.vagrantup.com/v2/cli/rsync-auto.html). + + +## Running with Docker + +Everything can be run using the `docker-compose.yml`, but with a few changes. + +### restore a database backup + +Download a database dump from the S3 backups. Then, use `psql` to import the database dump into a the running postgres container. + +Example: + +``` +psql -h localhost -U otm -d otm <~/20210308.dump +``` + +There are a few changes that need to be made: +* The volumes for nginx need to be updated. Both nginx and otm-core need to share a local volume for the static and media directories +* The otm-core and otm-tiler hostname in the nginx configs should be updated to localhost +* Running this requires localstack to be setup +* Each repository under otm-vagrant (otm-core, otm-ecoservice, otm-tiler) needs to have the Dockerfile built with otm-[name]:1.0 +* Create a symlink for otm-core called otm_core +* Create a symlink for otm-tiler called otm_tiler diff --git a/configs/etc/nginx/conf.d/default.conf b/configs/etc/nginx/conf.d/default.conf new file mode 100644 index 0000000..15d6678 --- /dev/null +++ b/configs/etc/nginx/conf.d/default.conf @@ -0,0 +1 @@ +# blank default.conf diff --git a/configs/etc/nginx/includes/upstreams/app b/configs/etc/nginx/includes/upstreams/app index b718152..43ca87b 100644 --- a/configs/etc/nginx/includes/upstreams/app +++ b/configs/etc/nginx/includes/upstreams/app @@ -1,3 +1,3 @@ upstream app { - server localhost:12000; + server otm-core:12000; } diff --git a/configs/etc/nginx/includes/upstreams/tiler b/configs/etc/nginx/includes/upstreams/tiler index f2f6375..3cbc26e 100644 --- a/configs/etc/nginx/includes/upstreams/tiler +++ b/configs/etc/nginx/includes/upstreams/tiler @@ -1,3 +1,3 @@ upstream tile { - server localhost:4000; + server otm-tiler:4000; } diff --git a/configs/etc/nginx/nginx.conf b/configs/etc/nginx/nginx.conf new file mode 100644 index 0000000..52219b9 --- /dev/null +++ b/configs/etc/nginx/nginx.conf @@ -0,0 +1,95 @@ +user www-data; +worker_processes 4; +pid /run/nginx.pid; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + ## + # nginx-naxsi config + ## + # Uncomment it if you installed nginx-naxsi + ## + + #include /etc/nginx/naxsi_core.rules; + + ## + # nginx-passenger config + ## + # Uncomment it if you installed nginx-passenger + ## + + #passenger_root /usr; + #passenger_ruby /usr/bin/ruby; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} + + +#mail { +# # See sample authentication script at: +# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript +# +# # auth_http localhost/auth.php; +# # pop3_capabilities "TOP" "USER"; +# # imap_capabilities "IMAP4rev1" "UIDPLUS"; +# +# server { +# listen localhost:110; +# protocol pop3; +# proxy on; +# } +# +# server { +# listen localhost:143; +# protocol imap; +# proxy on; +# } +#} diff --git a/configs/etc/nginx/sites-available/otm.conf b/configs/etc/nginx/sites-available/otm.conf index 741a57b..8161791 100644 --- a/configs/etc/nginx/sites-available/otm.conf +++ b/configs/etc/nginx/sites-available/otm.conf @@ -1,9 +1,10 @@ include includes/upstreams/*; server { - listen 80 default_server; + listen 8080 default_server; - client_max_body_size 20M; + #client_max_body_size 20M; + client_max_body_size 1M; include includes/locations/*; } diff --git a/configs/etc/nginx/sites-enabled/otm b/configs/etc/nginx/sites-enabled/otm new file mode 100644 index 0000000..808975e --- /dev/null +++ b/configs/etc/nginx/sites-enabled/otm @@ -0,0 +1,9 @@ +include includes/upstreams/*; + +server { + listen 8080 default_server; + + client_max_body_size 20M; + + include includes/locations/*; +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c317024 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,112 @@ +version: '3' +services: + nginx: + image: nginx:latest + restart: always + volumes: + - ./configs/etc/nginx/sites-available:/etc/nginx/sites-available + - ./configs/etc/nginx/sites-enabled:/etc/nginx/sites-enabled + - ./configs/etc/nginx/nginx.conf:/etc/nginx/nginx.conf + - ./configs/etc/nginx/conf.d/default.conf:/etc/nginx/conf.d/default.conf + - ./configs/etc/nginx/includes:/etc/nginx/includes + - ${HOME}/tools/docker/volumes/otm/static:/usr/local/otm/static + - ${HOME}/tools/docker/volumes/otm/media:/usr/local/otm/media + ports: + - '8080:8080' + + database: + image: kartoza/postgis:9.6-2.4 + environment: + - POSTGRES_USER=otm + - POSTGRES_PASS=otm + - POSTGRES_DBNAME=otm + - ALLOW_IP_RANGE=0.0.0.0/0 + volumes: + - pg_data:/var/lib/postgresql + ports: + - '5432:5432' + + redis: + image: redis:latest + ports: + - '6379:6379' + + localstack: + image: localstack/localstack + environment: + - SERVICES=s3 + - DEBUG=1 + - DATA_DIR=/usr/data + - PORT_WEB_UI=8081 + - AWS_ACCESS_KEY_ID=local_test + - AWS_SECRET_ACCESS_KEY=local_test + - AWS_DEFAULT_REGION=us-east-1 + ports: + - '4563-4599:4563-4599' + - '8081:8081' + volumes: + - ${HOME}/tools/localstack/data:/usr/data + + otm-core: + image: otm-core:1.0 + # command: gunicorn -w 2 -b "0.0.0.0:12000" opentreemap.wsgi:application + command: python manage.py runserver 0.0.0.0:12000 + #command: tail -f /dev/null + restart: always + depends_on: + - database + environment: + - ESRI_CLIENT_ID= + - ESRI_CLIENT_SECRET= + - GOOGLE_MAPS_KEY= + working_dir: /usr/local/otm/app/opentreemap + ports: + - '12000:12000' + # - '3000:3000' + volumes: + - ${HOME}/tools/docker/volumes/otm/static:/usr/local/otm/app/static + - ${HOME}/tools/docker/volumes/otm/media:/usr/local/otm/app/media + +# otm-celery: +# image: otm-core:1.0 +# #command: celery -A "opentreemap.celery.app" -b "redis://redis:6379" worker -l debug +# command: tail -f /dev/null +# working_dir: /usr/local/otm/app/opentreemap +# volumes: +# - ./otm_core/opentreemap:/usr/local/otm/app/opentreemap +# - ./docker/volumes/static:/usr/local/otm/static +# - ./docker/volumes/media:/usr/local/otm/media + + otm-tiler: + image: otm-tiler:1.0 + command: node --inspect=0.0.0.0:9229 server.js + #command: tail -f /dev/null + working_dir: /usr/local/tiler + environment: + - OTM_CACHE_HOST=redis + - OTM_DB_HOST=database + ports: + - '4000:4000' + - '9229:9229' + volumes: + - ./otm_tiler/:/usr/local/tiler + + otm-ecoservice: + image: otm-ecoservice:1.0 + command: ./ecoservice/ecobenefits + #command: tail -f /dev/null + working_dir: /usr/local/ecoservice + restart: always + depends_on: + - database + environment: + - OTM_ECO_HOST=0.0.0.0 + - OTM_DB_HOST=database + ports: + - '13000:13000' + +volumes: + pg_data: + driver: local + node_modules: + driver: local diff --git a/nginx/configs/etc/init/celery.conf b/nginx/configs/etc/init/celery.conf new file mode 100644 index 0000000..dc6a76c --- /dev/null +++ b/nginx/configs/etc/init/celery.conf @@ -0,0 +1,11 @@ +description "celery" + +start on (vagrant-mounted) +stop on shutdown + +setuid otm + +respawn +chdir /usr/local/otm/app/opentreemap + +exec celery -A "opentreemap.celery.app" -b "redis://localhost:6379" worker -l debug diff --git a/nginx/configs/etc/init/ecoservice.conf b/nginx/configs/etc/init/ecoservice.conf new file mode 100644 index 0000000..f4726f4 --- /dev/null +++ b/nginx/configs/etc/init/ecoservice.conf @@ -0,0 +1,13 @@ +description "Eco benefit service" + +start on vagrant-mounted +stop on shutdown + +respawn +respawn limit 50 5 + +kill timeout 5 + +chdir /usr/local/ecoservice/ecoservice + +exec ./ecobenefits 2>&1 1>/var/log/ecoservice diff --git a/nginx/configs/etc/init/otm-unicorn.conf b/nginx/configs/etc/init/otm-unicorn.conf new file mode 100644 index 0000000..eb51a17 --- /dev/null +++ b/nginx/configs/etc/init/otm-unicorn.conf @@ -0,0 +1,17 @@ +description "Gunicorn Upstart Script for otm" + +start on vagrant-mounted +stop on shutdown + +respawn +respawn limit 50 5 + +kill timeout 5 + +env ESRI_CLIENT_ID= +env ESRI_CLIENT_SECRET= +env GOOGLE_MAPS_KEY= + +chdir /usr/local/otm/app/opentreemap + +exec gunicorn -w 2 -b "0.0.0.0:12000" opentreemap.wsgi:application diff --git a/nginx/configs/etc/init/tiler.conf b/nginx/configs/etc/init/tiler.conf new file mode 100644 index 0000000..f259261 --- /dev/null +++ b/nginx/configs/etc/init/tiler.conf @@ -0,0 +1,12 @@ +description "OTM2 Map Tile Server" + +start on vagrant-mounted +stop on shutdown + +respawn +respawn limit 50 5 + +chdir /usr/local/tiler + +env HOME="/home/otm" +exec node server.js diff --git a/nginx/configs/etc/nginx/includes/locations/app b/nginx/configs/etc/nginx/includes/locations/app new file mode 100644 index 0000000..dd20367 --- /dev/null +++ b/nginx/configs/etc/nginx/includes/locations/app @@ -0,0 +1,17 @@ +location /static/ { + alias /usr/local/otm/static/; +} + +location /media/ { +alias /usr/local/otm/media/; +} + +location / { + proxy_set_header Host $http_host; + proxy_redirect off; + + if (!-f $request_filename) { + proxy_pass http://app; + break; + } +} diff --git a/nginx/configs/etc/nginx/includes/locations/tiler b/nginx/configs/etc/nginx/includes/locations/tiler new file mode 100644 index 0000000..7c30cdb --- /dev/null +++ b/nginx/configs/etc/nginx/includes/locations/tiler @@ -0,0 +1,5 @@ +location /tile/ { + proxy_redirect off; + + proxy_pass http://tile/; +} diff --git a/nginx/configs/etc/nginx/includes/upstreams/app b/nginx/configs/etc/nginx/includes/upstreams/app new file mode 100644 index 0000000..43ca87b --- /dev/null +++ b/nginx/configs/etc/nginx/includes/upstreams/app @@ -0,0 +1,3 @@ +upstream app { + server otm-core:12000; +} diff --git a/nginx/configs/etc/nginx/includes/upstreams/tiler b/nginx/configs/etc/nginx/includes/upstreams/tiler new file mode 100644 index 0000000..3cbc26e --- /dev/null +++ b/nginx/configs/etc/nginx/includes/upstreams/tiler @@ -0,0 +1,3 @@ +upstream tile { + server otm-tiler:4000; +} diff --git a/nginx/configs/etc/nginx/sites-available/otm.conf b/nginx/configs/etc/nginx/sites-available/otm.conf new file mode 100644 index 0000000..808975e --- /dev/null +++ b/nginx/configs/etc/nginx/sites-available/otm.conf @@ -0,0 +1,9 @@ +include includes/upstreams/*; + +server { + listen 8080 default_server; + + client_max_body_size 20M; + + include includes/locations/*; +} diff --git a/nginx/configs/usr/local/otm/app/opentreemap/opentreemap/settings/local_settings.py b/nginx/configs/usr/local/otm/app/opentreemap/opentreemap/settings/local_settings.py new file mode 100644 index 0000000..8c45180 --- /dev/null +++ b/nginx/configs/usr/local/otm/app/opentreemap/opentreemap/settings/local_settings.py @@ -0,0 +1,21 @@ +EXTRA_UNMANAGED_APPS = ('django_extensions',) + +STATIC_ROOT = '/usr/local/otm/static' +MEDIA_ROOT = '/usr/local/otm/media' + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': 'otm', + 'USER': 'otm', + 'PASSWORD': 'otm', + 'HOST': 'localhost', + 'PORT': '5432' + } +} + +CELERY_BROKER_URL = 'redis://localhost:6379/' +CELERY_RESULT_BACKEND = 'redis://localhost:6379/' + +EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend' +EMAIL_FILE_PATH = '/usr/local/otm/emails'