feat: mapeo instace and tileserver (#36)

* chore: upping version to 0.4.4 * fix: crawler script modified to run only once * feat: tileserver-gl with starting script and tile download automation * chore: rename volumes, should solve #35 * chore: demo tileserver building customized * fix: add old initialization * fix: change tmp dir for tileserver * fix: try using user 1000 for tileserver * chore: add shared mbtiles volume with syncthing and filebrowser * feat: use own tileserver-gl base image * chore: simplify tileserver dockerfile * fix: add .warc.gz to pywb sources as well * feat: add tiles cloud folder to syncthing default * fix: correct syntax for balena.yml applicationEnvironmentVariables * chore: add mapeo-bridge and mbtiles volumes across services and minor fixes * chore: add crowding translation badge * feat: add documentation for working with balena virtual-machines * chore: add issue template for github * chore: clenaup earthdefendertoolkit collection from pywb, to only use all * chore: remove unused portal service which now draws from docker hub image * fix: tileserver wait for tile sync instead of downloading * fix: working tileserver start script for checking files every 30secs * fix: correct naming of terrastories-db * chore: use nightly portal and code cleanup
digidem · Mar 25, 2023 · fac02c4 · fac02c4
1 parent b655a48
commit fac02c4
Show file tree

Hide file tree

Showing 21 changed files with 313 additions and 128 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create an issue to inform of a bug
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Device Type:**
+Raspberry Pi 4 or Nuc pc
+
+**Additional context**
+Add any other context about the problem here.
+
+**Possible Solution**
+Can you suggest any possible solution to this issue?
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/README.md b/README.md
@@ -6,6 +6,8 @@
 
 ![balena-tag-release](https://github.com/digidem/edt-offline/actions/workflows/balena-tag-release.yml/badge.svg)
 
+[![Crowdin](https://badges.crowdin.net/edt-offline-portal/localized.svg)](https://crowdin.com/project/edt-offline-portal)
+
 **EDT Offline** is a multi-device stack, which can be easily installed on single-board-computers, Nucs, or any recycled marchine. It creates an access point with the [Earth Defender's Toolkit](http://earthdefenderstoolkit.com/) platform running, even without the Internet. It provides applications, documentation, use cases, curated websites, and application bridges.
 
 The software stack and content are automatically updated while online, and there are different strategies to update them while offline.

diff --git a/balena.yml b/balena.yml
@@ -1,6 +1,6 @@
 name: "Earth Defender's Toolkit Offline"
 type: "sw.application"
-version: 0.4.3
+version: 0.4.4
 description: "A low-energy device that creates an access-point serving applications, bridging them and exposing curated content offline.
   Software stack and content are automatically updated while online, and can be updated offline."
 assets:
@@ -17,44 +17,45 @@ data:
     - BALENA_HOST_CONFIG_gpu_mem: 64
   applicationEnvironmentVariables:
     # BALENA INTERFACE
-    SET_HOSTNAME: "edt"
-    UDEV: on # Enables ability to auto mount USB drives in to the container
-    PORTAL_PORT: 8080
+    - SET_HOSTNAME: "edt"
+    - UDEV: on # Enables ability to auto mount USB drives in to the container
+    - PORTAL_PORT: 8080
     # WIFI
-    PWC_HOTSPOT_SSID: "Earth_Defenders_Toolkit" # Name as it appears in list of WiFi networks
-    PWC_PORT: 9090
+    - PWC_HOTSPOT_SSID: "Earth_Defenders_Toolkit" # Name as it appears in list of WiFi networks
+    - PWC_PORT: 9090
     # PWC_HOTSPOT_PASSWORD: "balena01" # Optional. Must be 8 characters or more
     # PWC_INTERFACE: "wlan0"
     # MAPEO BRIDGE
-    MAPEO_PROJECT_KEY: af43cbdb6827d3ddc5b39b20df174bf5c3e44d9bbcdccfe034af090440cebf69
-    MAPEO_STORAGE_PATH: /usr/src/output
-    MAPEO_TERRASTORIES_TYPE: terrastories
+    - MAPEO_PROJECT_KEY: af43cbdb6827d3ddc5b39b20df174bf5c3e44d9bbcdccfe034af090440cebf69
+    - MAPEO_STORAGE_PATH: /usr/src/output
+    - MAPEO_TERRASTORIES_TYPE: terrastories
     # Minio & Filebrowser
-    ADMIN_LOGIN: admin
-    ADMIN_PASSWORD: admin123
-    LOCALE: "en"
+    - ADMIN_LOGIN: admin
+    - ADMIN_PASSWORD: admin123
+    - LOCALE: "en"
     # Syncthing & Filebrowser
-    PUID: 1000
-    PGID: 1000
-    TZ: America/Brasil
-    SYNC_SOURCE_ID: 7MAA2AW-FBC5CFE-F32BALW-LKLZEBE-VSPN6XG-4PYESBH-NAZ42Y5-RQYULAL
-    SYNC_SOURCE_NAME: EDT-Cloud
-    SYNC_CRAWLS_ID: 6qegg-xuksh
-    SYNC_CRAWLS_NAME: Offline-Websites
-    SYNC_FDROID_ID: zwozu-ajxlu
-    SYNC_FDROID_NAME: FDROID-edt
-    SYNC_MAPEO_ID: atzye-79v6o
-    SYNC_MAPEO_NAME: Mapeo
-    SYNC_INSTALLERS_ID: gmyij-7gktt
-    SYNC_INSTALLERS_NAME: INSTALLERS-edt
+    - PUID: 1000
+    - PGID: 1000
+    - TZ: America/Brasil
+    - SYNC_SOURCE_ID: 7MAA2AW-FBC5CFE-F32BALW-LKLZEBE-VSPN6XG-4PYESBH-NAZ42Y5-RQYULAL
+    - SYNC_SOURCE_NAME: EDT-Cloud
+    - SYNC_CRAWLS_ID: 6qegg-xuksh
+    - SYNC_CRAWLS_NAME: Offline-Websites
+    - SYNC_FDROID_ID: zwozu-ajxlu
+    - SYNC_FDROID_NAME: FDROID-edt
+    - SYNC_MAPEO_ID: atzye-79v6o
+    - SYNC_MAPEO_NAME: Mapeo
+    - SYNC_INSTALLERS_ID: gmyij-7gktt
+    - SYNC_INSTALLERS_NAME: INSTALLERS-edt
+    - SYNC_TILES_ID: kfgre-slfl7
+    - SYNC_TILES_NAME: Map-Tiles
     # Terrastories
-    DB_USER_PASSWORD: postgres
-    RAILS_ENV: offline
-    RAILS_MASTER_KEY: '7a0c538b904d5919af7c11470f3b7a4b'
-    HOST_HOSTNAME: edt.local
-    CHOKIDAR_USEPOLLING: 'true'
-    USE_LOCAL_MAP_SERVER: 'true'
-    OFFLINE_MAP_STYLE: /tileserver/styles/terrastories-map/style.json
+    - DB_USER_PASSWORD: postgres
+    - RAILS_ENV: offline
+    - HOST_HOSTNAME: edt.local
+    - CHOKIDAR_USEPOLLING: 'true'
+    - USE_LOCAL_MAP_SERVER: 'true'
+    - OFFLINE_MAP_STYLE: /tileserver/styles/terrastories-map/style.json
   defaultDeviceType: "raspberrypi4-64"
   supportedDeviceTypes:
     - "raspberrypi3-64"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,7 +2,7 @@ version: "2.1"
 
 volumes:
   fdroid:
-  portal:
+  edt-offline-portal:
   mapeo:
   crawls:
   pywb-archive:
@@ -15,8 +15,8 @@ volumes:
   terrastories-data:
   terrastories-media:
   terrastories-import:
-  bdi_db:
-  pwc_db:
+  interface_db:
+  wifi_db:
 services:
   # https://github.com/digidem/starter-interface
   balena-interface:
@@ -28,11 +28,12 @@ services:
       - "80:80"
       - "8079:8079"
     volumes:
-      - "bdi_db:/app/db" # Stores UI database files
+      - interface_db:/app/db # Stores UI database files
       - crawls:/app/storage/websites
       - mapeo:/app/storage/mapeo
       - fdroid:/app/storage/fdroid
       - installers:/app/storage/installers
+      - mbtiles:/app/storage//mbtiles
       - terrastories-import:/app/storage/terrastories/import
       - terrastories-media:/app/storage/terrastories/media
     privileged: true # This can be removed if not using the USB mounting feature
@@ -48,26 +49,26 @@ services:
     network_mode: "host"
     restart: on-failure
     volumes:
-      - "pwc_db:/app/db" # Optional if not setting the hotspot ssid and password via the API
+      - wifi_db:/app/db # Optional if not setting the hotspot ssid and password via the API
     labels:
       io.balena.features.dbus: "1"
     cap_add:
       - NET_ADMIN
     privileged: true # This can be removed if you do not need the LED connectivity indicator.
-  # https://hub.docker.com/r/communityfirst/edt-offline-portal
+  # https://hub.docker.com/r/communityfirst/edt-offline-edt-portal
   portal:
-    image: communityfirst/edt-offline-portal:nightly-full
+    image: communityfirst/edt-offline-portal:nightly
     restart: always
     volumes:
-      - 'portal:/usr/src/nuxt-app'
+      - edt-offline-portal:/usr/src/nuxt-app
     ports:
       - 8080:3000
   # https://hub.docker.com/r/communityfirst/mapeo-bridge
   mapeo-bridge:
     image: communityfirst/mapeo-bridge
     network_mode: host
     volumes:
-      - "mapeo-bridge:/usr/src/output"
+      - mapeo-bridge:/usr/src/output
     restart: unless-stopped
   # https://hub.docker.com/r/filebrowser/filebrowser
   filebrowser:
@@ -78,6 +79,8 @@ services:
       - mapeo:/srv/mapeo
       - fdroid:/srv/fdroid
       - installers:/srv/installers
+      - mbtiles:/srv/mbtiles
+      - mapeo-bridge:/srv/terrastories-places
       - terrastories-import:/srv/terrastories/import
       - terrastories-media:/srv/terrastories/media
     ports:
@@ -91,34 +94,46 @@ services:
       - mapeo:/config/mapeo
       - fdroid:/config/fdroid
       - installers:/config/installers
+      - mbtiles:/config/mbtiles
     ports:
       - 8082:8384
       - 22000:22000/tcp
       - 22000:22000/udp
       - 21027:21027/udp
     restart: unless-stopped
   # https://hub.docker.com/_/postgres
-  db:
+  terrastories-db:
     image: postgres:11
     ports:
       - 5432:5432
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: terrastories
     volumes:
       - "postgres_data:/var/lib/postgresql/data"
   # https://hub.docker.com/r/terrastories/terrastories
   terrastories:
-    image: terrastories/terrastories
+    build: services/terrastories
     ports:
       - 8083:3000
     depends_on:
-      - db
+      - terrastories-db
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@db:5432/terrastories
+      RAILS_ENV: offline
+      CHOKIDAR_USEPOLLING: 'true'
+      USE_LOCAL_MAP_SERVER: 'true'
+      OFFLINE_MAP_STYLE: /tileserver/styles/terrastories-map/style.json
+      HOST_HOSTNAME: 0.0.0.0
     volumes:
       - bundler:/usr/local/bundle
       - terrastories-data:/api
       - terrastories-media:/media
       - terrastories-import:/api/import/media
   # https://hub.docker.com/r/maptiler/tileserver-gl
-  tile-server:
-    image: maptiler/tileserver-gl
+  tileserver:
+    build: services/tileserver
     ports:
       - 8085:8080
     volumes:

diff --git a/docker/demo/.env.example b/docker/demo/.env.example
@@ -19,6 +19,8 @@ SYNC_MAPEO_ID=atzye-79v6o
 SYNC_MAPEO_NAME=Mapeo
 SYNC_INSTALLERS_ID=gmyij-7gktt
 SYNC_INSTALLERS_NAME=INSTALLERS-edt
+SYNC_TILES_ID=kfgre-slfl7
+SYNC_TILES_NAME=Map-Tiles
 # Terrastories ENV
 MAPBOX_STYLE=mapbox://styles/mapbox/light-v10
 MAPBOX_ACCESS_TOKEN=your pk token here

diff --git a/docker/demo/docker-compose.yml b/docker/demo/docker-compose.yml
@@ -41,8 +41,10 @@ services:
     volumes:
       - crawls:/srv/websites
       - mapeo:/srv/mapeo
+      - mbtiles:/srv/mbtiles
       - fdroid:/srv/fdroid
       - installers:/srv/installers
+      - mapeo-bridge:/srv/terrastories-places
       - terrastories-import:/srv/terrastories/import
       - terrastories-media:/srv/terrastories/media
     environment:
@@ -72,11 +74,14 @@ services:
       SYNC_MAPEO_NAME: ${SYNC_MAPEO_NAME}
       SYNC_INSTALLERS_ID: ${SYNC_INSTALLERS_ID}
       SYNC_INSTALLERS_NAME: ${SYNC_INSTALLERS_NAME}
+      SYNC_TILES_ID: ${SYNC_TILES_ID}
+      SYNC_TILES_NAME: ${SYNC_TILES_NAME}
     volumes:
       - crawls:/config/websites
       - mapeo:/config/mapeo
       - fdroid:/config/fdroid
       - installers:/config/installers
+      - mbtiles:/config/mbtiles
     ports:
       - 8082:8384
       - 22000:22000/tcp
@@ -92,14 +97,17 @@ services:
       - crawls:/source
     restart: unless-stopped
   # Terrastories
-  tile-server:
-    image: maptiler/tileserver-gl
+  tileserver:
+    build: ../../services/tileserver
     ports:
       - 8085:8080
     volumes:
       - mbtiles:/data
+    environment:
+      PUID: 1000
+      PGID: 1000
     restart: unless-stopped
-  db:
+  terrastories-db:
     image: postgres:11
     ports:
       - 5432:5432
@@ -115,7 +123,7 @@ services:
     ports:
       - 8083:3000
     depends_on:
-      - db
+      - terrastories-db
     environment:
       DATABASE_URL: postgres://postgres:postgres@db:5432/terrastories
       RAILS_ENV: offline

diff --git a/docker/local/.env.example b/docker/local/.env.example
@@ -19,7 +19,8 @@ SYNC_MAPEO_ID=atzye-79v6o
 SYNC_MAPEO_NAME=Mapeo
 SYNC_INSTALLERS_ID=gmyij-7gktt
 SYNC_INSTALLERS_NAME=INSTALLERS-edt
+SYNC_TILES_ID=kfgre-slfl7
+SYNC_TILES_NAME=Map-Tiles
 # Terrastories ENV
 MAPBOX_STYLE=mapbox://styles/mapbox/light-v10
 MAPBOX_ACCESS_TOKEN=your pk token here
-DB_USER_PASSWORD=postgres
diff --git a/docker/local/docker-compose.yml b/docker/local/docker-compose.yml
@@ -71,6 +71,7 @@ services:
       - mapeo:/srv/mapeo
       - fdroid:/srv/fdroid
       - installers:/srv/installers
+      - mapeo-bridge:/srv/terrastories-places
       - terrastories-import:/srv/terrastories/import
       - terrastories-media:/srv/terrastories/media
     environment:
@@ -101,6 +102,8 @@ services:
       SYNC_MAPEO_NAME: ${SYNC_MAPEO_NAME}
       SYNC_INSTALLERS_ID: ${SYNC_INSTALLERS_ID}
       SYNC_INSTALLERS_NAME: ${SYNC_INSTALLERS_NAME}
+      SYNC_TILES_ID: ${SYNC_TILES_ID}
+      SYNC_TILES_NAME: ${SYNC_TILES_NAME}
     volumes:
       - crawls:/config/websites
       - mapeo:/config/mapeo
@@ -120,14 +123,15 @@ services:
     volumes:
       - pywb-archive:/webarchive
       - crawls:/source
-  # Terrastories
-  tile-server:
-    image: maptiler/tileserver-gl
+  # https://hub.docker.com/r/maptiler/tileserver-gl
+  tileserver:
+    build: ../../services/tileserver
     ports:
-      - 8085:80
+      - 8085:8080
     volumes:
       - mbtiles:/data
-  db:
+  # Terrastories
+  terrastories-db:
     image: postgres:11
     ports:
       - 5432:5432
@@ -141,7 +145,7 @@ services:
     ports:
       - 8083:3000
     depends_on:
-      - db
+      - terrastories-db
     environment:
       DB_USER_PASSWORD: ${DB_USER_PASSWORD}
       DB_HOST: db