Merge pull request #1453 from rodekruis/chore.upgrade-ubuntu

Chore.upgrade ubuntu

README.md

@@ -63,11 +63,11 @@ From root run
 ### Using Docker
 
 ```
-docker-compose -f docker-compose.yml up -d # for production
+docker compose -f docker-compose.yml up -d # for production
 
-docker-compose up -d # for development (NOTE: this does not start the ibf-dashboard)
+docker compose up -d # for development (NOTE: this does not start the ibf-dashboard)
 
-docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d # for development (same as previous line)
+docker compose -f docker-compose.yml -f docker-compose.override.yml up -d # for development (same as previous line)
 
 ```
 
@@ -89,7 +89,7 @@ For IBF-dashboard
 
 Suggestion: load everything through Docker, except IBF-dashboard. This has the benefit that changes in front-end code are immediately reflected, instead of having to rebuild.
 
-- `docker-compose up -d`
+- `docker compose up -d`
 - `cd interfaces/IBF-dashboard`
 - `npm start`
 
@@ -100,7 +100,7 @@ are connected to a database-server). For setting up a fully working version of t
 
 1. Seed database with initial static data
 
-- `docker-compose exec ibf-api-service npm run seed`
+- `docker compose exec ibf-api-service npm run seed`
 
 2. Load initial raster data
 

docker-compose.override.yml

@@ -5,9 +5,6 @@ services:
     command: ['npm', 'run', 'start:dev']
     environment:
       - NODE_ENV=development
-    volumes:
-      - 'api_service_node_modules:/home/ibf/api-service/node_modules'
-      - ./services/API-service:/home/ibf/api-service
     ports:
       - 3000:3000
     depends_on:
@@ -22,14 +19,6 @@ services:
   ibf-geoserver:
     ports:
       - 8081:8080
-    volumes:
-      - ./services/API-service/geoserver-volume/raster-files:/opt/geoserver/data_dir/workspaces/ibf-system/ibf-pipeline
-      - ./services/API-service/geoserver-volume/geoserver-layers:/opt/geoserver/data_dir/workspaces/ibf-system
-
-  nginx:
-    volumes:
-      - ./nginx/conf.d:/etc/nginx/user.conf.d
-      - ./nginx/letsencrypt:/etc/letsencrypt
 
   ibf-local-db:
     image: postgis/postgis

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   nginx:
     image: staticfloat/nginx-certbot
@@ -13,7 +11,7 @@ services:
     volumes:
       - ./nginx/conf.d:/etc/nginx/user.conf.d
       - ibf-dashboard-root:/var/www/ibf-dashboard
-      - letsencrypt:/etc/letsencrypt
+      - ./nginx/letsencrypt:/etc/letsencrypt
     networks:
       web-server-network:
     restart: unless-stopped
@@ -49,7 +47,8 @@ services:
       - TWILIO_WHATSAPP_NUMBER=${TWILIO_WHATSAPP_NUMBER}
       - NG_API_URL=${NG_API_URL}
     volumes:
-      - api-root:/home/ibf/api-service
+      - 'api_service_node_modules:/home/ibf/api-service/node_modules'
+      - ./services/API-service:/home/ibf/api-service
     networks:
       web-server-network:
       api-network:
@@ -94,8 +93,8 @@ services:
       - GEOSERVER_ADMIN_PASSWORD=${GEOSERVER_ADMIN_PASSWORD}
       - JAVA_OPTS="-DALLOW_ENV_PARAMETRIZATION=true"
     volumes:
-      - api-geoserver-files:/opt/geoserver/data_dir/workspaces/ibf-system/ibf-pipeline
-      - api-geoserver-workspace:/opt/geoserver/data_dir/workspaces/ibf-system
+      - ./services/API-service/geoserver-volume/raster-files:/opt/geoserver/data_dir/workspaces/ibf-system/ibf-pipeline
+      - ./services/API-service/geoserver-volume/geoserver-layers:/opt/geoserver/data_dir/workspaces/ibf-system
       - ./.env:/opt/geoserver/data_dir/geoserver-environment.properties
     networks:
       web-server-network:
@@ -104,26 +103,7 @@ services:
 
 volumes:
   ibf-dashboard-root:
-  api-root:
-    driver_opts:
-      type: bind
-      device: ./services/API-service
-      o: bind
-  api-geoserver-files:
-    driver_opts:
-      type: bind
-      device: ./services/API-service/geoserver-volume/raster-files
-      o: bind
-  api-geoserver-workspace:
-    driver_opts:
-      type: bind
-      device: ./services/API-service/geoserver-volume/geoserver-layers
-      o: bind
-  letsencrypt:
-    driver_opts:
-      type: bind
-      device: ./nginx/letsencrypt
-      o: bind
+  api_service_node_modules:
 
 networks:
   web-server-network:

docs/TROUBLESHOOT.md

@@ -22,13 +22,13 @@ Sometimes there is an issue with installing all packages for ibf-api-service. Yo
 - exit (switch to normal user)
 - cd /home/ibf-user/IBF-system/services/API-service
 - npm install
-- docker-compose restart ibf-api-service
+- docker compose restart ibf-api-service
 
 ### 502 Bad Gateway on API-calls/Swagger UI
 
-Sometimes for unknown reasons the API & Swagger are unavailable, while 'docker-compose logs ibf-api-service' tells you that the API is running correctly.
+Sometimes for unknown reasons the API & Swagger are unavailable, while 'docker compose logs ibf-api-service' tells you that the API is running correctly.
 
-- On the server run 'docker-compose restart nginx' in this case
+- On the server run 'docker compose restart nginx' in this case
 - which will usually solve it.
 
 ### Access to files on server
@@ -50,7 +50,7 @@ To check if everything starts up correctly:
 
 - All containers mentioned in the docker-compose file /IBF-system/docker-compose.yml should be up and running. Check e.g. via `docker container ls`
 - Application should load and respond properly on http://localhost:4200
-- Check the logs of `ibf-api-service` via `docker-compose logs -f ibf-api-service`. Is it saying that the Nest application started successfully?
+- Check the logs of `ibf-api-service` via `docker compose logs -f ibf-api-service`. Is it saying that the Nest application started successfully?
 - Can you visit the Swagger UI API documentation on http://localhost:3000/docs and try one of the GET endpoints?
 
 ### Problem with existing database
@@ -59,14 +59,14 @@ If there is a problem with your existing database schema, you can always try to
 
 - Access the database via e.g. DBeaver
 - Alternatively, access it via commandline using
-  - `docker-compose exec ibf-local-db bash`
+  - `docker compose exec ibf-local-db bash`
   - `psql -U <chosen local-db-username>`
 - Throw away schema via `drop schema "IBF-app" cascade;`
 - Recreate schema: `create schema "IBF-app";`
 - Exit the database again
-- Restart the ibf-api-service: `docker-compose restart ibf-api-service` or `docker-compose up -d ibf-api-service`
+- Restart the ibf-api-service: `docker compose restart ibf-api-service` or `docker compose up -d ibf-api-service`
 - This will automatically run migration-scripts upon restart, which involves an "initial migration script" which creates all necessary tables.
-- Wait until done. Check via `docker-compose logs -f ibf-api-service`
+- Wait until done. Check via `docker compose logs -f ibf-api-service`
 - As always (see main [README](../README.md))
   - Run the seed-script
   - Run mock-endpoint (or external pipeline)
@@ -76,30 +76,30 @@ If there is a problem with your existing database schema, you can always try to
 These below commands come in handy when you face such issue:
 
 - The Docker compose command starts and runs your entire app:
-  docker-compose up
+  docker compose up
 
 - For dev environment we can use:
-  docker-compose -f docker-compose.yml -f docker-compose.override.yml up
+  docker compose -f docker-compose.yml -f docker-compose.override.yml up
 
 - Execute this will create a container named ibf-api-service and start a Bash session:
-  docker-compose exec ibf-api-service bash
+  docker compose exec ibf-api-service bash
 
 - To run seed script:
-  docker-compose exec ibf-api-service npm run seed
+  docker compose exec ibf-api-service npm run seed
 
 - Detached mode (-d) run command in the background:
-  docker-compose up -d ibf-api-service:
+  docker compose up -d ibf-api-service:
 
 - Create the Docker image from the Dockerfile in this folder through:
   docker build -t ibf-api-service
 
 - To check the logs of IBF-api-service:
-  docker-compose logs -f ibf-api-service
+  docker compose logs -f ibf-api-service
 
 - To restart the IBF-api-service in docker:
-  docker-compose restart ibf-api-service
+  docker compose restart ibf-api-service
 
 - To install the IBF-api-services:
-  docker-compose exec ibf-api-service npm install
+  docker compose exec ibf-api-service npm install
 
 You can refer this official docker document [here](https://docs.docker.com/engine/reference/commandline/compose_exec/)

docs/VM_SETUP.md

@@ -7,105 +7,79 @@
 3. Open ports 80 (http), 443 (https) and 3099 (webhook)
 4. Set the DNS Name Label on the IP Address attached to the VM
 
-### In GitHub
-
-1. [Create web-hook](https://github.com/rodekruis/IBF-system/settings/hooks) to
-   listen on `http://ip-address:3099/`
-2. Set secret for web-hook access (for [this]() step)
-
 ### In VM
 
 1. User Management
 
-    1. Create user group - `sudo groupadd ibf-users`
-    2. Add `ibf-user` to group - `sudo usermod -a -G ibf-users ibf-user`
-    3. Add users to group - `sudo usermod -a -G ibf-users [email protected]`
-        1. Command to verify group members - `grep ibf-users /etc/group`
-    4. Change access of shared directory - `/home/ibf-user`
-        1. `chgrp -Rf ibf-users /home/ibf-user`
-        2. `sudo chown -R ibf-user:ibf-users /home/ibf-user`
-        3. `sudo chmod -R 775 /home/ibf-user`
-        4. Re-login to verify if you have access by running
-           `touch /home/ibf-user`
-    5. Add the following lines to `/etc/sudoers`
+   1. Create user group - `sudo groupadd ibf-users`
+   2. Create user `ibf-user` - `sudo adduser ibf-user` (with password also `ibf-user`)
+   3. Add `ibf-user` to group - `sudo usermod -a -G ibf-users ibf-user`
+   4. Add users to group - `sudo usermod -a -G ibf-users <username>`
+      1. Command to verify group members - `grep ibf-users /etc/group`
+   5. Change access of shared directory - `/home/ibf-user`
+      1. `chgrp -Rf ibf-users /home/ibf-user`
+      2. `sudo chown -R ibf-user:ibf-users /home/ibf-user`
+      3. `sudo chmod -R 775 /home/ibf-user`
+      4. Re-login to verify if you have access by running
+         `touch /home/ibf-user`
+   6. Open `/etc/sudoers` with `sudo nano /etc/sudoers` and add these lines
 
-    ```jsx
-    # Allow members of group ibf-users to execute systemctl daemon-reload
-    %ibf-users ALL=NOPASSWD: /bin/systemctl daemon-reload
+   ```jsx
+   # Allow members of group ibf-users to execute systemctl daemon-reload
+   %ibf-users ALL=NOPASSWD: /bin/systemctl daemon-reload
 
-    # Allow members of group ibf-users to execute service webhook restart
-    %ibf-users ALL=NOPASSWD: /usr/sbin/service webhook restart
-    ```
+   # Allow members of group ibf-users to execute service webhook restart
+   %ibf-users ALL=NOPASSWD: /usr/sbin/service webhook restart
+   ```
 
 2. Install Software
-    1. NodeJS
-       [Source](https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions)
-        1. `curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash -`
-        2. `sudo apt-get install -y nodejs`
-        3. Verification - `node -v`
-    2. Docker [Source](https://docs.docker.com/engine/install/ubuntu/)
-        1. `sudo apt-get remove docker docker-engine docker.io containerd runc`
-        2. `sudo apt-get update`
-        3. `sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common`
-        4. `curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -`
-        5. `sudo apt-key fingerprint 0EBFCD88`
-        6. `sudo add-apt-repository \ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) \ stable"`
-        7. `sudo apt-get update`
-        8. `sudo apt-get install docker-ce docker-ce-cli containerd.io`
-        9. Allow users to access docker commands
-            1. `sudo usermod -aG docker [email protected]`
-            2. Verification - `grep docker /etc/group`
-        10. Verification - `docker -v`
-    3. Docker Compose
-       [Source](https://docs.docker.com/compose/install/#install-compose-on-linux-systems)
-        1. `sudo curl -L "[https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$](https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$)(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose`
-        2. `sudo chmod +x /usr/local/bin/docker-compose`
-        3. Verification - `docker-compose -v`
-    4. Webhook Related
-        1. `sudo apt install postgresql-client-common postgresql-client`
+   1. NodeJS
+      [Source](https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions)
+      1. Follow instructions in Source for Node 16
+      2. Verification - `node -v`
+   2. Docker [Source](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository)
+      1. Follow instructions in Source all th
+      2. Verification - `docker -v`
+      3. Allow users to access docker commands
+         1. `sudo usermod -aG docker <username>`
+         2. Verification - `grep docker /etc/group`
 3. Setup IBF-system
-    1. Setup GIT
-        1. `git clone https://github.com/rodekruis/IBF-system.git`
-        2. Set the repo config to allow group access -
-           `git config core.sharedRepository group`
-           [Source](https://stackoverflow.com/a/6448326/1753041)
-        3. `sudo chgrp -R ibf-users /home/ibf-user/IBF-system`
-           [Source](https://stackoverflow.com/a/6448326/1753041)
-        4. `sudo chmod -R g+rwX /home/ibf-user/IBF-system`
-           [Source](https://stackoverflow.com/a/6448326/1753041)
-    2. Setup Environment Variables
-        1. [OLD - PIPELINE DECOUPLED NOW]: Create `services/IBF-pipeline/pipeline/secrets.py`
-            1. `cp /home/ibf-user/IBF-system/services/IBF-pipeline/pipeline/secrets.py.template /home/ibf-user/IBF-system/services/IBF-pipeline/pipeline/secrets.py`
-            2. Set the appropriate values in the
-               `/home/ibf-user/IBF-system/services/IBF-pipeline/pipeline/secrets.py`
-               file
-        2. Create `/home/ibf-user/IBF-system/.env`
-            1. `cp /home/ibf-user/IBF-system/example.env /home/ibf-user/IBF-system/.env`
-            2. Set the appropriate values in the `.env` file
-            3. Load the `.env` vars by `source /home/ibf-user/IBF-system/.env`
-            4. Test if the vars were loaded correctly `echo $NODE_ENV`
-    3. Setup web-hook
-        1. `sudo cp webhook.service /etc/systemd/system/`
-        2. Set `GITHUB_WEBHOOK_SECRET` value in
-           `/etc/systemd/system/webhook.service`
-        3. Verification - `ls /etc/systemd/system/`
-        4. In `/home/ibf-user/IBF-system` - `npm install github-webhook-handler`
-        5. `sudo service webhook start`
-        6. Verification - `sudo service webhook status`
-    4. `. tools/deploy.sh`
-4. Set up geoserver
-    1. Download
-       [raster-files.zip](https://rodekruis.sharepoint.com/sites/510-CRAVK-510/_layouts/15/guestaccess.aspx?folderid=0fa454e6dc0024dbdba7a178655bdc216&authkey=AcqhM85JHZY8cc6H7BTKgO0&expiration=2021-08-27T22%3A00%3A00.000Z&e=MnocDf)
-    2. Unzip the files using `apt install unzip` and `unzip raster-files.zip`, into  `services/API-service/geoserver-volume/raster-files/`
-    3. Check if the [Geoserver](http://ibf.510.global/geoserver/web) contains
-       necessary layers.
-5. Verify that external pipeline works correctly
-    1. There should be an external pipeline able to upload impact forecast data to this VM. Please check this together with the pipeline owner or the applicable disaster-types.
-        - Pipeline runs without error
-        - Dashboard shows correct data (including disaster-extent raster)
-        - Email is received if applicable
+   1. `cd /home/ibf-user`
+   2. `git clone https://github.com/rodekruis/IBF-system.git`
+   3. `cd /home/ibf-user/IBF-system`
+   4. Set the repo config to allow group access -
+      `git config core.sharedRepository group`
+      [Source](https://stackoverflow.com/a/6448326/1753041)
+   5. `sudo chgrp -R ibf-users /home/ibf-user/IBF-system`
+      [Source](https://stackoverflow.com/a/6448326/1753041)
+   6. `sudo chmod -R g+rwX /home/ibf-user/IBF-system`
+      [Source](https://stackoverflow.com/a/6448326/1753041)
+   7. Setup Environment Variables
+      1. Create `/home/ibf-user/IBF-system/.env`
+         1. `cp /home/ibf-user/IBF-system/example.env /home/ibf-user/IBF-system/.env`
+         2. Set the appropriate values in the `.env` file
+         3. Load the `.env` vars by `source /home/ibf-user/IBF-system/.env`
+         4. Test if the vars were loaded correctly `echo $NODE_ENV`
+   8. Load certificate: load `DigiCertGlobalRootCA.crt.pem` in `services/API-service/cert` for connection to Azure Postgres server (if applicable)
+   9. `. tools/deploy.sh`
+4. Load base data
 
-### Useful Commands / Tools
+   1. Load Geoserver source data
+      1. Download
+         [raster-files.zip](https://rodekruis.sharepoint.com/sites/510-CRAVK-510/_layouts/15/guestaccess.aspx?folderid=0fa454e6dc0024dbdba7a178655bdc216&authkey=AcqhM85JHZY8cc6H7BTKgO0&expiration=2021-08-27T22%3A00%3A00.000Z&e=MnocDf)
+      2. Unzip the files using `apt install unzip` and `unzip raster-files.zip`, into `services/API-service/geoserver-volume/raster-files/`
+   2. Seed database: `docker compose exec ibf-api-service npm run seed`
+   3. Run all mock scenarios via Swagger: `api/scripts/mock-all`
 
-1. Delete user from group? `sudo gpasswd -d [email protected] ibf-users`
-2. [Secret Generator](https://passwordsgenerator.net/)
+5. Setup web-hook
+   1. On Github
+      1. [Create web-hook](https://github.com/rodekruis/IBF-system/settings/hooks) to
+         listen on `http://ip-address:3099/`
+      2. Set secret for web-hook access
+   2. On VM:
+      1. `sudo cp tools/webhook.service /etc/systemd/system/`
+      2. Set `GITHUB_WEBHOOK_SECRET` value in `/etc/systemd/system/webhook.service` as same value set in Github Webhooks
+      3. Verification - `ls /etc/systemd/system/`
+      4. In `/home/ibf-user/IBF-system` - `npm install github-webhook-handler`
+      5. `sudo service webhook start`
+      6. Verification - `sudo service webhook status`

package.json

@@ -3,7 +3,7 @@
   "version": "0.258.0",
   "private": true,
   "scripts": {
-    "start:services": "docker-compose up -d",
+    "start:services": "docker compose up -d",
     "start:interface": "npm start --prefix interfaces/IBF-dashboard/ -- --port 4200",
     "install:interface": "npm install --no-optional --no-audit --no-fund --prefix interfaces/IBF-dashboard/",
     "open": "npx npm-run-all open:ibf-api-service open:ibf-dashboard",
@@ -33,4 +33,4 @@
     "puppeteer": "^8.0.0"
   },
   "pre-commit": "lint"
-}
+}