Merge pull request #216 from ch-routerank/pelias-initial-deployment

Geocoder: Pelias initial deployment

.reuse/dep5

@@ -3,7 +3,7 @@ Upstream-Name: odh-mentor-otp
 Upstream-Contact: NOI Techpark <[email protected]>
 Source: https://github.com/noi-techpark/odh-mentor-otp
 
-Files: .github/* infrastructure .pre-commit-config.yaml .gitignore .dockerignore AUTHOR.txt *.json dotenv.example osm.url *.jar */.nvmrc *.geojson docs/images/* gbfs/bikestations_meran.csv */dotenv.example gtfs2bbox/tests/* journey/yarn.lock journey/src/fonts/* journey/src/images/* vms/README vms/.gitignore journey/src/otp-ui/icons/openmove/README vms/signs/* journey/src/otp-ui/icons/.eslintrc journey/src/otp-ui/icons/*.svg otp-react-redux-config/config.yml
+Files: .github/* infrastructure .pre-commit-config.yaml .gitignore .gitmodules .dockerignore AUTHOR.txt *.json dotenv.example osm.url *.jar */.nvmrc *.geojson docs/images/* gbfs/bikestations_meran.csv */dotenv.example gtfs2bbox/tests/* journey/yarn.lock journey/src/fonts/* journey/src/images/* vms/README vms/.gitignore journey/src/otp-ui/icons/openmove/README vms/signs/* journey/src/otp-ui/icons/.eslintrc journey/src/otp-ui/icons/*.svg otp-react-redux-config/config.yml
 Copyright: (c) NOI Techpark <[email protected]>
 License: CC0-1.0
 
@@ -19,3 +19,7 @@ License: BSD-2-Clause
 Files: journey/src/styles/cookieconsent/cookieconsent.css journey/src/util/cookieconsent.js
 Copyright: (c) 2020-2023 Orest Bida
 License: MIT
+
+Files: web-demo/.gitignore web-demo/.babelrc
+Copyright: (c) 2017 Conveyal
+License: MIT

pelias/.gitignore

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+./data

pelias/README.md

@@ -0,0 +1,66 @@
+<!--
+SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+
+SPDX-License-Identifier: MIT
+-->
+
+# Italian area
+
+This project is configured to download/prepare/build a complete Pelias installation for South-Tyrol.
+
+# Setup
+
+Please refer to the instructions at <https://github.com/pelias/docker> in order to install and configure your docker environment.
+
+The minimum configuration required in order to run this project are [installing prerequisites](https://github.com/pelias/docker#prerequisites), [install the pelias command](https://github.com/pelias/docker#installing-the-pelias-command) and [configure the environment](https://github.com/pelias/docker#configure-environment)
+
+You also need `curl`, `jq` and `node` to import OpenTripPlanner stops and NOI Datahub Activities and Accomodations POI.
+
+Please ensure that's all working fine before continuing.
+
+# Run a Build
+
+To run a complete build, execute the following commands:
+
+## Pull relevant images and create elasticsearch shard (see pelias.json configuration)
+```bash
+pelias compose pull
+pelias elastic start
+pelias elastic wait
+pelias elastic create
+```
+
+## Download all the relevant information
+
+```bash
+pelias download all
+pelias prepare all
+./importers/download_and_prepare_stops.sh
+./importers/download_and_prepare_poi.sh
+```
+
+## Import all the data in the service
+```bash
+pelias import all
+```
+
+## Frequently update stops and POI
+```bash
+./importers/download_and_prepare_stops.sh
+./importers/download_and_prepare_poi.sh
+./importers/delete_old_poi_and_stops.sh
+pelias import csv
+```
+
+# Starting the service
+The API service can then be started with the following command
+
+```bash
+pelias compose up -d
+```
+
+# Make an Example Query
+
+You can now make queries against your new Pelias build:
+
+<http://localhost:4000/v1/search?text=Morena>

pelias/docker-compose.yml

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+version: '3'
+networks:
+  default:
+    driver: bridge
+services:
+  libpostal:
+    image: pelias/libpostal-service
+    container_name: pelias_libpostal
+    user: "${DOCKER_USER}"
+    restart: always
+    ports: [ "127.0.0.1:4400:4400" ]
+  schema:
+    image: pelias/schema:master
+    container_name: pelias_schema
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+  api:
+    image: pelias/api:master
+    container_name: pelias_api
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4000" ]
+    ports: [ "0.0.0.0:4000:4000" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+  placeholder:
+    image: pelias/placeholder:master
+    container_name: pelias_placeholder
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4100" ]
+    ports: [ "127.0.0.1:4100:4100" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  whosonfirst:
+    image: pelias/whosonfirst:master
+    container_name: pelias_whosonfirst
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  openstreetmap:
+    image: pelias/openstreetmap:master
+    container_name: pelias_openstreetmap
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  openaddresses:
+    image: pelias/openaddresses:master
+    container_name: pelias_openaddresses
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  csv-importer:
+    image: pelias/csv-importer:master
+    container_name: pelias_csv_importer
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+      - "./blacklist/:/data/blacklist"
+  polylines:
+    image: pelias/polylines:master
+    container_name: pelias_polylines
+    user: "${DOCKER_USER}"
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  interpolation:
+    image: pelias/interpolation:master
+    container_name: pelias_interpolation
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4300" ]
+    ports: [ "127.0.0.1:4300:4300" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  pip:
+    image: pelias/pip-service:master
+    container_name: pelias_pip-service
+    user: "${DOCKER_USER}"
+    restart: always
+    environment: [ "PORT=4200" ]
+    ports: [ "127.0.0.1:4200:4200" ]
+    volumes:
+      - "./pelias.json:/code/pelias.json"
+      - "${DATA_DIR}:/data"
+  elasticsearch:
+    image: pelias/elasticsearch:7.16.1
+    container_name: pelias_elasticsearch
+    user: "${DOCKER_USER}"
+    restart: always
+    ports: [ "127.0.0.1:9200:9200", "127.0.0.1:9300:9300" ]
+    volumes:
+      - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro"
+      - "${DATA_DIR}/elasticsearch:/usr/share/elasticsearch/data"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add: [ "IPC_LOCK" ]
+    security_opt:
+      - seccomp=unconfined

pelias/elasticsearch.yml

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+network.host: 0.0.0.0
+bootstrap.memory_lock: true
+indices.breaker.fielddata.limit: 85%
+indices.fielddata.cache.size: 75%

pelias/importers/delete_old_poi_and_stops.sh

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+function delete_old_stops {
+  curl -X POST "localhost:9200/pelias/_delete_by_query?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "query": {
+      "match": {
+        "source": "otp"
+      }
+    }
+  }
+  '
+}
+function delete_old_poi {
+  curl -X POST "localhost:9200/pelias/_delete_by_query?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "query": {
+      "match": {
+        "source": "noi-datahub-poi"
+      }
+    }
+  }
+  '
+}
+function delete_old_accomodation {
+  curl -X POST "localhost:9200/pelias/_delete_by_query?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "query": {
+      "match": {
+        "source": "noi-datahub-accomodation"
+      }
+    }
+  }
+  '
+}
+
+delete_old_stops
+delete_old_poi
+delete_old_accomodation

pelias/importers/download_and_prepare_poi.sh

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+curl "https://tourism.api.opendatahub.com/v1/ODHActivityPoi?pagesize=50000" > ./data/csv-importer/touristic-poi-all.json
+curl "https://tourism.api.opendatahub.com/v1/STA/ODHActivityPoi?language=en&referer=SuedtirolMobilWeb&fields=Id%2CDetail.en.Title%2CContactInfos.en.City&pagesize=20000" > ./data/csv-importer/touristic-poi-filtered-set.json
+
+curl "https://tourism.api.opendatahub.com/v1/Accommodation?pagesize=20000" > ./data/csv-importer/accomodation-poi-all.json 
+curl "https://tourism.api.opendatahub.com/v1/STA/Accommodation?language=en&referer=SuedtirolMobilWeb&fields=Id%2CAccoDetail.en.Name%2CAccoDetail.en.City&pagesize=10000" > ./data/csv-importer/accomodation-poi-filtered-set.json
+
+node ./importers/process-touristic-poi.js
+
+TOURISTIC_CSV=./data/csv-importer/touristic-poi.csv
+TOURISTIC_JSON_FILE=./data/csv-importer/touristic-poi.json
+ACCOMODATION_CSV=./data/csv-importer/accomodation-poi.csv
+ACCOMODATION_JSON_FILE=./data/csv-importer/accomodation-poi.json
+
+# transform JSON data to CSV with jq
+echo "source,layer,id,lat,lon,name,name_de,name_en,name_fr,name_it,category_json,addendum_json_poi" > $TOURISTIC_CSV
+cat $TOURISTIC_JSON_FILE | jq --raw-output ".[] | [
+    \"noi-datahub-poi\",\"venue\",\
+    .Id,\
+    .GpsPoints.position.Latitude,.GpsPoints.position.Longitude,\
+    .Detail.it.Title,.Detail.de.Title,.Detail.en.Title,.Detail.fr.Title,.Detail.it.Title,\
+    ([.Tags[] | \"touristic:\"+ (.Id | gsub(\" \"; \"_\"))] | tostring), \
+    (.|tostring) \
+    ] | @csv" >> $TOURISTIC_CSV;
+
+echo "source,layer,id,lat,lon,name,name_de,name_en,name_fr,name_it,category_json,addendum_json_accomodation" > $ACCOMODATION_CSV
+cat $ACCOMODATION_JSON_FILE | jq --raw-output ".[] | [
+    \"noi-datahub-accomodation\",\"venue\",\
+    .Id,\
+    .Latitude,.Longitude,\
+    .AccoDetail.it.Name,.AccoDetail.de.Name,.AccoDetail.en.Name,.AccoDetail.fr.Name,.AccoDetail.it.Name,\
+    ([(.AccoType,.AccoCategory) | \"accomodation:\"+ (.Id | gsub(\" \"; \"_\"))] | tostring), \
+    (.|tostring) \
+    ] | @csv" >> $ACCOMODATION_CSV;

pelias/importers/download_and_prepare_stops.sh

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2024 routeRANK <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+STOP_JSON_FILE=./data/csv-importer/stops.json
+STOP_CSV=./data/csv-importer/stops.csv
+
+
+# fetch JSON data from OpenTripPlanner 
+node ./importers/fetch-poi.js
+
+# transform JSON data to CSV with jq
+echo "source,layer,id,name,lat,lon,popularity,categories,addendum_json_stop" > $STOP_CSV
+cat $STOP_JSON_FILE | jq --raw-output '.[] | ["otp","stops",.gtfsId,.name,.lat,.lon,.popularity,(.categories|tostring),(.|tostring)] | @csv' >> $STOP_CSV