Merge pull request #7 from ACMEAtronOmatic/upstream-changes

Upstream changes

.env

@@ -4,7 +4,7 @@
 TILESET_FILE=transportation.yaml
 
 # Use 3-part patch version to ignore patch updates, e.g. 5.0.0
-TOOLS_VERSION=5.3
+TOOLS_VERSION=6.1
 
 # Make sure these values are in sync with the ones in .env-postgres file
 PGDATABASE=openmaptiles
@@ -18,16 +18,17 @@ PGPORT=5432
 # By default, the Makefile will use the content of data/$(area).bbox file if it exists.
 BBOX=-90.8056635,35.6632467,-79.8779282,41.8913732
 
-# Which zooms to generate in make generate-tiles
+# Which zooms to generate with make generate-tiles-pg
 MIN_ZOOM=0
 MAX_ZOOM=8
 
+# `MID_ZOOM` setting only works with `make generate-tiles-pg` command. Make sure MID_ZOOM < MAX_ZOOM.
+# See https://github.com/openmaptiles/openmaptiles-tools/pull/383
+# MID_ZOOM=11
+
 # Use true (case sensitive) to allow data updates
 DIFF_MODE=false
 
-# Hide some output from Mapnik tile generation for clarity
-FILTER_MAPNIK_OUTPUT=1
-
 # Some area data like openstreetmap.fr can contain invalid references
 # that must be cleaned up before using it for borders -- set it to true.
 BORDERS_CLEANUP=false
@@ -49,7 +50,5 @@ MAX_PARALLEL_PSQL=5
 # Number of parallel threads to use when generating vector map tiles
 COPY_CONCURRENCY=10
 
-# Variables for generate tiles using PGquery
+# Variables for generate tiles using tilelive-pgquery
 PGHOSTS_LIST=
-NO_GZIP=1
-USE_KEY_COLUMN=1

.github/workflows/integrity.yml

@@ -0,0 +1,71 @@
+# Workflow to run basic integrity checks on OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles Integrity CI
+
+on:
+ push:
+ branches: [ master, master-tools ]
+ pull_request:
+
+jobs:
+
+ integrity_test:
+ name: Run integrity test
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Checkout the changes
+ uses: actions/checkout@v2
+
+ - name: Run quickstart for a small area
+ env:
+ area: monaco
+ QUIET: 1
+ run: |
+ echo MIN_ZOOM=0 >> .env
+ echo MAX_ZOOM=14 >> .env
+ ./quickstart.sh $area
+
+ - name: Save quickstart.log
+ uses: actions/upload-artifact@v2
+ with:
+ name: quickstart.log
+ path: quickstart.log
+
+ - name: Test etldoc images
+ run: |
+ export TEST_MODE=yes
+ make generate-devdoc
+
+ - name: Run quickstart and update in DIFF mode
+ env:
+ area: europe/monaco
+ QUIET: 1
+ run: |
+ echo MIN_ZOOM=0 >> .env
+ echo MAX_ZOOM=14 >> .env
+ echo DIFF_MODE=true >> .env
+ # Cleanup
+ rm -fr data build cache
+ # Create data/$area.repl.json
+ make download-geofabrik area=$area
+ # Download 2+ month old data
+ export old_date=$(date --date="$(date +%Y-%m-15) -2 month" +'%y%m01')
+ echo Downloading $old_date extract of $area
+ docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -O data/$area.osm.pbf http://download.geofabrik.de/$area-$old_date.osm.pbf"
+ # Initial import and tile generation
+ ./quickstart.sh $area
+ sleep 2
+ echo Downloading updates
+ # Loop to recover from potential "ERROR 429: Too Many Requests"
+ docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "
+ while ! osmupdate --keep-tempfiles --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz ; do 
+ sleep 2;
+ echo Sleeping...;
+ sleep 630;
+ done"
+ echo Downloading updates completed
+ echo Importing updates
+ make import-diff
+ echo Generating new tiles
+ make generate-tiles-pg

.github/workflows/tests.yml → .github/workflows/performance.yml

@@ -1,42 +1,13 @@
-# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
+# Workflow to run performance tests OMT`s new Pull Requests and commits pushed into OMT repo
 
-name: OpenMapTiles CI
+name: OpenMapTiles Performance CI
 
 on:
  push:
  branches: [ master, master-tools ]
  pull_request:
 
 jobs:
-
- integrity_test:
- name: Run integrity test
- runs-on: ubuntu-latest
- steps:
-
- - name: Checkout the changes
- uses: actions/checkout@v2
-
- - name: Run quickstart for a small area
- env:
- area: monaco
- MIN_ZOOM: 0
- MAX_ZOOM: 14
- QUIET: 1
- run: |
- ./quickstart.sh $area
-
- - name: Save quickstart.log
- uses: actions/upload-artifact@v2
- with:
- name: quickstart.log
- path: quickstart.log
-
- - name: Test etldoc images
- run: |
- export TEST_MODE=yes
- make generate-devdoc
-
  performance:
  name: Evaluate performance
  runs-on: self-hosted
@@ -182,23 +153,17 @@ jobs:
  make start-db
  profile 1_data make import-data
  profile 2_osm make import-osm
- profile 3_borders make import-borders
 
  if [ -f ../ci_cache/wikidata-cache.json ]; then
  cp ../ci_cache/wikidata-cache.json cache/wikidata-cache.json
  fi
- profile 4_wikidata make import-wikidata
- profile 5_sql make import-sql
+ profile 3_wikidata make import-wikidata
+ profile 4_sql make import-sql
 
  # Get database total size, in MB
  # Once Makefile has a few more improvements, we can use this approach instead:
  # echo $'\\set QUIET on \\a \\x off \\t \\\\ select pg_database_size(current_database())/1024/1024;' | make -s psql
- if grep -qE '^ import-osm:$' docker-compose.yml; then
- # old version using dedicated import-osm docker image
- DB_SIZE_MB=$(docker-compose run --rm -u $(id -u):$(id -g) import-osm ./psql.sh -qtAc 'select pg_database_size(current_database())/1024/1024;')
- else
- DB_SIZE_MB=$(docker-compose run --rm -u $(id -u):$(id -g) openmaptiles-tools psql.sh -qtAc 'select pg_database_size(current_database())/1024/1024;')
- fi
+ DB_SIZE_MB=$(docker-compose run --rm -u $(id -u):$(id -g) openmaptiles-tools psql.sh -qtAc 'select pg_database_size(current_database())/1024/1024;')
  docker-compose run --rm -u $(id -u):$(id -g) openmaptiles-tools pg_dump --schema-only > "${PROFILE_DIR}/schema.sql"
  echo "$DB_SIZE_MB" > "${PROFILE_DIR}/db_size.tsv"
  }
@@ -218,6 +183,7 @@ jobs:
  git reset --hard ${CURRENT_SHA}^1
  fi
 
+ docker-compose pull
  PROFILE_DIR=../perf_cache
  create_db
  if [ ! -f ../ci_cache/wikidata-cache.json ]; then
@@ -239,6 +205,7 @@ jobs:
  echo "Found cached performance results"
  fi
 
+ docker-compose pull
  pushd ../perf_cache
  echo "Should be in perf_cache"
  pwd

.github/workflows/pr-updater.yml

@@ -1,21 +1,20 @@
 name: Update PR comments
 
 on:
- # This number should correspond to the IGNORE_RUNS_OLDER_THAN value below.
- # When setting up for the first time, use "on: push" instead of "on: schedule"
- # and set IGNORE_RUNS_OLDER_THAN to a very high number until it runs once.
- schedule:
- - cron: '*/6 * * * *'
+ workflow_run:
+ workflows: ["OpenMapTiles Performance CI"]
+ types: [completed]
 
 jobs:
  update_PRs:
  runs-on: ubuntu-latest
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
 
  steps:
  - name: main
  env:
  GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
- WORKFLOW_NAME: "OpenMapTiles CI"
+ WORKFLOW_NAME: "OpenMapTiles Performance CI"
  # the name of the artifact whose content comment published by PR. Must have a single markdown file inside.
  MSG_ARTIFACT_NAME: "pr_message"
  # How far back to look for finished runs, in minutes.

.github/workflows/sql-tests.yml

@@ -0,0 +1,22 @@
+# Workflow to run unit tests on OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles SQL Test CI
+
+on:
+ push:
+ branches: [ master, master-tools ]
+ pull_request:
+
+jobs:
+
+ unit_tests:
+ name: Run unit test
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Checkout the changes
+ uses: actions/checkout@v2
+
+ - name: Run unit tests
+ run: |
+ make clean && make test-sql

CONTRIBUTING.md

@@ -41,3 +41,48 @@ When you modify import data rules in `mapping.yaml` or `*.sql`, please update:
 5. check if OMT styles are affected by the PR and if there is a need for style updates
 
 When you are making PR that adds new spatial features to OpenMapTiles schema, please make also PR for at least one of our GL styles to show it on the map. Visual check is crucial.
+
+# SQL unit testing
+
+It is recommended that you create a [unit test](TESTING.md) when modifying the behavior of the SQL layer. This will ensure that your changes are working as expected when importing or updating OSM data into an OpenMapTiles database.
+
+# Verifying that updates still work
+
+When testing a PR, you should also verify that the update process completes without an error. Please modify, if necessary, and run the script below.
+
+**Note:**
+
+The verification requires the script to append temporary changes to the `.env` file. Please restore the original version from git using `git checkout .env` or remove these changes before submitting a PR.
+
+```
+(
+set -e
+
+cat >> .env << EOM
+
+# temporary changes for verifying that updates still work
+# Ensure DIFF_MODE is active
+DIFF_MODE=true
+# Ensure all zoom levels are tested
+MAX_ZOOM=14
+EOM
+
+# Set the test area to the appropriate geofabrik extract
+export area=north-america/us/indiana
+
+# Build 1-month-old tiles
+rm -fr data build cache
+make destroy-db
+make download-geofabrik area=$area
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -nv -O data/$area.osm.pbf http://download.geofabrik.de/$area-$(date --date="$(date +%Y-%m-15) -1 month" +'%y%m01').osm.pbf"
+./quickstart.sh $area
+cat << EOM
+
+# Update with the changes since a month+ ago
+
+EOM
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "osmupdate --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz"
+make import-diff
+make generate-tiles-pg
+) < /dev/null
+```

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors. 
+Copyright (c) 2021, MapTiler.com & OpenMapTiles contributors. 
 All rights reserved. 
 
 The vector tile schema has been developed by Klokan Technologies GmbH and
@@ -55,6 +55,6 @@ For printed and static maps a similar attribution should be made in a textual
 description near the image, in the same fashion as if you cite a photograph.
 
 Exceptions to OpenMapTiles attribution requirement can be in a written form granted
-by Klokan Technologies GmbH (info@klokantech.com). 
-The project contributors grant Klokan Technologies GmbH the license to give such
+by MapTiler (info@maptiler.com). 
+The project contributors grant MapTiler AG the license to give such
 exceptions on a commercial basis.