Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

More generic OGR converter to use GPKG files #44

Open
wants to merge 25 commits into
base: master
Choose a base branch
from
Open

More generic OGR converter to use GPKG files #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
03548f4
More generic OGR converter to use GPKG files
pailakka Apr 13, 2022
cbed97b
Convert GPKG with new converter
pailakka Apr 13, 2022
cc24377
Use osgeo gdal container as base
pailakka Apr 14, 2022
c5b37ec
Minimize jar and remove signature
pailakka Apr 14, 2022
5def37a
Do not minimize :I
pailakka Apr 14, 2022
05f42f0
Use osgeo gdal full image
pailakka Apr 14, 2022
7262a8d
Paraller process and faster read
pailakka Apr 15, 2022
a26486f
Logging and more aggressive parallel
pailakka Apr 15, 2022
f78b422
Use unzipped gpkg
pailakka Apr 15, 2022
ae5b4e4
Update proto sources & convert stringtable short -> int
pailakka Apr 16, 2022
8d19535
Smaller chunks
pailakka Apr 18, 2022
763af78
Store batched osm pbf and fix strintables
pailakka Apr 29, 2022
e1423ce
Older java version
pailakka Apr 29, 2022
f1727a3
Fix writing osmpbf from recursive input
pailakka Apr 30, 2022
3eed678
Fix writing osmpbf with non recursive input
pailakka Apr 30, 2022
49308c0
Smaller chunks
pailakka May 1, 2022
8453ad4
Better splitting
pailakka May 2, 2022
ac18861
Revert "Better splitting"
pailakka May 3, 2022
04c4dda
Do not batch reading
pailakka May 3, 2022
980bd4d
True threading
pailakka May 5, 2022
9212dae
Fix deadlocks
pailakka May 6, 2022
d32545e
Batched write
pailakka May 7, 2022
92916db
Smaller blocks
pailakka May 11, 2022
deb9296
Fix merging
pailakka May 20, 2022
ed15d6c
Merge branch 'generic-ogr-converter' of github.com:pailakka/mtk2garmi…
pailakka May 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions get-additional-data/get_additional_data.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
set -e

mkdir -p /additional-data
curl --fail --verbose --retry 5 --retry-delay 30 -o /additional-data/grid.zip "https://tiedostopalvelu.maanmittauslaitos.fi/geoserver/karttalehti/ows?service=wfs&request=GetFeature&typeName=Grid&outputFormat=shape-zip&cql_filter=gridSize=%2712x12%27"
#curl --fail --verbose --retry 5 --retry-delay 30 -o /additional-data/grid.zip "https://tiedostopalvelu.maanmittauslaitos.fi/geoserver/karttalehti/ows?service=wfs&request=GetFeature&typeName=Grid&outputFormat=shape-zip&cql_filter=gridSize=%2712x12%27"
curl --fail --verbose --retry 5 --retry-delay 30 -o /additional-data/finland-latest.osm.pbf "http://download.geofabrik.de/europe/finland-latest.osm.pbf"
ogr2ogr --config OGR_WFS_PAGE_SIZE 4000 --debug on -nln syvyyskayra_v -f "ESRI shapefile" /additional-data/syvyyskayra_v.shp WFS:https://julkinen.traficom.fi/inspirepalvelu/rajoitettu/wfs?typeName=rajoitettu:DepthContour_L
ogr2ogr --config OGR_WFS_PAGE_SIZE 4000 --debug on -nln syvyyspiste_p -f "ESRI shapefile" /additional-data/syvyyspiste_p.shp WFS:https://julkinen.traficom.fi/inspirepalvelu/rajoitettu/wfs?typeName=rajoitettu:Sounding_P
Expand All @@ -16,4 +16,4 @@ ogrinfo -so /additional-data/syvyyskayra_v.shp syvyyskayra_v
#ogrinfo -so /additional-data/kesaretkeilyreitit.gml kesaretkeilyreitit
#ogrinfo -so /additional-data/ulkoilureitit.gml ulkoilureitit
#ogrinfo -so /additional-data/luontopolut.gml luontopolut
#ogrinfo -so /additional-data/point_dump.gml point_dump
#ogrinfo -so /additional-data/point_dump.gml point_dump
41 changes: 29 additions & 12 deletions mapcreator/convert_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ set -euxo pipefail

printf -v date '%(%Y%m%d)T' -1

if test -f ".env"; then
echo "sourcing .env"
source .env
fi

docker-compose down -v

docker pull quay.io/azavea/openjdk-gdal:3.1-jdk11-slim
docker build --tag teemupel/mtk2garmin-ubuntugis-base -f ./ubuntugis-base/Dockerfile ./ubuntugis-base
docker push teemupel/mtk2garmin-ubuntugis-base

Expand All @@ -20,31 +24,44 @@ fi

docker-compose pull

time docker-compose run mml-client /go/src/app/mml-muutostietopalvelu-client load -p maastotietokanta -t kaikki -f application/gml+xml -d /mtkdata
time docker-compose run mml-client /go/src/app/mml-muutostietopalvelu-client load -p kiinteistorekisterikartta -t karttalehdittain -f application/x-shapefile -d /krkdata
zmaasto=""
zkorkeus=""

if test -f "/opt/mtkdata/mtkmaasto.zip"; then
zmaasto="-z /opt/mtkdata/mtkmaasto.zip"
fi

if test -f "/opt/mtkdata/mtkkorkeus.zip"; then
zkorkeus="-z /opt/mtkdata/mtkkorkeus.zip"
fi

time curl "$zmaasto" -o /opt/mtkdata/mtkmaasto.zip "https://tiedostopalvelu.maanmittauslaitos.fi/tp/tilauslataus/tuotteet/maastotietokanta/geopackage_maasto/mtkmaasto.zip?api_key=$MTK_API_KEY"
time curl "$zkorkeus" -o /opt/mtkdata/mtkkorkeus.zip "https://tiedostopalvelu.maanmittauslaitos.fi/tp/tilauslataus/tuotteet/maastotietokanta/geopackage_korkeus/mtkkorkeus.zip?api_key=$MTK_API_KEY"
time 7z e /opt/mtkdata/mtkmaasto.zip -o/opt/mtkdata/
time 7z e /opt/mtkdata/mtkkorkeus.zip -o/opt/mtkdata/
time docker-compose run mml-client /go/src/app/mml-muutostietopalvelu-client load -p kiinteistorekisterikartta -t karttalehdittain -f application/x-shapefile -d /krkdata

docker-compose up --no-start additional-data
docker-compose up --no-start mapstyles

time docker-compose run mtk2garmin-converter java -jar /opt/mtk2garmin/target/mtk2garmin-0.0.2.jar /opt/mtk2garmin/mtk2garmin.conf
time docker-compose run mtk2garmin-converter java -jar /opt/mtk2garmin/target/mtk2garmin-0.5.jar /opt/mtk2garmin/mtk2garmin.conf

time docker-compose run merger ./merge_files.sh

time docker-compose run mkgmap ./run_mkgmap.sh

time docker-compose run mapsforge /app/bin/osmosis \
--rbf file=/convertedpbf/all_osm.osm.pbf workers=2 \
--mapfile-writer file=/output/mtk_all.map bbox=59.4507573,19.0714057,70.1120744,31.6133108 \
simplification-max-zoom=12 simplification-factor=16 threads=4 \
zoom-interval-conf=5,4,7,8,8,11,12,12,13,14,14,21 \
label-position=true polylabel=true \
tag-conf-file=/mapstyles/mapsforge_peruskartta/mml_tag-mapping_tidy.xml type=hd comment="(c) NLS, Metsahallitus, Liikennevirasto, OpenStreetMap contributors 2019"

--rbf file=/convertedpbf/all_osm.osm.pbf workers=2 \
--mapfile-writer file=/output/mtk_all.map bbox=59.4507573,19.0714057,70.1120744,31.6133108 \
simplification-max-zoom=12 simplification-factor=16 threads=4 \
zoom-interval-conf=5,4,7,8,8,11,12,12,13,14,14,21 \
label-position=true polylabel=true \
tag-conf-file=/mapstyles/mapsforge_peruskartta/mml_tag-mapping_tidy.xml type=hd comment="(c) NLS, Metsahallitus, Liikennevirasto, OpenStreetMap contributors 2019"

time docker-compose run osxconverter

time docker-compose run nsis /output/mtkgarmin/osmmap.nsi
time docker-compose run nsis /output/mtkgarmin_noparcel/osmmap.nsi

time docker-compose run site
#time docker-compose run site
docker-compose down -v
Loading