Use Sonny's elevation data

This uses https://sonny.4lima.de/ elevation data for Europe.

This fixes tile generation for Iceland, Faroes and Svalbard.

Please consider a donation to Sonny for the great DTM that he is
producing and publishing for free.

.github/workflows/build-tiles.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Test
         run: |
           TILENAME=${{ inputs.setname }} make -f Makefile.tiles ${{ inputs.setname }}_tile_list_chunks
-          TILENAME=${{ inputs.setname }} make -n -f Makefile.tiles ${{ inputs.zip }}
+          TILENAME=${{ inputs.setname }} make -n -f Makefile.tiles ${{ inputs.zip }} -j $(nproc)
 
       - name: Setup Ortho4XP
         run: make Ortho4XP
@@ -51,7 +51,7 @@ jobs:
         id: build
         run: |
           TILENAME=${{ inputs.setname }} make -f Makefile.tiles ${{ inputs.setname }}_tile_list_chunks
-          TILENAME=${{ inputs.setname }} timeout 300m make -f Makefile.tiles ${{ inputs.zip }} && echo "done=true" >> "$GITHUB_OUTPUT" || echo "done=false" >> "$GITHUB_OUTPUT"
+          TILENAME=${{ inputs.setname }} timeout 300m make -f Makefile.tiles ${{ inputs.zip }} -j $(nproc) && echo "done=true" >> "$GITHUB_OUTPUT" || echo "done=false" >> "$GITHUB_OUTPUT"
           
       - name: Show work
         if: steps.build.outputs.done == 'true'

.gitignore

@@ -2,3 +2,5 @@
 *.swp
 Ortho4XP
 *_tile_list.*
+/var/
+/.venv/

Makefile.tiles

@@ -1,30 +1,52 @@
+# Requires Github CLI (gh) command to facilitate authenticated requests to the API
+#
+# Quick start:
+# make -f Makefile.tiles clean 
+# make -f Makefile.tiles test_tile_list_chunks
+# make -f Makefile.tiles -j $(nproc --ignore=2)
+
+# @todo: make release in kubilus/autoortho-scenery, see ./bin/prepareAssetsElevationData
+ELEV_RELEASE_JSON_ENDPOINT?=repos/jonaseberle/autoortho-scenery/releases/tags/elevation-v0.0.1
 SPLITSIZE?=125
 SHELL=/bin/bash
 ZL?=16
 
 TILENAME?=test
 
-TILES:=$(addprefix z_$(TILENAME)_, $(shell ls $(TILENAME)_tile_list.* | awk -F. '{ print $$2 }') ) 
+TILES:=$(addprefix z_$(TILENAME)_, $(shell ls $(TILENAME)_tile_list.* 2>/dev/null | awk -F. '{ print $$2 }') ) 
 TILE_ZIPS=$(addsuffix .zip, $(TILES))
 
 ZIPS=$(TILE_ZIPS)
 
+# paranthesis to use in shell commands
+# make chokes on () in shell commands
+OP:=(
+CP:=)
+
 # Get the tiles listed in each list file
 .SECONDEXPANSION:
-TILE_FILES = $(addsuffix .dsf, $(addprefix Ortho4XP/Tiles/*/*/*/, $(basename $(shell cat $(TILENAME)_tile_list.$* ) ) ) )
-#TILE_FILES = $(basename $(shell cat $(TILENAME)_tile_list.$* ) )
-
+TILE_FILES = $(addsuffix .dsf, $(addprefix Ortho4XP/Tiles/*/*/*/, $(basename $(shell cat $(TILENAME)_tile_list.$* 2>/dev/null ) ) ) )
 
 all: $(ZIPS)
 
+# creates directories
+%/:
+	@echo [$@]
+	@mkdir -p $@
+
+#
+# Work on tile lists
+#
+
 z_$(TILENAME)_%: $(TILENAME)_tile_list.% $${TILE_FILES}
-	@echo "Going to do some $@"
+	@echo [$@]
 	@echo "Expected tile len:"
 	@cat $< | wc -l
 	@echo "Actual tile len:"
 	@unzip -l [email protected] *.dsf | grep dsf | wc -l
 
 z_$(TILENAME)_%.zip.info: $(TILENAME)_tile_list.%
+	@echo [$@]
 	@echo "-----------------------------------"
 	@echo "Post processing info:"
 	comm --total -3  <( unzip -l $(basename $@) *.dsf | awk -F/ '/.*\.dsf/ { print $$4 }' | sort ) <( cat $< | sort )
@@ -34,23 +56,67 @@ z_$(TILENAME)_%.zip.info: $(TILENAME)_tile_list.%
 	echo "Expected tile len: $$EXPECTED"; \
 	echo "Actual tile len: $$ACTUAL"; \
 	[ $$EXPECTED -eq $$ACTUAL ]
+
 #
 # Ortho4XP setup
 #
 
 ortho4xp.diff:
+	@echo [$@]
 	cd Ortho4XP && git diff > ../ortho4xp.diff
 
 Ortho4XP:
+	@echo [$@]
 	git clone --depth=1 https://github.com/oscarpilote/Ortho4XP.git
 	cd $@ && patch -p1 -u < ../ortho4xp.diff
 	cp extract_overlay.py $@/.
 	cp Ortho4XP.cfg $@/.
 	mkdir $@/tmp
+	@echo "Setting up symlinks in order to not care about Ortho4XP's expected directory structure in ./Elevation_data..."
+	mkdir -p $@/Elevation_data && cd $@/Elevation_data \
+		&& bash -c 'for lat in {-9..9}; do for lon in {-18..18}; do ln -snfr ./ "./$$(printf "%+d0%+03d0" "$$lat" "$$lon")"; done; done'
+
+#
+# Custom tile elevation
+#
 
+# generates the targets var/run/neighboursOfTile_%.elevation with surrounding tiles' elevations 
+# as prerequisites (takes a little while, 360*180 rules):
+var/run/Makefile.elevationRules: | $$(@D)/
+	@echo [$@]
+	@bin/genMakefileElevationRules > $@
+include var/run/Makefile.elevationRules
+
+var/run/tile_%.elevation: var/cache/elevation/elevation_%.zip Ortho4XP | $$(@D)/
+	@echo [$@]
+	@# unzip if file not empty, but fail on unzip error
+	@if [ -s "var/cache/elevation/elevation_$*.zip" ]; then \
+		unzip -o -d Ortho4XP/Elevation_data/ var/cache/elevation/elevation_$*.zip; \
+	fi
+	@touch $@
+
+var/run/elevationRelease.json: | $$(@D)/
+	@echo [$@]
+	gh api $(ELEV_RELEASE_JSON_ENDPOINT) --paginate > $@
+
+var/cache/elevation/elevation_%.zip: var/run/elevationRelease.json | $$(@D)/
+	@echo [$@]
+	@url=$$(jq -r '.assets[] | select$(OP).name == "elevation_$*.zip"$(CP) .browser_download_url' \
+			var/run/elevationRelease.json) \
+		&& [ -n "$$url" ] \
+		&& ( cd $(@D) && wget --continue --quiet "$$url" ) \
+		&& echo "[$@] downloaded $$url" \
+		|| ( echo "[$@] no custom elevation data for $* found"; true )
+	@# always create the file
+	@touch $@
+
+#
+# Split tile list
+#
 
 #$(TILENAME)_tile_list.%: $(TILENAME)_tile_list_chunks
 $(TILENAME)_tile_list_chunks: $(TILENAME)_tile_list
+	@echo [$@]
 	split $< -d -l $(SPLITSIZE) $<.
 
 
@@ -63,33 +129,44 @@ $(TILENAME)_tile_list_chunks: $(TILENAME)_tile_list
 #%.dsf: Ortho4XP/Tiles/zOrtho4XP_%/.done
 #	echo $<
 #Ortho4XP/Tiles/zOrtho4XP_%/.done: Ortho4XP 
-Ortho4XP/Tiles/*/*/*/%.dsf: Ortho4XP
-	@echo "Make tile $*"	
+Ortho4XP/Tiles/*/*/*/%.dsf: Ortho4XP var/run/neighboursOfTile_%.elevation
+	@echo [$@]
+	@echo "Make tile $*"
 	@echo "Setup per tile config, if possible"
 	mkdir -p Ortho4XP/Tiles/zOrtho4XP_$*
-	-cp Ortho4XP_$*.cfg Ortho4XP/Tiles/zOrtho4XP_$*/.
+	-cp Ortho4XP_$*.cfg Ortho4XP/Tiles/zOrtho4XP_$*/. 2>/dev/null
 	@echo "Make tile $(@)" 
 	set -e;\
 	export COORDS=$$(echo $(@) | sed -e 's/.*\/\([-+][0-9]\+\)\([-+][0-9]\+\).dsf/\1 \2/g');\
 	echo $$COORDS; \
- 	cd $< && python3 Ortho4XP_v130.py $$COORDS BI $(ZL) || ( echo "ERROR DETECTED! Retry tile $@ with noroads config."; cp $(CURDIR)/Ortho4XP_noroads.cfg $(CURDIR)/Ortho4XP/Tiles/zOrtho4XP_$*/Ortho4XP_$*.cfg && python3 Ortho4XP_v130.py $$COORDS BI $(ZL) )
+	cd Ortho4XP && python3 Ortho4XP_v130.py $$COORDS BI $(ZL) || ( echo "ERROR DETECTED! Retry tile $@ with noroads config."; cp $(CURDIR)/Ortho4XP_noroads.cfg $(CURDIR)/Ortho4XP/Tiles/zOrtho4XP_$*/Ortho4XP_$*.cfg && python3 Ortho4XP_v130.py $$COORDS BI $(ZL) )
 	#touch $@
 
 .SECONDARY: $(TILE_FILES) 
 
 # Static pattern rule for the zip files
 $(ZIPS): z_%.zip: z_%
+	@echo [$@]
 	mkdir -p $<
 	cp -r Ortho4XP/Tiles/zOrtho4XP_*/'Earth nav data' $</.
 	cp -r Ortho4XP/Tiles/zOrtho4XP_*/terrain $</.
 	cp -r Ortho4XP/Tiles/zOrtho4XP_*/textures $</.
 	zip -r $@ $<
 
 %.sha256: %		
+	@echo [$@]
 	sha256sum $< > $@
 
 clean:
+	@echo [$@]
 	-rm -rf Ortho4XP
-	-rm $(ZIPS)
-	-rm -rf z_$(TILENAME)
-	-rm $(TILENAME)_tile_list.*
+	-rm -rf var/run
+	-rm -f $(ZIPS)
+	-rm -rf z_$(TILENAME)*
+	-rm -f $(TILENAME)_tile_list.*
+
+distclean: clean
+	@echo [$@]
+	-rm -rf var
+	-rm -f z_*
+	-rm -f *_tile_list.*

bin/genMakefileElevationRules

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Outputs Makefile rules for elevations.
+# Ortho4XP uses all surrounding tiles' elevations for generation of a tile.
+# The generated rules look like this:
+# var/run/neighboursOfTile_+12+123.elevation: var/run/tile_+12+123.elevation ... |$(@D)
+
+# saner programming env: these switches turn some bugs into errors
+set -o errexit -o pipefail -o noclobber -o nounset
+
+for lat in $(seq -90 90); do
+	for lon in $(seq -180 179); do
+		printf "var/run/neighboursOfTile_%+03d%+04d.elevation:" "$lat" "$lon"
+		for dLon in -1 0 1; do
+			for dLat in -1 0 1; do
+				_lat="$((lat + dLat))"
+				# wrap around although not needed in 100 years. Elevation data for the Aleutians anybody?
+				_lon="$(( ($lon + $dLon + 540) % 360 - 180))"
+				printf "var/run/tile_%+03d%+04d.elevation " "$_lat" "$_lon"
+			done
+		done
+		printf '|$(@D)'"\n"
+	done
+done

bin/prepareAssetsElevationData

@@ -0,0 +1,30 @@
+#!/bin/bash
+# saner programming env: these switches turn some bugs into errors
+set -o errexit -o pipefail -o noclobber -o nounset
+
+zipsDir="${1:-}"
+dir="$(dirname "$0")"
+targetDir="$(readlink -f "$dir/../var/cache/elevation")"
+
+if ! [ -d "$zipsDir" ]; then
+	cat <<-EOL
+		Usage: $0 <directory with single .zips from sonny.4lima.de>
+		
+		The source .zips will be copied to
+		‹$targetDir›
+		and renamed to tile name format for being uploaded to a release.
+
+		Example with the Github CLI: 
+		‹for i in var/cache/elevation/*; do gh release upload elevation-v0.0.1 \$i; done›
+	EOL
+	exit 1
+fi
+
+mkdir -p "$targetDir"
+
+for filePath in "$zipsDir"/*.zip; do
+	fileNameElevationFormat="$(basename "$filePath")"
+	fileNameTile="elevation_$(sed -E -e 's/N|E/+/g' -e 's/S|W/-/g' <<< "$fileNameElevationFormat")"
+	printf "%s\n  → %s\n" "$fileNameElevationFormat" "$fileNameTile"
+	cp "$filePath" "$targetDir/$fileNameTile"
+done

test_tile_list

@@ -1,2 +1,10 @@
 +00-051.dsf
 +00-052.dsf
++64-016.dsf
++61-008.dsf
++61-007.dsf
++62-008.dsf
++62-007.dsf
++78+013.dsf
++78+014.dsf
++78+015.dsf