Improve bash commands, extract ensure-test-data.sh from test.sh (#106)

Fix #81

Makefile

@@ -143,8 +143,18 @@ celery-worker-test-bash:
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm celery_worker_test bash
 
 test:
-	docker-compose -f docker-compose.deps.yml -f docker-compose.dev.yml build layman_dev
+	mkdir -p tmp
+	if [ "$$(docker images -q layman_dev 2> /dev/null)" = "" ]; then \
+		docker-compose -f docker-compose.deps.yml -f docker-compose.dev.yml build layman_dev ; \
+	fi;
+	if [ "$$(docker images -q timgen 2> /dev/null)" = "" ]; then \
+		docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml build timgen ; \
+	fi;
+	if [ "$$(docker images -q layman_client_test 2> /dev/null)" = "" ]; then \
+		docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml build layman_client_test ; \
+	fi;
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test
+	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "bash ensure-test-data.sh"
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_gs_auth.py"
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test
 	docker-compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test

docker-compose.test.yml

@@ -15,12 +15,11 @@ services:
         - ./test:/code/test
         - ./tmp:/code/tmp
         - ./test.sh:/code/test.sh
+        - ./ensure-test-data.sh:/code/ensure-test-data.sh
         - ./deps/geoserver/data:/geoserver/data_dir
         - ./deps/geoserver/sample/geoserver_data:/geoserver/initial_data_dir
       depends_on:
         - celery_worker_test
-        - timgen
-        - layman_client_test
 
    celery_worker_test:
       image: layman_dev:latest
@@ -41,6 +40,8 @@ services:
         - postgresql
         - geoserver
         - micka
+        - timgen
+        - layman_client_test
 
    timgen:
       container_name: timgen

ensure-test-data.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -ex
+
+bash src/clear-python-cache.sh
+
+mkdir -p tmp/naturalearth/110m/cultural
+mkdir -p tmp/naturalearth/50m/cultural
+mkdir -p tmp/naturalearth/10m/cultural
+mkdir -p tmp/data200/trans/jtsk
+mkdir -p tmp/sm5/vektor/jtsk
+
+ne_110m_cultural=tmp/naturalearth/110m_cultural.zip
+if ! [ -f $ne_110m_cultural ]; then
+  curl -L -o $ne_110m_cultural "https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/110m_cultural.zip"
+  unzip -q $ne_110m_cultural -d tmp/naturalearth/110m/cultural
+fi
+
+ne_110m_cultural_admin_0_countries=tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson
+if ! [ -f $ne_110m_cultural_admin_0_countries ]; then
+  curl -L -o $ne_110m_cultural_admin_0_countries "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_110m_admin_0_countries.geojson"
+fi
+
+ne_10m_cultural_admin_0_countries=tmp/naturalearth/10m/cultural/ne_10m_admin_0_countries.geojson
+if ! [ -f $ne_10m_cultural_admin_0_countries ]; then
+  curl -L -o $ne_10m_cultural_admin_0_countries "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_10m_admin_0_countries.geojson"
+fi
+
+ne_50m_cultural_admin_0_countries=tmp/naturalearth/50m/cultural/ne_50m_admin_0_countries.geojson
+if ! [ -f $ne_50m_cultural_admin_0_countries ]; then
+  curl -L -o $ne_50m_cultural_admin_0_countries "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_50m_admin_0_countries.geojson"
+fi
+
+ne_110m_cultural_populated_places=tmp/naturalearth/110m/cultural/ne_110m_populated_places.geojson
+if ! [ -f $ne_110m_cultural_populated_places ]; then
+  curl -L -o $ne_110m_cultural_populated_places "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_110m_populated_places.geojson"
+fi
+
+data200trans=tmp/data200/trans/trans-jtsk.zip
+data200transJtsk=tmp/data200/trans/jtsk
+data200transRoad=tmp/data200/trans/RoadL.shp
+if ! [ -f $data200transRoad ]; then
+  curl -A "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36" -L -o $data200trans "https://geoportal.cuzk.cz/ZAKAZKY/Data200/TRANS.zip"
+  unzip -q $data200trans -d $data200transJtsk
+  for f in tmp/data200/trans/jtsk/RoadL.shp
+  do
+   echo "Processing $f"
+    ogr2ogr -t_srs EPSG:3857 -lco ENCODING=UTF-8 tmp/data200/trans/$(basename $f) $f
+  done
+fi
+
+sm5building=tmp/sm5/vektor/sm5.zip
+sm5VectorJtsk=tmp/sm5/vektor/jtsk
+sm5VectorBuilding=tmp/sm5/vektor/Budova.shp
+if ! [ -f $sm5VectorBuilding ]; then
+  curl -A "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36" -L -o $sm5building "https://geoportal.cuzk.cz/UKAZKOVA_DATA/SM5_VEKTOR_NOVA.zip"
+  unzip -q $sm5building -d $sm5VectorJtsk
+  for f in tmp/sm5/vektor/jtsk/SM5_VEKTOR_NOVA/shp/Budova.shp
+  do
+   echo "Processing $f"
+    ogr2ogr -s_srs EPSG:5514 -t_srs EPSG:3857 -lco ENCODING=UTF-8 tmp/sm5/vektor/$(basename $f) $f
+  done
+fi

start.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -ex
+
 bash src/clear-python-cache.sh
 
 

test.sh

@@ -1,64 +1,8 @@
 #!/bin/bash
 
-bash src/clear-python-cache.sh
-
-mkdir -p tmp/naturalearth/110m/cultural
-mkdir -p tmp/naturalearth/50m/cultural
-mkdir -p tmp/naturalearth/10m/cultural
-mkdir -p tmp/data200/trans/jtsk
-mkdir -p tmp/sm5/vektor/jtsk
-
-ne_110m_cultural=tmp/naturalearth/110m_cultural.zip
-if ! [ -f $ne_110m_cultural ]; then
-  curl -L -o $ne_110m_cultural "https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/110m_cultural.zip"
-  unzip -q $ne_110m_cultural -d tmp/naturalearth/110m/cultural
-fi
-
-ne_110m_cultural_admin_0_countries=tmp/naturalearth/110m/cultural/ne_110m_admin_0_countries.geojson
-if ! [ -f $ne_110m_cultural_admin_0_countries ]; then
-  curl -L -o $ne_110m_cultural_admin_0_countries "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_110m_admin_0_countries.geojson"
-fi
-
-ne_10m_cultural_admin_0_countries=tmp/naturalearth/10m/cultural/ne_10m_admin_0_countries.geojson
-if ! [ -f $ne_10m_cultural_admin_0_countries ]; then
-  curl -L -o $ne_10m_cultural_admin_0_countries "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_10m_admin_0_countries.geojson"
-fi
+set -ex
 
-ne_50m_cultural_admin_0_countries=tmp/naturalearth/50m/cultural/ne_50m_admin_0_countries.geojson
-if ! [ -f $ne_50m_cultural_admin_0_countries ]; then
-  curl -L -o $ne_50m_cultural_admin_0_countries "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_50m_admin_0_countries.geojson"
-fi
-
-ne_110m_cultural_populated_places=tmp/naturalearth/110m/cultural/ne_110m_populated_places.geojson
-if ! [ -f $ne_110m_cultural_populated_places ]; then
-  curl -L -o $ne_110m_cultural_populated_places "https://github.com/nvkelso/natural-earth-vector/raw/master/geojson/ne_110m_populated_places.geojson"
-fi
-
-data200trans=tmp/data200/trans/trans-jtsk.zip
-data200transJtsk=tmp/data200/trans/jtsk
-data200transRoad=tmp/data200/trans/RoadL.shp
-if ! [ -f $data200transRoad ]; then
-  curl -A "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36" -L -o $data200trans "https://geoportal.cuzk.cz/ZAKAZKY/Data200/TRANS.zip"
-  unzip -q $data200trans -d $data200transJtsk
-  for f in tmp/data200/trans/jtsk/RoadL.shp
-  do
-   echo "Processing $f"
-    ogr2ogr -t_srs EPSG:3857 -lco ENCODING=UTF-8 tmp/data200/trans/$(basename $f) $f
-  done
-fi
-
-sm5building=tmp/sm5/vektor/sm5.zip
-sm5VectorJtsk=tmp/sm5/vektor/jtsk
-sm5VectorBuilding=tmp/sm5/vektor/Budova.shp
-if ! [ -f $sm5VectorBuilding ]; then
-  curl -A "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36" -L -o $sm5building "https://geoportal.cuzk.cz/UKAZKOVA_DATA/SM5_VEKTOR_NOVA.zip"
-  unzip -q $sm5building -d $sm5VectorJtsk
-  for f in tmp/sm5/vektor/jtsk/SM5_VEKTOR_NOVA/shp/Budova.shp
-  do
-   echo "Processing $f"
-    ogr2ogr -s_srs EPSG:5514 -t_srs EPSG:3857 -lco ENCODING=UTF-8 tmp/sm5/vektor/$(basename $f) $f
-  done
-fi
+bash src/clear-python-cache.sh
 
 mkdir -p tmp/artifacts
 rm -rf tmp/artifacts/*