Use unified tools version 3.1.0 for all images (#707)

* Use unified tools version for all images
* do not start postserve as part of quickstart, but added a help message how to start it
* wait for SQL start with pgwait

Makefile

@@ -1,8 +1,11 @@
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit
-DC_OPTS:=--rm
+DC_OPTS?=--rm
 
 # container runs as the current user rather than root (so that created files are not root-owned)
-DC_USER_OPTS:=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+
+TOOLS_VERSION?=$(shell cat TOOLS_VERSION)
+export TOOLS_VERSION
 
 .PHONY: all
 all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
@@ -72,6 +75,8 @@ clean-docker:
 .PHONY: db-start
 db-start:
 	docker-compose up -d postgres
+	@echo "Wait for PostgreSQL to start..."
+	docker-compose run $(DC_OPTS) import-osm  ./pgwait.sh
 
 .PHONY: download-geofabrik
 download-geofabrik:
@@ -96,12 +101,12 @@ import-osm: db-start all
 
 .PHONY: import-sql
 import-sql: db-start all
-	docker-compose run $(DC_OPTS) import-sql
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
 
 .PHONY: import-osmsql
 import-osmsql: db-start all
 	docker-compose run $(DC_OPTS) import-osm
-	docker-compose run $(DC_OPTS) import-sql
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
 
 .PHONY: generate-tiles
 generate-tiles: db-start all
@@ -136,11 +141,11 @@ start-tileserver:
 	docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl
 
 .PHONY: start-postserve
-start-postserve:
+start-postserve: db-start
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
-	@echo "* Bring up postserve at localhost:8090/tiles/{z}/{x}/{y}.pbf"
+	@echo "* Bring up postserve at localhost:8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
@@ -150,7 +155,8 @@ start-postserve:
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Start maputnik/editor "
-	@echo "*       ----------------------------> check localhost:8088 "
+	@echo "*       ---> go to http://localhost:8088"
+	@echo "*       ---> set 'data source' to  http://localhost:8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
@@ -191,7 +197,7 @@ mapping-graph:
 	@echo 'Valid layers: $(mappingLayers)'
 
 mapping-graph-%: ./layers/%/mapping.yaml build/devdoc
-	docker run $(DC_USER_OPTS) -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
+	docker run $(DC_USER_OPTS) -v $$(pwd):/tileset openmaptiles/openmaptiles-tools:${TOOLS_VERSION} generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
 	cp ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png
 
 # generate all etl and mapping graphs
@@ -199,7 +205,7 @@ generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(
 
 .PHONY: import-sql-dev
 import-sql-dev:
-	docker-compose run $(DC_OPTS) import-sql /bin/bash
+	docker-compose run $(DC_OPTS) openmaptiles-tools bash
 
 .PHONY: import-osm-dev
 import-osm-dev:

TOOLS_VERSION

@@ -0,0 +1 @@
+3.1.0

docker-compose.yml

@@ -4,7 +4,7 @@ volumes:
   cache:
 services:
   postgres:
-    image: "openmaptiles/postgis:2.9"
+    image: "openmaptiles/postgis:${TOOLS_VERSION}"
     volumes:
       - pgdata:/var/lib/postgresql/data
     networks:
@@ -13,22 +13,22 @@ services:
       - "5432"
     env_file: .env
   import-natural-earth:
-    image: "openmaptiles/import-natural-earth:1.4"
+    image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
     env_file: .env
     networks:
       - postgres_conn
   import-water:
-    image: "openmaptiles/import-water:1.2"
+    image: "openmaptiles/import-water:${TOOLS_VERSION}"
     env_file: .env
     networks:
       - postgres_conn
   import-lakelines:
-    image: "openmaptiles/import-lakelines:1.0"
+    image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
     env_file: .env
     networks:
       - postgres_conn
   import-osm:
-    image: "openmaptiles/import-osm:0.5"
+    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
     env_file: .env
     environment:
       DIFF_MODE: ${DIFF_MODE}
@@ -39,12 +39,12 @@ services:
       - ./build:/mapping
       - cache:/cache
   import-osmborder:
-    image: "openmaptiles/import-osmborder:0.4"
+    image: "openmaptiles/import-osmborder:${TOOLS_VERSION}"
     env_file: .env
     networks:
       - postgres_conn
   import-osm-diff:
-    image: "openmaptiles/import-osm:0.5"
+    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
     env_file: .env
     command: ./import_diff.sh
     environment:
@@ -56,7 +56,7 @@ services:
       - ./build:/mapping
       - cache:/cache
   update-osm:
-    image: "openmaptiles/import-osm:0.5"
+    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
     env_file: .env
     environment:
       DIFF_MODE: ${DIFF_MODE}
@@ -68,14 +68,18 @@ services:
       - ./build:/mapping
       - cache:/cache
   import-sql:
-    image: "openmaptiles/import-sql:1.0"
+    # This target is obsolete, and was left for backwards compatibility
+    # Use openmaptiles-tools target instead
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+    command: import-sql
     env_file: .env
     networks:
       - postgres_conn
     volumes:
+      - .:/tileset
       - ./build:/sql
   import-wikidata:
-    image: "openmaptiles/import-wikidata:0.1"
+    image: "openmaptiles/import-wikidata:${TOOLS_VERSION}"
     env_file: .env
     command: import-wikidata
     networks:
@@ -83,7 +87,7 @@ services:
     volumes:
       - ./wikidata:/import
   openmaptiles-tools:
-    image: "openmaptiles/openmaptiles-tools:0.9.2"
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
     env_file: .env
     networks:
       - postgres_conn
@@ -91,7 +95,7 @@ services:
       - .:/tileset
       - ./build:/sql
   generate-changed-vectortiles:
-    image: "openmaptiles/generate-vectortiles:0.1.1"
+    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
     command: ./export-list.sh
     volumes:
       - ./data:/export
@@ -100,7 +104,7 @@ services:
       - postgres_conn
     env_file: .env
   generate-vectortiles:
-    image: "openmaptiles/generate-vectortiles:0.1.1"
+    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
     volumes:
       - ./data:/export
       - ./build/openmaptiles.tm2source:/tm2source
@@ -108,20 +112,19 @@ services:
       - postgres_conn
     env_file: .env
     environment:
-      #BBOX: "8.25,46.97,9.58,47.52"
-      #BBOX: "5.97,45.75,10.7,47.72"
       BBOX: ${BBOX}
       MIN_ZOOM: ${MIN_ZOOM}
       MAX_ZOOM: ${MAX_ZOOM}
   postserve:
-    image: "openmaptiles/postserve:0.2"
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+    command: postserve openmaptiles.yaml --verbose
     env_file: .env
     networks:
       - postgres_conn
     ports:
-      - "8090:8080"
+      - "8090:8090"
     volumes:
-      - ./build/openmaptiles.tm2source:/mapping
+      - .:/tileset
 
 networks:
   postgres_conn:

qa/layer_freq.sh

@@ -13,7 +13,8 @@ do
 echo " "
 echo "## $layerid z$z  - freq"
 
-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
 
 SQLCODE=$(cat <<-END
 select $classvars , count(*) as _count_ from

qa/layer_numvar_analyze.sh

@@ -13,7 +13,8 @@ do
 echo " "
 echo "## $layerid z$z - $var "
 
-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
 
 SQLCODE=$(cat <<-END
 SELECT

qa/layer_toplength.sh

@@ -11,7 +11,8 @@ do
 echo " "
 echo "## $layerid z$z   max length ($classvar)"
 
-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
 
 SQLCODE=$(cat <<-END
 SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from

quickstart.sh

@@ -35,6 +35,8 @@ MIN_DOCKER_VER=1.12.3
 STARTTIME=$(date +%s)
 STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
 githash=$( git rev-parse HEAD )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/TOOLS_VERSION")}"
+export TOOLS_VERSION
 
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit,
 # as well as pass any other common parameters.
@@ -249,7 +251,7 @@ echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Start SQL postprocessing:  ./build/tileset.sql -> PostgreSQL "
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-sql "
-docker-compose run $DC_OPTS import-sql
+docker-compose run $DC_OPTS openmaptiles-tools import-sql
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -258,8 +260,8 @@ make psql-analyze
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Bring up postserve at localhost:8090/tiles/{z}/{x}/{y}.pbf"
+echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
-docker-compose up -d postserve
+docker-compose run $DC_OPTS openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -341,6 +343,10 @@ echo "It takes $((ENDTIME - STARTTIME)) seconds to complete"
 echo "We saved the log file to $log_file  ( for debugging ) You can compare with the travis log !"
 echo " "
 echo "Start experimenting! And check the QUICKSTART.MD file!"
+echo " "
+echo "*  Use   make start-postserve    to explore tile generation on request"
+echo "*  Use   make start-tileserver   to view pre-generated tiles"
+echo " "
 echo "Available help commands (make help)  "
 make help