Use unified tools version 3.1.0 for all images (#707)

* Use unified tools version for all images * do not start postserve as part of quickstart, but added a help message how to start it * wait for SQL start with pgwait
2019-12-12 12:40:31 -05:00 · 2019-12-12 12:40:31 -05:00 · 132747d9b5
commit 132747d9b5
--- a/24
+++ b/24
@ -1,8 +1,11 @@
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit
-DC_OPTS:=--rm
+DC_OPTS?=--rm

 # container runs as the current user rather than root (so that created files are not root-owned)
-DC_USER_OPTS:=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+
+TOOLS_VERSION?=$(shell cat TOOLS_VERSION)
+export TOOLS_VERSION

 .PHONY: all
 all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
@ -72,6 +75,8 @@ clean-docker:
 .PHONY: db-start
 db-start:
 	docker-compose up -d postgres
+	@echo "Wait for PostgreSQL to start..."
+	docker-compose run $(DC_OPTS) import-osm  ./pgwait.sh

 .PHONY: download-geofabrik
 download-geofabrik:
@ -96,12 +101,12 @@ import-osm: db-start all

 .PHONY: import-sql
 import-sql: db-start all
-	docker-compose run $(DC_OPTS) import-sql
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql

 .PHONY: import-osmsql
 import-osmsql: db-start all
 	docker-compose run $(DC_OPTS) import-osm
-	docker-compose run $(DC_OPTS) import-sql
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql

 .PHONY: generate-tiles
 generate-tiles: db-start all
@ -136,11 +141,11 @@ start-tileserver:
 	docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl

 .PHONY: start-postserve
-start-postserve:
+start-postserve: db-start
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
-	@echo "* Bring up postserve at localhost:8090/tiles/{z}/{x}/{y}.pbf"
+	@echo "* Bring up postserve at localhost:8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
@ -150,7 +155,8 @@ start-postserve:
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Start maputnik/editor "
-	@echo "*       ----------------------------> check localhost:8088 "
+	@echo "*       ---> go to http://localhost:8088"
+	@echo "*       ---> set 'data source' to  http://localhost:8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
@ -191,7 +197,7 @@ mapping-graph:
 	@echo 'Valid layers: $(mappingLayers)'

 mapping-graph-%: ./layers/%/mapping.yaml build/devdoc
-	docker run $(DC_USER_OPTS) -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
+	docker run $(DC_USER_OPTS) -v $$(pwd):/tileset openmaptiles/openmaptiles-tools:${TOOLS_VERSION} generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
 	cp ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png

 # generate all etl and mapping graphs
@ -199,7 +205,7 @@ generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(

 .PHONY: import-sql-dev
 import-sql-dev:
-	docker-compose run $(DC_OPTS) import-sql /bin/bash
+	docker-compose run $(DC_OPTS) openmaptiles-tools bash

 .PHONY: import-osm-dev
 import-osm-dev:
--- a/1
+++ b/1
@ -0,0 +1 @@
+3.1.0
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -4,7 +4,7 @@ volumes:
  cache:
 services:
  postgres:
-    image: "openmaptiles/postgis:2.9"
+    image: "openmaptiles/postgis:${TOOLS_VERSION}"
    volumes:
      - pgdata:/var/lib/postgresql/data
    networks:
@ -13,22 +13,22 @@ services:
      - "5432"
    env_file: .env
  import-natural-earth:
-    image: "openmaptiles/import-natural-earth:1.4"
+    image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
    env_file: .env
    networks:
      - postgres_conn
  import-water:
-    image: "openmaptiles/import-water:1.2"
+    image: "openmaptiles/import-water:${TOOLS_VERSION}"
    env_file: .env
    networks:
      - postgres_conn
  import-lakelines:
-    image: "openmaptiles/import-lakelines:1.0"
+    image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
    env_file: .env
    networks:
      - postgres_conn
  import-osm:
-    image: "openmaptiles/import-osm:0.5"
+    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
    environment:
      DIFF_MODE: ${DIFF_MODE}
@ -39,12 +39,12 @@ services:
      - ./build:/mapping
      - cache:/cache
  import-osmborder:
-    image: "openmaptiles/import-osmborder:0.4"
+    image: "openmaptiles/import-osmborder:${TOOLS_VERSION}"
    env_file: .env
    networks:
      - postgres_conn
  import-osm-diff:
-    image: "openmaptiles/import-osm:0.5"
+    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
    command: ./import_diff.sh
    environment:
@ -56,7 +56,7 @@ services:
      - ./build:/mapping
      - cache:/cache
  update-osm:
-    image: "openmaptiles/import-osm:0.5"
+    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
    environment:
      DIFF_MODE: ${DIFF_MODE}
@ -68,14 +68,18 @@ services:
      - ./build:/mapping
      - cache:/cache
  import-sql:
-    image: "openmaptiles/import-sql:1.0"
+    # This target is obsolete, and was left for backwards compatibility
+    # Use openmaptiles-tools target instead
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+    command: import-sql
    env_file: .env
    networks:
      - postgres_conn
    volumes:
+      - .:/tileset
      - ./build:/sql
  import-wikidata:
-    image: "openmaptiles/import-wikidata:0.1"
+    image: "openmaptiles/import-wikidata:${TOOLS_VERSION}"
    env_file: .env
    command: import-wikidata
    networks:
@ -83,7 +87,7 @@ services:
    volumes:
      - ./wikidata:/import
  openmaptiles-tools:
-    image: "openmaptiles/openmaptiles-tools:0.9.2"
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
    env_file: .env
    networks:
      - postgres_conn
@ -91,7 +95,7 @@ services:
      - .:/tileset
      - ./build:/sql
  generate-changed-vectortiles:
-    image: "openmaptiles/generate-vectortiles:0.1.1"
+    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
    command: ./export-list.sh
    volumes:
      - ./data:/export
@ -100,7 +104,7 @@ services:
      - postgres_conn
    env_file: .env
  generate-vectortiles:
-    image: "openmaptiles/generate-vectortiles:0.1.1"
+    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
    volumes:
      - ./data:/export
      - ./build/openmaptiles.tm2source:/tm2source
@ -108,20 +112,19 @@ services:
      - postgres_conn
    env_file: .env
    environment:
-      #BBOX: "8.25,46.97,9.58,47.52"
-      #BBOX: "5.97,45.75,10.7,47.72"
      BBOX: ${BBOX}
      MIN_ZOOM: ${MIN_ZOOM}
      MAX_ZOOM: ${MAX_ZOOM}
  postserve:
-    image: "openmaptiles/postserve:0.2"
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+    command: postserve openmaptiles.yaml --verbose
    env_file: .env
    networks:
      - postgres_conn
    ports:
-      - "8090:8080"
+      - "8090:8090"
    volumes:
-      - ./build/openmaptiles.tm2source:/mapping
+      - .:/tileset

 networks:
  postgres_conn:
--- a/qa/layer_freq.sh
+++ b/qa/layer_freq.sh
@ -13,7 +13,8 @@ do
 echo " "
 echo "## $layerid z$z  - freq"

-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )

 SQLCODE=$(cat <<-END
 select $classvars , count(*) as _count_ from
--- a/qa/layer_numvar_analyze.sh
+++ b/qa/layer_numvar_analyze.sh
@ -13,7 +13,8 @@ do
 echo " "
 echo "## $layerid z$z - $var "

-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )

 SQLCODE=$(cat <<-END
 SELECT
--- a/qa/layer_toplength.sh
+++ b/qa/layer_toplength.sh
@ -11,7 +11,8 @@ do
 echo " "
 echo "## $layerid z$z   max length ($classvar)"

-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery  layers/${layerid}/${layerid}.yaml $z )

 SQLCODE=$(cat <<-END
 SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from
--- a/quickstart.sh
+++ b/quickstart.sh
@ -35,6 +35,8 @@ MIN_DOCKER_VER=1.12.3
 STARTTIME=$(date +%s)
 STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
 githash=$( git rev-parse HEAD )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/TOOLS_VERSION")}"
+export TOOLS_VERSION

 # Options to run with docker and docker-compose - ensure the container is destroyed on exit,
 # as well as pass any other common parameters.
@ -249,7 +251,7 @@ echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Start SQL postprocessing:  ./build/tileset.sql -> PostgreSQL "
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-sql "
-docker-compose run $DC_OPTS import-sql
+docker-compose run $DC_OPTS openmaptiles-tools import-sql

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -258,8 +260,8 @@ make psql-analyze

 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Bring up postserve at localhost:8090/tiles/{z}/{x}/{y}.pbf"
-docker-compose up -d postserve
+echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
+docker-compose run $DC_OPTS openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -341,6 +343,10 @@ echo "It takes $((ENDTIME - STARTTIME)) seconds to complete"
 echo "We saved the log file to $log_file  ( for debugging ) You can compare with the travis log !"
 echo " "
 echo "Start experimenting! And check the QUICKSTART.MD file!"
+echo " "
+echo "*  Use   make start-postserve    to explore tile generation on request"
+echo "*  Use   make start-tileserver   to view pre-generated tiles"
+echo " "
 echo "Available help commands (make help)  "
 make help