From 132747d9b5d1f1c7b9f6c56792c316a03e4e0e67 Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan <yuriastrakhan@gmail.com>
Date: Thu, 12 Dec 2019 12:40:31 -0500
Subject: [PATCH] Use unified tools version 3.1.0 for all images (#707)
* Use unified tools version for all images
* do not start postserve as part of quickstart, but added a help message how to start it
* wait for SQL start with pgwait
---
Makefile | 24 ++++++++++++++---------
TOOLS_VERSION | 1 +
docker-compose.yml | 39 ++++++++++++++++++++------------------
qa/layer_freq.sh | 3 ++-
qa/layer_numvar_analyze.sh | 3 ++-
qa/layer_toplength.sh | 3 ++-
quickstart.sh | 12 +++++++++---
7 files changed, 52 insertions(+), 33 deletions(-)
create mode 100644 TOOLS_VERSION
diff --git a/Makefile b/Makefile
index 6e8287f6..c3a7d93c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,11 @@
# Options to run with docker and docker-compose - ensure the container is destroyed on exit
-DC_OPTS:=--rm
+DC_OPTS?=--rm
# container runs as the current user rather than root (so that created files are not root-owned)
-DC_USER_OPTS:=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+
+TOOLS_VERSION?=$(shell cat TOOLS_VERSION)
+export TOOLS_VERSION
.PHONY: all
all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
@@ -72,6 +75,8 @@ clean-docker:
.PHONY: db-start
db-start:
docker-compose up -d postgres
+ @echo "Wait for PostgreSQL to start..."
+ docker-compose run $(DC_OPTS) import-osm ./pgwait.sh
.PHONY: download-geofabrik
download-geofabrik:
@@ -96,12 +101,12 @@ import-osm: db-start all
.PHONY: import-sql
import-sql: db-start all
- docker-compose run $(DC_OPTS) import-sql
+ docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
.PHONY: import-osmsql
import-osmsql: db-start all
docker-compose run $(DC_OPTS) import-osm
- docker-compose run $(DC_OPTS) import-sql
+ docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
.PHONY: generate-tiles
generate-tiles: db-start all
@@ -136,11 +141,11 @@ start-tileserver:
docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl
.PHONY: start-postserve
-start-postserve:
+start-postserve: db-start
@echo " "
@echo "***********************************************************"
@echo "* "
- @echo "* Bring up postserve at localhost:8090/tiles/{z}/{x}/{y}.pbf"
+ @echo "* Bring up postserve at localhost:8090"
@echo "* "
@echo "***********************************************************"
@echo " "
@@ -150,7 +155,8 @@ start-postserve:
@echo "***********************************************************"
@echo "* "
@echo "* Start maputnik/editor "
- @echo "* ----------------------------> check localhost:8088 "
+ @echo "* ---> go to http://localhost:8088"
+ @echo "* ---> set 'data source' to http://localhost:8090"
@echo "* "
@echo "***********************************************************"
@echo " "
@@ -191,7 +197,7 @@ mapping-graph:
@echo 'Valid layers: $(mappingLayers)'
mapping-graph-%: ./layers/%/mapping.yaml build/devdoc
- docker run $(DC_USER_OPTS) -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
+ docker run $(DC_USER_OPTS) -v $$(pwd):/tileset openmaptiles/openmaptiles-tools:${TOOLS_VERSION} generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
cp ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png
# generate all etl and mapping graphs
@@ -199,7 +205,7 @@ generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(
.PHONY: import-sql-dev
import-sql-dev:
- docker-compose run $(DC_OPTS) import-sql /bin/bash
+ docker-compose run $(DC_OPTS) openmaptiles-tools bash
.PHONY: import-osm-dev
import-osm-dev:
diff --git a/TOOLS_VERSION b/TOOLS_VERSION
new file mode 100644
index 00000000..fd2a0186
--- /dev/null
+++ b/TOOLS_VERSION
@@ -0,0 +1 @@
+3.1.0
diff --git a/docker-compose.yml b/docker-compose.yml
index 02927a3a..e13d8829 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,7 +4,7 @@ volumes:
cache:
services:
postgres:
- image: "openmaptiles/postgis:2.9"
+ image: "openmaptiles/postgis:${TOOLS_VERSION}"
volumes:
- pgdata:/var/lib/postgresql/data
networks:
@@ -13,22 +13,22 @@ services:
- "5432"
env_file: .env
import-natural-earth:
- image: "openmaptiles/import-natural-earth:1.4"
+ image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
import-water:
- image: "openmaptiles/import-water:1.2"
+ image: "openmaptiles/import-water:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
import-lakelines:
- image: "openmaptiles/import-lakelines:1.0"
+ image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
import-osm:
- image: "openmaptiles/import-osm:0.5"
+ image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env
environment:
DIFF_MODE: ${DIFF_MODE}
@@ -39,12 +39,12 @@ services:
- ./build:/mapping
- cache:/cache
import-osmborder:
- image: "openmaptiles/import-osmborder:0.4"
+ image: "openmaptiles/import-osmborder:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
import-osm-diff:
- image: "openmaptiles/import-osm:0.5"
+ image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env
command: ./import_diff.sh
environment:
@@ -56,7 +56,7 @@ services:
- ./build:/mapping
- cache:/cache
update-osm:
- image: "openmaptiles/import-osm:0.5"
+ image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env
environment:
DIFF_MODE: ${DIFF_MODE}
@@ -68,14 +68,18 @@ services:
- ./build:/mapping
- cache:/cache
import-sql:
- image: "openmaptiles/import-sql:1.0"
+ # This target is obsolete, and was left for backwards compatibility
+ # Use openmaptiles-tools target instead
+ image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+ command: import-sql
env_file: .env
networks:
- postgres_conn
volumes:
+ - .:/tileset
- ./build:/sql
import-wikidata:
- image: "openmaptiles/import-wikidata:0.1"
+ image: "openmaptiles/import-wikidata:${TOOLS_VERSION}"
env_file: .env
command: import-wikidata
networks:
@@ -83,7 +87,7 @@ services:
volumes:
- ./wikidata:/import
openmaptiles-tools:
- image: "openmaptiles/openmaptiles-tools:0.9.2"
+ image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
@@ -91,7 +95,7 @@ services:
- .:/tileset
- ./build:/sql
generate-changed-vectortiles:
- image: "openmaptiles/generate-vectortiles:0.1.1"
+ image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
command: ./export-list.sh
volumes:
- ./data:/export
@@ -100,7 +104,7 @@ services:
- postgres_conn
env_file: .env
generate-vectortiles:
- image: "openmaptiles/generate-vectortiles:0.1.1"
+ image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
volumes:
- ./data:/export
- ./build/openmaptiles.tm2source:/tm2source
@@ -108,20 +112,19 @@ services:
- postgres_conn
env_file: .env
environment:
- #BBOX: "8.25,46.97,9.58,47.52"
- #BBOX: "5.97,45.75,10.7,47.72"
BBOX: ${BBOX}
MIN_ZOOM: ${MIN_ZOOM}
MAX_ZOOM: ${MAX_ZOOM}
postserve:
- image: "openmaptiles/postserve:0.2"
+ image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+ command: postserve openmaptiles.yaml --verbose
env_file: .env
networks:
- postgres_conn
ports:
- - "8090:8080"
+ - "8090:8090"
volumes:
- - ./build/openmaptiles.tm2source:/mapping
+ - .:/tileset
networks:
postgres_conn:
diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh
index 4c9dd91d..aa32074e 100755
--- a/qa/layer_freq.sh
+++ b/qa/layer_freq.sh
@@ -13,7 +13,8 @@ do
echo " "
echo "## $layerid z$z - freq"
-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
select $classvars , count(*) as _count_ from
diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh
index b6ea7b71..7ed5aaf6 100755
--- a/qa/layer_numvar_analyze.sh
+++ b/qa/layer_numvar_analyze.sh
@@ -13,7 +13,8 @@ do
echo " "
echo "## $layerid z$z - $var "
-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
SELECT
diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh
index a43a93b9..a4d14319 100755
--- a/qa/layer_toplength.sh
+++ b/qa/layer_toplength.sh
@@ -11,7 +11,8 @@ do
echo " "
echo "## $layerid z$z max length ($classvar)"
-SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/../TOOLS_VERSION")}"
+SQL=$(docker run --rm -v "$(pwd):/tileset" "openmaptiles/openmaptiles-tools:$TOOLS_VERSION" generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from
diff --git a/quickstart.sh b/quickstart.sh
index 874fb990..1a64a72a 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -35,6 +35,8 @@ MIN_DOCKER_VER=1.12.3
STARTTIME=$(date +%s)
STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
githash=$( git rev-parse HEAD )
+: "${TOOLS_VERSION:=$(cat "$(dirname "$0")/TOOLS_VERSION")}"
+export TOOLS_VERSION
# Options to run with docker and docker-compose - ensure the container is destroyed on exit,
# as well as pass any other common parameters.
@@ -249,7 +251,7 @@ echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-sql "
-docker-compose run $DC_OPTS import-sql
+docker-compose run $DC_OPTS openmaptiles-tools import-sql
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -258,8 +260,8 @@ make psql-analyze
echo " "
echo "-------------------------------------------------------------------------------------"
-echo "====> : Bring up postserve at localhost:8090/tiles/{z}/{x}/{y}.pbf"
-docker-compose up -d postserve
+echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
+docker-compose run $DC_OPTS openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -341,6 +343,10 @@ echo "It takes $((ENDTIME - STARTTIME)) seconds to complete"
echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !"
echo " "
echo "Start experimenting! And check the QUICKSTART.MD file!"
+echo " "
+echo "* Use make start-postserve to explore tile generation on request"
+echo "* Use make start-tileserver to view pre-generated tiles"
+echo " "
echo "Available help commands (make help) "
make help