Upgrade to tools v5 - rm import-osm, new downloader... (#785)

Update to tools v5. See https://github.com/openmaptiles/openmaptiles-tools/releases/tag/v5.0.0 for the list of all changes. Other OMT-repo specific changes:

* removes `import-osm` docker usage, replacing it with `openmaptiles-tools`
* quickstart builds faster because it uses postgres with preloaded water, natural earth, and lake centerlines tables.

### Makefile targets
* `tools-dev` will open a shell in a docker to experiment and debug (instead of `import-sql-dev` and `import-osm-dev`)
* separated `start-maputnik` from `start-postserve`
* renamed `clean-docker` into `db-destroy` to make it more explicit
* cleaner `db-start`, `db-stop`, `db-destroy` targets
* `db-start-preloaded` is the same as `db-start`, except that it uses `postgis-preloaded` -- an image with preloaded water, natural-earth, and lake centerline data
* `db-start` will not recreate the container if it already exists -- this way if it was started as preloaded, it will not be rebuilt.
* better output messages

### Quickstart
* uses `postgis-preloaded` image by default to make quickstart quicker.  To start with a clean db, pass 2 parameters to quickstart, e.g. `./quickstart.sh albania empty`

.env

@@ -1,12 +1,14 @@
 # This file defines default environment variables for all images
 
-TOOLS_VERSION=4.1.0
+# Use 3-part patch version to ignore patch updates, e.g. 5.0.0
+TOOLS_VERSION=5.0
 
-POSTGRES_DB=openmaptiles
+# Make sure these values are in sync with the ones in .env-postgres file
-POSTGRES_USER=openmaptiles
+PGDATABASE=openmaptiles
-POSTGRES_PASSWORD=openmaptiles
+PGUSER=openmaptiles
-POSTGRES_HOST=postgres
+PGPASSWORD=openmaptiles
-POSTGRES_PORT=5432
+PGHOST=postgres
+PGPORT=5432
 
 QUICKSTART_MIN_ZOOM=0
 QUICKSTART_MAX_ZOOM=7

.env-postgres

@@ -0,0 +1,8 @@
+# This file defines environment variables for the PostgreSQL image.
+# The main docker PostgreSQL image requires these vars rather than
+# the standard PG* ones that all PostgreSQL tools use.
+
+# Make sure these values are in sync with the ones in .env file
+POSTGRES_DB=openmaptiles
+POSTGRES_USER=openmaptiles
+POSTGRES_PASSWORD=openmaptiles

Makefile

@@ -6,6 +6,9 @@ SHELL         = /bin/bash
 # Containers run as the current user rather than root (so that created files are not root-owned)
 DC_OPTS?=--rm -u $(shell id -u):$(shell id -g)
 
+# If set to a non-empty value, will use postgis-preloaded instead of postgis docker image
+USE_PRELOADED_IMAGE?=
+
 # Allow a custom docker-compose project name
 ifeq ($(strip $(DC_PROJECT)),)
   override DC_PROJECT:=$(notdir $(shell pwd))
@@ -68,9 +71,8 @@ help:
 	@echo "  make tools-dev                       # start openmaptiles-tools /bin/bash terminal"
 	@echo "  make db-destroy                      # remove docker containers and PostgreSQL data volume"
 	@echo "  make db-start                        # start PostgreSQL, creating it if it doesn't exist"
+	@echo "  make db-start-preloaded              # start PostgreSQL, creating data-prepopulated one if it doesn't exist"
 	@echo "  make db-stop                         # stop PostgreSQL database without destroying the data"
-	@echo "  make import-sql-dev                  # start import-sql /bin/bash terminal"
-	@echo "  make import-osm-dev                  # start import-osm /bin/bash terminal (imposm3)"
 	@echo "  make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)"
 	@echo "  make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB"
 	@echo "  make remove-docker-images            # remove openmaptiles docker images"
@@ -110,11 +112,20 @@ db-destroy:
 	docker volume ls -q -f "name=^$(DC_PROJECT)_" | $(XARGS) docker volume rm
 	rm -rf cache
 
-.PHONY: db-start
+.PHONY: db-start-nowait
-db-start:
+db-start-nowait:
+	@echo "Starting postgres docker compose target using $${POSTGIS_IMAGE:-default} image (no recreate if exists)" && \
 	$(DOCKER_COMPOSE) up --no-recreate -d postgres
+
+.PHONY: db-start
+db-start: db-start-nowait
 	@echo "Wait for PostgreSQL to start..."
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools pgwait
+
+# Wrap db-start target but use the preloaded image
+.PHONY: db-start-preloaded
+db-start-preloaded: export POSTGIS_IMAGE=openmaptiles/postgis-preloaded
+db-start-preloaded: db-start
 
 .PHONY: db-stop
 db-stop:
@@ -151,34 +162,39 @@ else
 endif
 
 .PHONY: psql
-psql: db-start
+psql: db-start-nowait
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && psql.sh'
 
 .PHONY: import-osm
-import-osm: db-start all
+import-osm: all db-start-nowait
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm'
 
-.PHONY: import-osmsql
+.PHONY: update-osm
-import-osmsql: db-start all import-osm import-sql
+update-osm: all db-start-nowait
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-update'
+
+.PHONY: import-diff
+import-diff: all db-start-nowait
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-diff'
 
 .PHONY: import-data
 import-data: db-start
 	$(DOCKER_COMPOSE) run $(DC_OPTS) import-data
 
 .PHONY: import-borders
-import-borders: db-start
+import-borders: db-start-nowait
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-borders'
 
 .PHONY: import-sql
-import-sql: db-start all
+import-sql: all db-start-nowait
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql | \
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
 	  awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1'
 
 .PHONY: generate-tiles
 ifneq ($(wildcard data/docker-compose-config.yml),)
   DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
 endif
-generate-tiles: init-dirs db-start all
+generate-tiles: init-dirs all db-start
 	rm -rf data/tiles.mbtiles
 	echo "Generating tiles ..."; \
 	$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
@@ -258,49 +274,37 @@ generate-devdoc: init-dirs
 tools-dev:
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash
 
-.PHONY: import-osm-dev
-import-osm-dev:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm /bin/bash
-
 .PHONY: import-wikidata
 import-wikidata:
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml
 
 .PHONY: psql-pg-stat-reset
 psql-pg-stat-reset:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();'
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();'
-
-.PHONY: forced-clean-sql
-forced-clean-sql:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \
-		-c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \
-		-c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \
-		-c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \
-		-c "GRANT ALL ON SCHEMA public TO public; COMMENT ON SCHEMA public IS 'standard public schema';"
 
 .PHONY: list-views
 list-views:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
 		-c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;"
 
 .PHONY: list-tables
 list-tables:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
 		-c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;"
 
 .PHONY: psql-list-tables
 psql-list-tables:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+"
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+"
 
 .PHONY: psql-vacuum-analyze
 psql-vacuum-analyze:
 	@echo "Start - postgresql: VACUUM ANALYZE VERBOSE;"
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;'
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;'
 
 .PHONY: psql-analyze
 psql-analyze:
 	@echo "Start - postgresql: ANALYZE VERBOSE;"
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;'
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;'
 
 .PHONY: list-docker-images
 list-docker-images:
@@ -313,7 +317,12 @@ ifneq ($(strip $(NO_REFRESH)),)
 else
 	@echo ""
 	@echo "Refreshing docker images... Use NO_REFRESH=1 to skip."
-	$(DOCKER_COMPOSE) pull --ignore-pull-failures $(QUIET_FLAG)
+ifneq ($(strip $(USE_PRELOADED_IMAGE)),)
+	POSTGIS_IMAGE=openmaptiles/postgis-preloaded \
+		docker-compose pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres
+else
+	docker-compose pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres import-data
+endif
 endif
 
 .PHONY: remove-docker-images

QUICKSTART.md

@@ -426,10 +426,8 @@ Hints for developers:
   make psql-analyze                    # PostgreSQL: ANALYZE
   make generate-qareports              # generate reports [./build/qareports]
   make generate-devdoc                 # generate devdoc  [./build/devdoc]
-  make import-sql-dev                  # start import-sql  /bin/bash terminal
+  make tools-dev                       # start import-sql  /bin/bash terminal
-  make import-osm-dev                  # start import-osm  /bin/bash terminal (imposm3)
+  make db-destroy                      # remove docker containers, PG data volume
-  make clean-docker                    # remove docker containers, PG data volume
-  make forced-clean-sql                # drop all PostgreSQL tables for clean environment
   make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)
   make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB
   make remove-docker-images            # remove openmaptiles docker images

UPDATE.md

@@ -15,7 +15,7 @@ the OSM change feed and import it into the database.
 After each run you should also have a list of tiles that have updated.
 
 ```
-docker-compose run update-osm
+make update-osm
 ```
 
 ### Import Change File
@@ -23,7 +23,7 @@ docker-compose run update-osm
 Given you have a file `changes.osc.gz` in your import folder. Once you ran the import command you should also have a list of tiles that have updated.
 
 ```
-docker-compose run import-osm-diff
+make import-diff
 ```
 
 ## Generate Changed Tiles

docker-compose.yml

@@ -12,13 +12,14 @@ services:
 
   postgres:
     image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
+    # Use "command: postgres -c jit=off" for PostgreSQL 11+ because of slow large MVT query processing
     volumes:
       - pgdata:/var/lib/postgresql/data
     networks:
       - postgres_conn
     ports:
       - "5432"
-    env_file: .env
+    env_file: .env-postgres
 
   import-data:
     image: "openmaptiles/import-data:${TOOLS_VERSION}"
@@ -26,44 +27,6 @@ services:
     networks:
       - postgres_conn
 
-  import-osm:
-    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
-    env_file: .env
-    environment:
-      DIFF_MODE: ${DIFF_MODE}
-    networks:
-      - postgres_conn
-    volumes:
-      - ./data:/import
-      - ./build:/mapping
-      - ./cache:/cache
-
-  import-osm-diff:
-    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
-    env_file: .env
-    command: ./import_diff.sh
-    environment:
-      DIFF_MODE: ${DIFF_MODE}
-    networks:
-      - postgres_conn
-    volumes:
-      - ./data:/import
-      - ./build:/mapping
-      - ./cache:/cache
-
-  update-osm:
-    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
-    env_file: .env
-    environment:
-      DIFF_MODE: ${DIFF_MODE}
-    command: ./import_update.sh
-    networks:
-      - postgres_conn
-    volumes:
-      - ./data:/import
-      - ./build:/mapping
-      - ./cache:/cache
-
   openmaptiles-tools:
     image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
     env_file: .env

layers/building/building.sql

@@ -102,8 +102,8 @@ RETURNS TABLE(geometry geometry, osm_id bigint, render_height int, render_min_he
         -- etldoc: osm_building_polygon -> layer_building:z14_
         SELECT DISTINCT ON (osm_id)
            osm_id, geometry,
-           ceil( COALESCE(height, levels*3.66,5))::int AS render_height,
+           ceil(COALESCE(height, levels*3.66, 5))::int AS render_height,
-           floor(COALESCE(min_height, min_level*3.66,0))::int AS render_min_height,
+           floor(COALESCE(min_height, min_level*3.66, 0))::int AS render_min_height,
            material,
            colour,
            hide_3d

quickstart.sh

@@ -66,6 +66,11 @@ if [ "$(version "$DOCKER_VER")" -lt "$(version "$MIN_DOCKER_VER")" ]; then
   exit 1
 fi
 
+# If there are no arguments, or just the area is set, use preloaded docker image to speed up
+# to force all steps, use two arguments, e.g. "./quickstart monaco empty"
+(( $# == 0 || $# == 1 )) && USE_PRELOADED_IMAGE=true || USE_PRELOADED_IMAGE=""
+export USE_PRELOADED_IMAGE
+
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Pulling or refreshing OpenMapTiles docker images "
@@ -82,6 +87,7 @@ echo "--------------------------------------------------------------------------
 echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ]    "
 echo "      : This will be logged to the $log_file file (for debugging) and to the screen"
 echo "      : Area             : $osm_area "
+echo "      : Preloaded Image  : $USE_PRELOADED_IMAGE "
 echo "      : Git version      : $githash "
 echo "      : Started          : $STARTDATE "
 echo "      : Your bash version: $BASH_VERSION"
@@ -166,30 +172,42 @@ make all
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Start PostgreSQL service ; create PostgreSQL data volume "
+if [[ "$USE_PRELOADED_IMAGE" == true ]]; then
-echo "      : Source code: https://github.com/openmaptiles/postgis "
+  echo "====> : Start PostgreSQL service using postgis image preloaded with this data:"
-echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
+  echo "      : * Water data from http://osmdata.openstreetmap.de"
-make db-start
+  echo "      :   Data license: https://osmdata.openstreetmap.de/info/license.html"
-
+  echo "      : * Natural Earth from http://www.naturalearthdata.com"
-echo " "
+  echo "      :   Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
-echo "-------------------------------------------------------------------------------------"
+  echo "      : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
-echo "====> : Drop and Recreate PostgreSQL  public schema "
+  echo "      :"
-# Drop all PostgreSQL tables
+  echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
-# This adds an extra safety belt if the user modifies the docker volume settings
+  echo "      :   includes all data from the import-data image"
-make forced-clean-sql
+  echo "      :"
-
+  echo "      : Use two-parameter quickstart to start with an empty database:"
-echo "====> : Importing all the data:"
+  echo "      :   ./quickstart.sh albania empty"
-echo "      : * Water data from http://osmdata.openstreetmap.de"
+  echo "      : If desired, you can manually import data by one using these commands:"
-echo "      :   Data license: https://osmdata.openstreetmap.de/info/license.html"
+  echo "      :   make db-destroy"
-echo "      : * Natural Earth from http://www.naturalearthdata.com"
+  echo "      :   make db-start"
-echo "      :   Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
+  echo "      :   make import-data"
-echo "      : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
+  echo "      :"
-echo "      :"
+  echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/postgis-preloaded"
-echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
+  echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
-echo "      :   includes all data from the import-data image"
+  make db-start-preloaded
-echo "      :"
+else
-echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
+  echo "====> : Start PostgreSQL service using empty database and importing all the data:"
-make import-data
+  echo "      : * Water data from http://osmdata.openstreetmap.de"
+  echo "      :   Data license: https://osmdata.openstreetmap.de/info/license.html"
+  echo "      : * Natural Earth from http://www.naturalearthdata.com"
+  echo "      :   Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
+  echo "      : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
+  echo "      :"
+  echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
+  echo "      :   includes all data from the import-data image"
+  echo "      :"
+  echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
+  make db-start
+  make import-data
+fi
 
 echo " "
 echo "-------------------------------------------------------------------------------------"