diff --git a/.env b/.env index 70d706a0..c92ecc79 100644 --- a/.env +++ b/.env @@ -1,12 +1,14 @@ # This file defines default environment variables for all images -TOOLS_VERSION=4.1.0 +# Use 3-part patch version to ignore patch updates, e.g. 5.0.0 +TOOLS_VERSION=5.0 -POSTGRES_DB=openmaptiles -POSTGRES_USER=openmaptiles -POSTGRES_PASSWORD=openmaptiles -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 +# Make sure these values are in sync with the ones in .env-postgres file +PGDATABASE=openmaptiles +PGUSER=openmaptiles +PGPASSWORD=openmaptiles +PGHOST=postgres +PGPORT=5432 QUICKSTART_MIN_ZOOM=0 QUICKSTART_MAX_ZOOM=7 diff --git a/.env-postgres b/.env-postgres new file mode 100644 index 00000000..692e16ed --- /dev/null +++ b/.env-postgres @@ -0,0 +1,8 @@ +# This file defines environment variables for the PostgreSQL image. +# The main docker PostgreSQL image requires these vars rather than +# the standard PG* ones that all PostgreSQL tools use. + +# Make sure these values are in sync with the ones in .env file +POSTGRES_DB=openmaptiles +POSTGRES_USER=openmaptiles +POSTGRES_PASSWORD=openmaptiles diff --git a/Makefile b/Makefile index 47f07465..0e685c2c 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ SHELL = /bin/bash # Containers run as the current user rather than root (so that created files are not root-owned) DC_OPTS?=--rm -u $(shell id -u):$(shell id -g) +# If set to a non-empty value, will use postgis-preloaded instead of postgis docker image +USE_PRELOADED_IMAGE?= + # Allow a custom docker-compose project name ifeq ($(strip $(DC_PROJECT)),) override DC_PROJECT:=$(notdir $(shell pwd)) @@ -68,9 +71,8 @@ help: @echo " make tools-dev # start openmaptiles-tools /bin/bash terminal" @echo " make db-destroy # remove docker containers and PostgreSQL data volume" @echo " make db-start # start PostgreSQL, creating it if it doesn't exist" + @echo " make db-start-preloaded # start PostgreSQL, creating data-prepopulated one if it doesn't exist" @echo " make db-stop # stop PostgreSQL database without destroying the data" - @echo " make import-sql-dev # start import-sql /bin/bash terminal" - @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)" @echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" @echo " make remove-docker-images # remove openmaptiles docker images" @@ -110,11 +112,20 @@ db-destroy: docker volume ls -q -f "name=^$(DC_PROJECT)_" | $(XARGS) docker volume rm rm -rf cache -.PHONY: db-start -db-start: +.PHONY: db-start-nowait +db-start-nowait: + @echo "Starting postgres docker compose target using $${POSTGIS_IMAGE:-default} image (no recreate if exists)" && \ $(DOCKER_COMPOSE) up --no-recreate -d postgres + +.PHONY: db-start +db-start: db-start-nowait @echo "Wait for PostgreSQL to start..." - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools pgwait + +# Wrap db-start target but use the preloaded image +.PHONY: db-start-preloaded +db-start-preloaded: export POSTGIS_IMAGE=openmaptiles/postgis-preloaded +db-start-preloaded: db-start .PHONY: db-stop db-stop: @@ -151,34 +162,39 @@ else endif .PHONY: psql -psql: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh +psql: db-start-nowait + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && psql.sh' .PHONY: import-osm -import-osm: db-start all - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm +import-osm: all db-start-nowait + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm' -.PHONY: import-osmsql -import-osmsql: db-start all import-osm import-sql +.PHONY: update-osm +update-osm: all db-start-nowait + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-update' + +.PHONY: import-diff +import-diff: all db-start-nowait + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-diff' .PHONY: import-data import-data: db-start $(DOCKER_COMPOSE) run $(DC_OPTS) import-data .PHONY: import-borders -import-borders: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders +import-borders: db-start-nowait + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-borders' .PHONY: import-sql -import-sql: db-start all - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql | \ +import-sql: all db-start-nowait + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \ awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1' .PHONY: generate-tiles ifneq ($(wildcard data/docker-compose-config.yml),) DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml endif -generate-tiles: init-dirs db-start all +generate-tiles: init-dirs all db-start rm -rf data/tiles.mbtiles echo "Generating tiles ..."; \ $(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles @@ -258,49 +274,37 @@ generate-devdoc: init-dirs tools-dev: $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -.PHONY: import-osm-dev -import-osm-dev: - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm /bin/bash - .PHONY: import-wikidata import-wikidata: $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml .PHONY: psql-pg-stat-reset psql-pg-stat-reset: - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();' - -.PHONY: forced-clean-sql -forced-clean-sql: - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \ - -c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \ - -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \ - -c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \ - -c "GRANT ALL ON SCHEMA public TO public; COMMENT ON SCHEMA public IS 'standard public schema';" + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();' .PHONY: list-views list-views: - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ -c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;" .PHONY: list-tables list-tables: - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ -c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;" .PHONY: psql-list-tables psql-list-tables: - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+" + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+" .PHONY: psql-vacuum-analyze psql-vacuum-analyze: @echo "Start - postgresql: VACUUM ANALYZE VERBOSE;" - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;' + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;' .PHONY: psql-analyze psql-analyze: @echo "Start - postgresql: ANALYZE VERBOSE;" - $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;' + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;' .PHONY: list-docker-images list-docker-images: @@ -313,7 +317,12 @@ ifneq ($(strip $(NO_REFRESH)),) else @echo "" @echo "Refreshing docker images... Use NO_REFRESH=1 to skip." - $(DOCKER_COMPOSE) pull --ignore-pull-failures $(QUIET_FLAG) +ifneq ($(strip $(USE_PRELOADED_IMAGE)),) + POSTGIS_IMAGE=openmaptiles/postgis-preloaded \ + docker-compose pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres +else + docker-compose pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres import-data +endif endif .PHONY: remove-docker-images diff --git a/QUICKSTART.md b/QUICKSTART.md index 5f2e291b..336d1e16 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -426,10 +426,8 @@ Hints for developers: make psql-analyze # PostgreSQL: ANALYZE make generate-qareports # generate reports [./build/qareports] make generate-devdoc # generate devdoc [./build/devdoc] - make import-sql-dev # start import-sql /bin/bash terminal - make import-osm-dev # start import-osm /bin/bash terminal (imposm3) - make clean-docker # remove docker containers, PG data volume - make forced-clean-sql # drop all PostgreSQL tables for clean environment + make tools-dev # start import-sql /bin/bash terminal + make db-destroy # remove docker containers, PG data volume make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s) make refresh-docker-images # refresh openmaptiles docker images from Docker HUB make remove-docker-images # remove openmaptiles docker images diff --git a/UPDATE.md b/UPDATE.md index f107c1ab..b78b788c 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -15,7 +15,7 @@ the OSM change feed and import it into the database. After each run you should also have a list of tiles that have updated. ``` -docker-compose run update-osm +make update-osm ``` ### Import Change File @@ -23,7 +23,7 @@ docker-compose run update-osm Given you have a file `changes.osc.gz` in your import folder. Once you ran the import command you should also have a list of tiles that have updated. ``` -docker-compose run import-osm-diff +make import-diff ``` ## Generate Changed Tiles diff --git a/docker-compose.yml b/docker-compose.yml index 75c479f1..1e92f1f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,13 +12,14 @@ services: postgres: image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}" + # Use "command: postgres -c jit=off" for PostgreSQL 11+ because of slow large MVT query processing volumes: - pgdata:/var/lib/postgresql/data networks: - postgres_conn ports: - "5432" - env_file: .env + env_file: .env-postgres import-data: image: "openmaptiles/import-data:${TOOLS_VERSION}" @@ -26,44 +27,6 @@ services: networks: - postgres_conn - import-osm: - image: "openmaptiles/import-osm:${TOOLS_VERSION}" - env_file: .env - environment: - DIFF_MODE: ${DIFF_MODE} - networks: - - postgres_conn - volumes: - - ./data:/import - - ./build:/mapping - - ./cache:/cache - - import-osm-diff: - image: "openmaptiles/import-osm:${TOOLS_VERSION}" - env_file: .env - command: ./import_diff.sh - environment: - DIFF_MODE: ${DIFF_MODE} - networks: - - postgres_conn - volumes: - - ./data:/import - - ./build:/mapping - - ./cache:/cache - - update-osm: - image: "openmaptiles/import-osm:${TOOLS_VERSION}" - env_file: .env - environment: - DIFF_MODE: ${DIFF_MODE} - command: ./import_update.sh - networks: - - postgres_conn - volumes: - - ./data:/import - - ./build:/mapping - - ./cache:/cache - openmaptiles-tools: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" env_file: .env diff --git a/layers/building/building.sql b/layers/building/building.sql index 0e0685d2..d41f2336 100644 --- a/layers/building/building.sql +++ b/layers/building/building.sql @@ -102,8 +102,8 @@ RETURNS TABLE(geometry geometry, osm_id bigint, render_height int, render_min_he -- etldoc: osm_building_polygon -> layer_building:z14_ SELECT DISTINCT ON (osm_id) osm_id, geometry, - ceil( COALESCE(height, levels*3.66,5))::int AS render_height, - floor(COALESCE(min_height, min_level*3.66,0))::int AS render_min_height, + ceil(COALESCE(height, levels*3.66, 5))::int AS render_height, + floor(COALESCE(min_height, min_level*3.66, 0))::int AS render_min_height, material, colour, hide_3d diff --git a/quickstart.sh b/quickstart.sh index 0d372ce9..db4b3b8d 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -66,6 +66,11 @@ if [ "$(version "$DOCKER_VER")" -lt "$(version "$MIN_DOCKER_VER")" ]; then exit 1 fi +# If there are no arguments, or just the area is set, use preloaded docker image to speed up +# to force all steps, use two arguments, e.g. "./quickstart monaco empty" +(( $# == 0 || $# == 1 )) && USE_PRELOADED_IMAGE=true || USE_PRELOADED_IMAGE="" +export USE_PRELOADED_IMAGE + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Pulling or refreshing OpenMapTiles docker images " @@ -82,6 +87,7 @@ echo "-------------------------------------------------------------------------- echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " echo " : This will be logged to the $log_file file (for debugging) and to the screen" echo " : Area : $osm_area " +echo " : Preloaded Image : $USE_PRELOADED_IMAGE " echo " : Git version : $githash " echo " : Started : $STARTDATE " echo " : Your bash version: $BASH_VERSION" @@ -166,30 +172,42 @@ make all echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " -echo " : Source code: https://github.com/openmaptiles/postgis " -echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" -make db-start - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Drop and Recreate PostgreSQL public schema " -# Drop all PostgreSQL tables -# This adds an extra safety belt if the user modifies the docker volume settings -make forced-clean-sql - -echo "====> : Importing all the data:" -echo " : * Water data from http://osmdata.openstreetmap.de" -echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" -echo " : * Natural Earth from http://www.naturalearthdata.com" -echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" -echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" -echo " :" -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" -echo " : includes all data from the import-data image" -echo " :" -echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" -make import-data +if [[ "$USE_PRELOADED_IMAGE" == true ]]; then + echo "====> : Start PostgreSQL service using postgis image preloaded with this data:" + echo " : * Water data from http://osmdata.openstreetmap.de" + echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" + echo " : * Natural Earth from http://www.naturalearthdata.com" + echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" + echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" + echo " :" + echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" + echo " : includes all data from the import-data image" + echo " :" + echo " : Use two-parameter quickstart to start with an empty database:" + echo " : ./quickstart.sh albania empty" + echo " : If desired, you can manually import data by one using these commands:" + echo " : make db-destroy" + echo " : make db-start" + echo " : make import-data" + echo " :" + echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/postgis-preloaded" + echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" + make db-start-preloaded +else + echo "====> : Start PostgreSQL service using empty database and importing all the data:" + echo " : * Water data from http://osmdata.openstreetmap.de" + echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" + echo " : * Natural Earth from http://www.naturalearthdata.com" + echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" + echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" + echo " :" + echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" + echo " : includes all data from the import-data image" + echo " :" + echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" + make db-start + make import-data +fi echo " " echo "-------------------------------------------------------------------------------------"