From 490acf9beb22ec06d118326a2311f9d265e181f1 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 6 Mar 2020 13:15:54 -0500 Subject: [PATCH] Use tools v4 with on-the-fly boundary generation (#750) * Switch OMT to use the new tools v4.0.0 * borders are dynamically generated from the PBF file instead of downloading a prepared CSV file * all tools are executed as current user instead of root, thus files are easier to modify/delete if needed * all data is stored in the local file system instead of docker volumes (Docker currently has a limitation of non-root operation for internal volumes). This also makes it easier to examine and test it. * New `init-dirs` make target creates all the needed dirs - `build, data, cache` * `make clean` deletes the whole `build` dir instead of individual files. * `clean-docker` for backward compatibility deletes `cache` dirs (it used to be a volume) * all `psql` calls are now done with `ON_ERROR_STOP=1` * got rid of `pgclimb-*` targets -- same results can be done with `psql` (`pgclimb-list-views` & `pgclimb-list-tables` renamed to `list-views` and `list-tables`) --- .env | 2 +- .gitignore | 3 ++ Makefile | 121 ++++++++++++++++++++++++++------------------- QUICKSTART.md | 5 +- README.md | 24 +++++---- docker-compose.yml | 14 ++---- quickstart.sh | 54 +++++++------------- 7 files changed, 114 insertions(+), 109 deletions(-) diff --git a/.env b/.env index 61fca597..b9fe82a7 100644 --- a/.env +++ b/.env @@ -6,7 +6,7 @@ POSTGRES_PORT=5432 QUICKSTART_MIN_ZOOM=0 QUICKSTART_MAX_ZOOM=7 DIFF_MODE=false -TOOLS_VERSION=3.1.0 +TOOLS_VERSION=4.0.0 BBOX=-180.0,-85.0511,180.0,85.0511 MIN_ZOOM=0 diff --git a/.gitignore b/.gitignore index 57c0c6ea..70f14627 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ data/* # generated source files build/* +# Import cache +cache/* + # any IDE files .idea/ .vscode/ diff --git a/Makefile b/Makefile index f7ed98a0..729b04c9 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ # Options to run with docker and docker-compose - ensure the container is destroyed on exit -DC_OPTS?=--rm - -# container runs as the current user rather than root (so that created files are not root-owned) -DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER}) +# Containers run as the current user rather than root (so that created files are not root-owned) +DC_OPTS?=--rm -u $$(id -u $${USER}):$$(id -g $${USER}) # If running in the test mode, compare files rather than copy them TEST_MODE?=no @@ -53,30 +51,30 @@ help: @echo " make help # help about available commands" @echo "==============================================================================" -.PHONY: build -build: - mkdir -p build +.PHONY: init-dirs +init-dirs: + mkdir -p build && mkdir -p data && mkdir -p cache -build/openmaptiles.tm2source/data.yml: build +build/openmaptiles.tm2source/data.yml: init-dirs mkdir -p build/openmaptiles.tm2source docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > $@ -build/mapping.yaml: build +build/mapping.yaml: init-dirs docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@ -build/tileset.sql: build - # FIXME: switch to openmaptiles-tools after v3.2+ is published - docker-compose run $(DC_OPTS) openmaptiles-tools-latest generate-sql openmaptiles.yaml > $@ +build/tileset.sql: init-dirs + docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@ .PHONY: clean clean: - rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql + rm -rf build .PHONY: clean-docker clean-docker: docker-compose down -v --remove-orphans docker-compose rm -fv docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true + rm -rf cache .PHONY: db-start db-start: @@ -84,13 +82,17 @@ db-start: @echo "Wait for PostgreSQL to start..." docker-compose run $(DC_OPTS) import-osm ./pgwait.sh +.PHONY: db-stop +db-stop: + docker-compose stop postgres + .PHONY: download-geofabrik -download-geofabrik: - @echo =============== download-geofabrik ======================= - @echo Download area : $(area) - @echo [[ example: make download-geofabrik area=albania ]] - @echo [[ list areas: make download-geofabrik-list ]] - docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area) +download-geofabrik: init-dirs + @echo =============== download-geofabrik ======================= + @echo Download area: $(area) + @echo [[ example: make download-geofabrik area=albania ]] + @echo [[ list areas: make download-geofabrik-list ]] + docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area) ls -la ./data/$(area).* @echo "Generated config file: ./data/docker-compose-config.yml" @echo " " @@ -110,23 +112,37 @@ import-sql: db-start all docker-compose run $(DC_OPTS) openmaptiles-tools import-sql .PHONY: import-osmsql -import-osmsql: db-start all - docker-compose run $(DC_OPTS) import-osm - docker-compose run $(DC_OPTS) openmaptiles-tools import-sql +import-osmsql: db-start all import-osm import-sql + +.PHONY: import-borders +import-borders: db-start + docker-compose run $(DC_OPTS) openmaptiles-tools import-borders + +.PHONY: import-water +import-water: db-start + docker-compose run $(DC_OPTS) import-water + +.PHONY: import-natural-earth +import-natural-earth: db-start + docker-compose run $(DC_OPTS) import-natural-earth + +.PHONY: import-lakelines +import-lakelines: db-start + docker-compose run $(DC_OPTS) import-lakelines .PHONY: generate-tiles -generate-tiles: db-start all +generate-tiles: init-dirs db-start all rm -rf data/tiles.mbtiles if [ -f ./data/docker-compose-config.yml ]; then \ - docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $(DC_OPTS) generate-vectortiles; \ + docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml \ + run $(DC_OPTS) generate-vectortiles; \ else \ docker-compose run $(DC_OPTS) generate-vectortiles; \ fi - docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles - docker-compose run $(DC_OPTS) openmaptiles-tools chmod 666 ./data/tiles.mbtiles + docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles .PHONY: start-tileserver -start-tileserver: +start-tileserver: init-dirs @echo " " @echo "***********************************************************" @echo "* " @@ -175,9 +191,9 @@ generate-qareports: # generate all etl and mapping graphs .PHONY: generate-devdoc -generate-devdoc: +generate-devdoc: init-dirs mkdir -p ./build/devdoc && \ - docker-compose run $(DC_USER_OPTS) openmaptiles-tools-latest sh -c \ + docker-compose run $(DC_OPTS) openmaptiles-tools-latest sh -c \ 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' @@ -191,50 +207,51 @@ import-osm-dev: # the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command .PHONY: list -list: - docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh +list: download-geofabrik-list -# same as a `make list` .PHONY: download-geofabrik-list download-geofabrik-list: - docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh + docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh .PHONY: import-wikidata import-wikidata: - # FIXME: switch to openmaptiles-tools after v3.2+ is published - docker-compose run $(DC_OPTS) openmaptiles-tools-latest import-wikidata openmaptiles.yaml - -.PHONY: psql-list-tables -psql-list-tables: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c "\d+" + docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml .PHONY: psql-pg-stat-reset psql-pg-stat-reset: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();' + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();' .PHONY: forced-clean-sql forced-clean-sql: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; " - docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';" + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \ + -c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \ + -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \ + -c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \ + -c "GRANT ALL ON SCHEMA public TO public; COMMENT ON SCHEMA public IS 'standard public schema';" -.PHONY: pgclimb-list-views -pgclimb-list-views: - docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv +.PHONY: list-views +list-views: + @docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + -c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;" -.PHONY: pgclimb-list-tables -pgclimb-list-tables: - docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv +.PHONY: list-tables +list-tables: + @docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + -c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;" + +.PHONY: psql-list-tables +psql-list-tables: + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+" .PHONY: psql-vacuum-analyze psql-vacuum-analyze: @echo "Start - postgresql: VACUUM ANALYZE VERBOSE;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c 'VACUUM ANALYZE VERBOSE;' + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;' .PHONY: psql-analyze psql-analyze: - @echo "Start - postgresql: ANALYZE VERBOSE ;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c 'ANALYZE VERBOSE;' + @echo "Start - postgresql: ANALYZE VERBOSE;" + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;' .PHONY: list-docker-images list-docker-images: diff --git a/QUICKSTART.md b/QUICKSTART.md index 90fed7ef..dc3e5b14 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -411,7 +411,6 @@ the current output: OpenMapTiles https://github.com/openmaptiles/openmaptiles Hints for testing areas make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> - make list # list actual geofabrik OSM extracts for download -> <> ./quickstart.sh <> # example: ./quickstart.sh madagascar Hints for designers: @@ -434,8 +433,8 @@ Hints for developers: make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s) make refresh-docker-images # refresh openmaptiles docker images from Docker HUB make remove-docker-images # remove openmaptiles docker images - make pgclimb-list-views # list PostgreSQL public schema views - make pgclimb-list-tables # list PostgreSQL public schema tables + make list-views # list PostgreSQL public schema views + make list-tables # list PostgreSQL public schema tables cat .env # list PG database and MIN_ZOOM and MAX_ZOOM information cat ./quickstart.log # backup of the last ./quickstart.sh make help # help about available commands diff --git a/README.md b/README.md index 9ed9f332..32678537 100644 --- a/README.md +++ b/README.md @@ -94,16 +94,15 @@ or use the provided `quickstart.sh` script. Now start up the database container. ```bash -docker-compose up -d postgres +make db-start ``` Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines). ```bash -docker-compose run import-water -docker-compose run import-natural-earth -docker-compose run import-lakelines -docker-compose run import-osmborder +make import-water +make import-natural-earth +make import-lakelines ``` [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory. @@ -113,11 +112,18 @@ cd data wget http://download.geofabrik.de/europe/albania-latest.osm.pbf ``` -[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from -`build/mapping.yaml` (which has been created by `make`). +OR ```bash -docker-compose run import-osm +make download-geofabrik area=albania +``` + +[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from +`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. + +```bash +make import-osm +make import-borders ``` Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field. @@ -142,7 +148,7 @@ Now you are ready to **generate the vector tiles**. Using environment variables you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`). ``` -docker-compose run generate-vectortiles +make generate-tiles ``` ## License diff --git a/docker-compose.yml b/docker-compose.yml index a47a954f..41087d41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ version: "2" volumes: pgdata: - cache: services: postgres: image: "openmaptiles/postgis:${TOOLS_VERSION}" @@ -37,12 +36,7 @@ services: volumes: - ./data:/import - ./build:/mapping - - cache:/cache - import-osmborder: - image: "openmaptiles/import-osmborder:${TOOLS_VERSION}" - env_file: .env - networks: - - postgres_conn + - ./cache:/cache import-osm-diff: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -54,7 +48,7 @@ services: volumes: - ./data:/import - ./build:/mapping - - cache:/cache + - ./cache:/cache update-osm: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -66,7 +60,7 @@ services: volumes: - ./data:/import - ./build:/mapping - - cache:/cache + - ./cache:/cache openmaptiles-tools: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" env_file: .env @@ -74,6 +68,7 @@ services: - postgres_conn volumes: - .:/tileset + - ./data:/import - ./build:/sql openmaptiles-tools-latest: # This target exists for experimental tools that have not yet been published. @@ -84,6 +79,7 @@ services: - postgres_conn volumes: - .:/tileset + - ./data:/import - ./build:/sql generate-changed-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" diff --git a/quickstart.sh b/quickstart.sh index 911b00c8..7970fa67 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -39,7 +39,7 @@ githash=$( git rev-parse HEAD ) # Options to run with docker and docker-compose - ensure the container is destroyed on exit, # as well as pass any other common parameters. # In the future this should use -u $(id -u "$USER"):$(id -g "$USER") instead of running docker as root. -DC_OPTS="--rm" +DC_OPTS="--rm -u $(id -u "$USER"):$(id -g "$USER")" log_file=./quickstart.log rm -f $log_file @@ -130,10 +130,8 @@ docker images | grep openmaptiles echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) " -mkdir -p pgdata -mkdir -p build -mkdir -p data +echo "====> : Create directories if they don't exist" +make init-dirs echo " " echo "-------------------------------------------------------------------------------------" @@ -185,7 +183,7 @@ echo "-------------------------------------------------------------------------- echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" -docker-compose up -d postgres +make db-start echo " " echo "-------------------------------------------------------------------------------------" @@ -200,15 +198,7 @@ echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ i echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water " echo " : Data license: https://osmdata.openstreetmap.de/info/license.html " echo " : Thank you: https://osmdata.openstreetmap.de/info/ " -docker-compose run $DC_OPTS import-water - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing border data from http://openstreetmap.org into PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osmborder" -echo " : Data license: http://www.openstreetmap.org/copyright" -echo " : Thank you: https://github.com/pnorman/osmborder " -docker-compose run $DC_OPTS import-osmborder +make import-water echo " " echo "-------------------------------------------------------------------------------------" @@ -216,7 +206,7 @@ echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth " echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " echo " : Thank you: Natural Earth Contributors! " -docker-compose run $DC_OPTS import-natural-earth +make import-natural-earth echo " " echo "-------------------------------------------------------------------------------------" @@ -224,7 +214,7 @@ echo "====> : Start importing OpenStreetMap Lakelines data " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines " echo " : https://github.com/lukasmartinelli/osm-lakelines " echo " : Data license: .. " -docker-compose run $DC_OPTS import-lakelines +make import-lakelines echo " " echo "-------------------------------------------------------------------------------------" @@ -234,7 +224,15 @@ echo " : Thank you Omniscale! " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm " echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " echo " : Thank you OpenStreetMap Contributors ! " -docker-compose run $DC_OPTS import-osm +make import-osm + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing border data from ./data/${testdata} into PostgreSQL using osmborder" +echo " : Source code: https://github.com/pnorman/osmborder" +echo " : Data license: http://www.openstreetmap.org/copyright" +echo " : Thank you: Paul Norman" +make import-borders echo " " echo "-------------------------------------------------------------------------------------" @@ -242,7 +240,7 @@ echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql" # If the output contains a WARNING, stop further processing # Adapted from https://unix.stackexchange.com/questions/307562 -docker-compose run $DC_OPTS openmaptiles-tools import-sql | \ +make import-sql | \ awk -v s=": WARNING:" '$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1' echo " " @@ -274,19 +272,12 @@ echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-til echo " : " echo " : You will see a lot of deprecated warning in the log! This is normal! " echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " - -docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $DC_OPTS generate-vectortiles - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Add special metadata to mbtiles! " -docker-compose run $DC_OPTS openmaptiles-tools generate-metadata ./data/tiles.mbtiles -docker-compose run $DC_OPTS openmaptiles-tools chmod 666 ./data/tiles.mbtiles +make generate-tiles echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" -docker-compose stop postgres +make db-stop echo " " echo "-------------------------------------------------------------------------------------" @@ -320,13 +311,6 @@ echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " docker images | grep openmaptiles - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" -echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " -docker volume ls -q | grep openmaptiles - echo " " echo "-------------------------------------------------------------------------------------" echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) "