Use tools v4 with on-the-fly boundary generation (#750)

* Switch OMT to use the new tools v4.0.0 * borders are dynamically generated from the PBF file instead of downloading a prepared CSV file * all tools are executed as current user instead of root, thus files are easier to modify/delete if needed * all data is stored in the local file system instead of docker volumes (Docker currently has a limitation of non-root operation for internal volumes). This also makes it easier to examine and test it. * New `init-dirs` make target creates all the needed dirs - `build, data, cache` * `make clean` deletes the whole `build` dir instead of individual files. * `clean-docker` for backward compatibility deletes `cache` dirs (it used to be a volume) * all `psql` calls are now done with `ON_ERROR_STOP=1` * got rid of `pgclimb-*` targets -- same results can be done with `psql` (`pgclimb-list-views` & `pgclimb-list-tables` renamed to `list-views` and `list-tables`)
2020-03-06 13:15:54 -05:00 · 2020-03-06 13:15:54 -05:00 · 490acf9beb
commit 490acf9beb
--- a/.env
+++ b/.env
@ -6,7 +6,7 @@ POSTGRES_PORT=5432
 QUICKSTART_MIN_ZOOM=0
 QUICKSTART_MAX_ZOOM=7
 DIFF_MODE=false
-TOOLS_VERSION=3.1.0
+TOOLS_VERSION=4.0.0

 BBOX=-180.0,-85.0511,180.0,85.0511
 MIN_ZOOM=0
--- a/.gitignore
+++ b/.gitignore
@ -14,6 +14,9 @@ data/*
 # generated source files
 build/*

+# Import cache
+cache/*
+
 # any IDE files
 .idea/
 .vscode/
--- a/121
+++ b/121
@ -1,8 +1,6 @@
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit
-DC_OPTS?=--rm
-
-# container runs as the current user rather than root (so that created files are not root-owned)
-DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+# Containers run as the current user rather than root (so that created files are not root-owned)
+DC_OPTS?=--rm -u $$(id -u $${USER}):$$(id -g $${USER})

 # If running in the test mode, compare files rather than copy them
 TEST_MODE?=no
@ -53,30 +51,30 @@ help:
 	@echo "  make help                            # help about available commands"
 	@echo "=============================================================================="

-.PHONY: build
-build:
-	mkdir -p build
+.PHONY: init-dirs
+init-dirs:
+	mkdir -p build && mkdir -p data && mkdir -p cache

-build/openmaptiles.tm2source/data.yml: build
+build/openmaptiles.tm2source/data.yml: init-dirs
 	mkdir -p build/openmaptiles.tm2source
 	docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > $@

-build/mapping.yaml: build
+build/mapping.yaml: init-dirs
 	docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@

-build/tileset.sql: build
-	# FIXME:   switch to openmaptiles-tools after v3.2+ is published
-	docker-compose run $(DC_OPTS) openmaptiles-tools-latest generate-sql openmaptiles.yaml > $@
+build/tileset.sql: init-dirs
+	docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@

 .PHONY: clean
 clean:
-	rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql
+	rm -rf build

 .PHONY: clean-docker
 clean-docker:
 	docker-compose down -v --remove-orphans
 	docker-compose rm -fv
 	docker volume ls -q | grep openmaptiles  | xargs -r docker volume rm || true
+	rm -rf cache

 .PHONY: db-start
 db-start:
@ -84,13 +82,17 @@ db-start:
 	@echo "Wait for PostgreSQL to start..."
 	docker-compose run $(DC_OPTS) import-osm  ./pgwait.sh

+.PHONY: db-stop
+db-stop:
+	docker-compose stop postgres
+
 .PHONY: download-geofabrik
-download-geofabrik:
-	@echo ===============  download-geofabrik =======================
-	@echo Download area :   $(area)
-	@echo [[ example: make download-geofabrik  area=albania ]]
-	@echo [[ list areas:  make download-geofabrik-list       ]]
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik.sh $(area)
+download-geofabrik: init-dirs
+	@echo =============== download-geofabrik =======================
+	@echo Download area:   $(area)
+	@echo [[ example: make download-geofabrik area=albania ]]
+	@echo [[ list areas:  make download-geofabrik-list     ]]
+	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area)
 	ls -la ./data/$(area).*
 	@echo "Generated config file: ./data/docker-compose-config.yml"
 	@echo " "
@ -110,23 +112,37 @@ import-sql: db-start all
 	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql

 .PHONY: import-osmsql
-import-osmsql: db-start all
-	docker-compose run $(DC_OPTS) import-osm
-	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
+import-osmsql: db-start all import-osm import-sql
+
+.PHONY: import-borders
+import-borders: db-start
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-borders
+
+.PHONY: import-water
+import-water: db-start
+	docker-compose run $(DC_OPTS) import-water
+
+.PHONY: import-natural-earth
+import-natural-earth: db-start
+	docker-compose run $(DC_OPTS) import-natural-earth
+
+.PHONY: import-lakelines
+import-lakelines: db-start
+	docker-compose run $(DC_OPTS) import-lakelines

 .PHONY: generate-tiles
-generate-tiles: db-start all
+generate-tiles: init-dirs db-start all
 	rm -rf data/tiles.mbtiles
 	if [ -f ./data/docker-compose-config.yml ]; then \
-		docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $(DC_OPTS) generate-vectortiles; \
+		docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml \
+					   run $(DC_OPTS) generate-vectortiles; \
 	else \
 		docker-compose run $(DC_OPTS) generate-vectortiles; \
 	fi
-	docker-compose run $(DC_OPTS) openmaptiles-tools  generate-metadata ./data/tiles.mbtiles
-	docker-compose run $(DC_OPTS) openmaptiles-tools  chmod 666         ./data/tiles.mbtiles
+	docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles

 .PHONY: start-tileserver
-start-tileserver:
+start-tileserver: init-dirs
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
@ -175,9 +191,9 @@ generate-qareports:

 # generate all etl and mapping graphs
 .PHONY: generate-devdoc
-generate-devdoc:
+generate-devdoc: init-dirs
 	mkdir -p ./build/devdoc && \
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools-latest sh -c \
+	docker-compose run $(DC_OPTS) openmaptiles-tools-latest sh -c \
 			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
 			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'

@ -191,50 +207,51 @@ import-osm-dev:

 # the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command
 .PHONY: list
-list:
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
+list: download-geofabrik-list

-# same as a `make list`
 .PHONY: download-geofabrik-list
 download-geofabrik-list:
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
+	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh

 .PHONY: import-wikidata
 import-wikidata:
-	# FIXME:   switch to openmaptiles-tools after v3.2+ is published
-	docker-compose run $(DC_OPTS) openmaptiles-tools-latest import-wikidata openmaptiles.yaml
-
-.PHONY: psql-list-tables
-psql-list-tables:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c "\d+"
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml

 .PHONY: psql-pg-stat-reset
 psql-pg-stat-reset:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'SELECT pg_stat_statements_reset();'
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();'

 .PHONY: forced-clean-sql
 forced-clean-sql:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; "
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';"
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \
+		-c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \
+		-c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \
+		-c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \
+		-c "GRANT ALL ON SCHEMA public TO public; COMMENT ON SCHEMA public IS 'standard public schema';"

-.PHONY: pgclimb-list-views
-pgclimb-list-views:
-	docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv
+.PHONY: list-views
+list-views:
+	@docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+		-c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;"

-.PHONY: pgclimb-list-tables
-pgclimb-list-tables:
-	docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv
+.PHONY: list-tables
+list-tables:
+	@docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+		-c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;"
+
+.PHONY: psql-list-tables
+psql-list-tables:
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+"

 .PHONY: psql-vacuum-analyze
 psql-vacuum-analyze:
 	@echo "Start - postgresql: VACUUM ANALYZE VERBOSE;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'VACUUM ANALYZE VERBOSE;'
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;'

 .PHONY: psql-analyze
 psql-analyze:
-	@echo "Start - postgresql: ANALYZE VERBOSE ;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'ANALYZE VERBOSE;'
+	@echo "Start - postgresql: ANALYZE VERBOSE;"
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;'

 .PHONY: list-docker-images
 list-docker-images:
--- a/QUICKSTART.md
+++ b/QUICKSTART.md
@ -411,7 +411,6 @@ the current output:
 OpenMapTiles  https://github.com/openmaptiles/openmaptiles
 Hints for testing areas
  make download-geofabrik-list         # list actual geofabrik OSM extracts for download -> <<your-area>>
-  make list                            # list actual geofabrik OSM extracts for download -> <<your-area>>
  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar

 Hints for designers:
@ -434,8 +433,8 @@ Hints for developers:
  make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)
  make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB
  make remove-docker-images            # remove openmaptiles docker images
-  make pgclimb-list-views              # list PostgreSQL public schema views
-  make pgclimb-list-tables             # list PostgreSQL public schema tables
+  make list-views                      # list PostgreSQL public schema views
+  make list-tables                     # list PostgreSQL public schema tables
  cat  .env                            # list PG database and MIN_ZOOM and MAX_ZOOM information
  cat ./quickstart.log                 # backup  of the last ./quickstart.sh
  make help                            # help about available commands
--- a/README.md
+++ b/README.md
@ -94,16 +94,15 @@ or use the provided `quickstart.sh` script.
 Now start up the database container.

 ```bash
-docker-compose up -d postgres
+make db-start
 ```

 Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).

 ```bash
-docker-compose run import-water
-docker-compose run import-natural-earth
-docker-compose run import-lakelines
-docker-compose run import-osmborder
+make import-water
+make import-natural-earth
+make import-lakelines
 ```

 [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
@ -113,11 +112,18 @@ cd data
 wget http://download.geofabrik.de/europe/albania-latest.osm.pbf
 ```

-[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
-`build/mapping.yaml` (which has been created by `make`).
+OR

 ```bash
-docker-compose run import-osm
+make download-geofabrik area=albania
+```
+
+[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
+`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton.  Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
+
+```bash
+make import-osm
+make import-borders
 ```

 Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field.
@ -142,7 +148,7 @@ Now you are ready to **generate the vector tiles**. Using environment variables
 you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`).

 ```
-docker-compose run generate-vectortiles
+make generate-tiles
 ```

 ## License
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,7 +1,6 @@
 version: "2"
 volumes:
  pgdata:
-  cache:
 services:
  postgres:
    image: "openmaptiles/postgis:${TOOLS_VERSION}"
@ -37,12 +36,7 @@ services:
    volumes:
      - ./data:/import
      - ./build:/mapping
-      - cache:/cache
-  import-osmborder:
-    image: "openmaptiles/import-osmborder:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
+      - ./cache:/cache
  import-osm-diff:
    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
@ -54,7 +48,7 @@ services:
    volumes:
      - ./data:/import
      - ./build:/mapping
-      - cache:/cache
+      - ./cache:/cache
  update-osm:
    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
@ -66,7 +60,7 @@ services:
    volumes:
      - ./data:/import
      - ./build:/mapping
-      - cache:/cache
+      - ./cache:/cache
  openmaptiles-tools:
    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
    env_file: .env
@ -74,6 +68,7 @@ services:
      - postgres_conn
    volumes:
      - .:/tileset
+      - ./data:/import
      - ./build:/sql
  openmaptiles-tools-latest:
    # This target exists for experimental tools that have not yet been published.
@ -84,6 +79,7 @@ services:
      - postgres_conn
    volumes:
      - .:/tileset
+      - ./data:/import
      - ./build:/sql
  generate-changed-vectortiles:
    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
--- a/quickstart.sh
+++ b/quickstart.sh
@ -39,7 +39,7 @@ githash=$( git rev-parse HEAD )
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit,
 # as well as pass any other common parameters.
 # In the future this should use -u $(id -u "$USER"):$(id -g "$USER") instead of running docker as root.
-DC_OPTS="--rm"
+DC_OPTS="--rm -u $(id -u "$USER"):$(id -g "$USER")"

 log_file=./quickstart.log
 rm -f $log_file
@ -130,10 +130,8 @@ docker images | grep openmaptiles

 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) "
-mkdir -p pgdata
-mkdir -p build
-mkdir -p data
+echo "====> : Create directories if they don't exist"
+make init-dirs

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -185,7 +183,7 @@ echo "--------------------------------------------------------------------------
 echo "====> : Start PostgreSQL service ; create PostgreSQL data volume "
 echo "      : Source code: https://github.com/openmaptiles/postgis "
 echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
-docker-compose up -d postgres
+make db-start

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -200,15 +198,7 @@ echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ i
 echo "      : Source code:  https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water "
 echo "      : Data license: https://osmdata.openstreetmap.de/info/license.html "
 echo "      : Thank you: https://osmdata.openstreetmap.de/info/ "
-docker-compose run $DC_OPTS import-water
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing border data from http://openstreetmap.org into PostgreSQL "
-echo "      : Source code:  https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osmborder"
-echo "      : Data license: http://www.openstreetmap.org/copyright"
-echo "      : Thank you: https://github.com/pnorman/osmborder "
-docker-compose run $DC_OPTS import-osmborder
+make import-water

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -216,7 +206,7 @@ echo "====> : Start importing  http://www.naturalearthdata.com  into PostgreSQL
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth "
 echo "      : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use  "
 echo "      : Thank you: Natural Earth Contributors! "
-docker-compose run $DC_OPTS import-natural-earth
+make import-natural-earth

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -224,7 +214,7 @@ echo "====> : Start importing OpenStreetMap Lakelines data "
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines "
 echo "      :              https://github.com/lukasmartinelli/osm-lakelines "
 echo "      : Data license: .. "
-docker-compose run $DC_OPTS import-lakelines
+make import-lakelines

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -234,7 +224,15 @@ echo "      :   Thank you Omniscale! "
 echo "      :   Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm "
 echo "      : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) "
 echo "      : Thank you OpenStreetMap Contributors ! "
-docker-compose run $DC_OPTS import-osm
+make import-osm
+
+echo " "
+echo "-------------------------------------------------------------------------------------"
+echo "====> : Start importing border data from ./data/${testdata} into PostgreSQL using osmborder"
+echo "      : Source code: https://github.com/pnorman/osmborder"
+echo "      : Data license: http://www.openstreetmap.org/copyright"
+echo "      : Thank you: Paul Norman"
+make import-borders

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -242,7 +240,7 @@ echo "====> : Start SQL postprocessing:  ./build/tileset.sql -> PostgreSQL "
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql"
 # If the output contains a WARNING, stop further processing
 # Adapted from https://unix.stackexchange.com/questions/307562
-docker-compose run $DC_OPTS openmaptiles-tools import-sql | \
+make import-sql | \
    awk -v s=": WARNING:" '$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1'

 echo " "
@ -274,19 +272,12 @@ echo "      : See other MVT tools : https://github.com/mapbox/awesome-vector-til
 echo "      :  "
 echo "      : You will see a lot of deprecated warning in the log! This is normal!  "
 echo "      :    like :  Mapnik LOG>  ... is deprecated and will be removed in Mapnik 4.x ... "
-
-docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $DC_OPTS generate-vectortiles
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Add special metadata to mbtiles! "
-docker-compose run $DC_OPTS openmaptiles-tools  generate-metadata ./data/tiles.mbtiles
-docker-compose run $DC_OPTS openmaptiles-tools  chmod 666         ./data/tiles.mbtiles
+make generate-tiles

 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )"
-docker-compose stop postgres
+make db-stop

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -320,13 +311,6 @@ echo "====> : (disk space) We have created a lot of docker images: "
 echo "      : Hint: you can remove with:  docker rmi IMAGE "
 docker images | grep openmaptiles

-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:"
-echo "      : Hint: you can remove with : docker volume rm openmaptiles_pgdata "
-docker volume ls -q | grep openmaptiles
-
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) "