From 1a9f6132c3be36a220886f6a0fe63049cd9f6f1b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 4 Jun 2020 15:45:04 -0400 Subject: [PATCH] New generate-dc-config target, rm QUICKSTART_MIN/MAX_ZOOM (#915) * Set `MAX_ZOOM` to 7 by default. * Remove `QUICKSTART_MIN/MAX_ZOOM` - unneeded complexity with two env vars. We can just use `MIN_ZOOM` and `MAX_ZOOM`. See also #261 * Generate dc-config yaml file with a new `make generate-dc-config` step. It will compute BBOX based on the downloaded data file. This step is not needed for planet generation. * Generate Imposm replication file only when `DIFF_MODE` is `true`. Not needed otherwise. If the data source does not support it, it will throw an error. --- .env | 13 ++++++---- .github/workflows/tests.yml | 7 ++--- Makefile | 51 +++++++++++++++++++------------------ QUICKSTART.md | 17 +++++-------- README.md | 8 +++--- quickstart.sh | 11 +++++++- 6 files changed, 57 insertions(+), 50 deletions(-) diff --git a/.env b/.env index c4156760..f6cb6929 100644 --- a/.env +++ b/.env @@ -10,13 +10,16 @@ PGPASSWORD=openmaptiles PGHOST=postgres PGPORT=5432 -QUICKSTART_MIN_ZOOM=0 -QUICKSTART_MAX_ZOOM=7 -DIFF_MODE=false - +# BBOX may get overwritten by the computed bbox of the specific area: +# make generate-dc-config BBOX=-180.0,-85.0511,180.0,85.0511 + +# Which zooms to generate in make generate-tiles MIN_ZOOM=0 -MAX_ZOOM=14 +MAX_ZOOM=7 + +# Use true (case sensitive) to allow data updates +DIFF_MODE=false # Hide some output from Mapnik tile generation for clarity FILTER_MAPNIK_OUTPUT=1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index abb258c8..0b896b8a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,12 +20,9 @@ jobs: - name: Run quickstart for a small area env: area: monaco - QUICKSTART_MIN_ZOOM: 0 - QUICKSTART_MAX_ZOOM: 14 + MIN_ZOOM: 0 + MAX_ZOOM: 14 run: | - # For now, change the quickstart values directly in the .env file - # TODO: We should probably use env vars instead - sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env export QUIET=1 ./quickstart.sh $area diff --git a/Makefile b/Makefile index 2e044fc5..df2b06d0 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,12 @@ endif # Set OpenMapTiles host OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost) +# This defines an easy $(newline) value to act as a "\n". Make sure to keep exactly two empty lines after newline. +define newline + + +endef + # # Determine area to work on @@ -161,6 +167,7 @@ help: @echo " make download-geofabrik area=albania # download OSM data from geofabrik.de and create config file" @echo " make download-osmfr area=asia/qatar # download OSM data from openstreetmap.fr and create config file" @echo " make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file" + @echo " make generate-dc-config # scan data file and generate tile generation config file with bbox" @echo " make psql # start PostgreSQL console" @echo " make psql-list-tables # list all PostgreSQL tables" @echo " make vacuum-db # PostgreSQL: VACUUM ANALYZE" @@ -255,41 +262,35 @@ OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@)) .PHONY: $(ALL_DOWNLOADS) $(ALL_DOWNLOADS): init-dirs @$(assert_area_is_given) -ifeq (,$(wildcard $(PBF_FILE))) ifneq ($(strip $(url)),) - $(if $(OSM_SERVER),$(error url parameter can only be used with the 'make download area=... url=...')) + $(if $(OSM_SERVER),$(error url parameter can only be used with non-specific download target:$(newline) make download area=$(area) url="$(url)"$(newline))) endif - @echo "Downloading $(area) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)" +ifeq (,$(wildcard $(PBF_FILE))) + @echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)" @$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \ - download-osm $(OSM_SERVER) $(DOWNLOAD_AREA) \ - --minzoom $$QUICKSTART_MIN_ZOOM \ - --maxzoom $$QUICKSTART_MAX_ZOOM \ - --make-dc $(AREA_DC_CONFIG_FILE) \ - --imposm-cfg $(IMPOSM_CONFIG_FILE) \ - --output $(PBF_FILE) \ - 2>&1 \ - | tee /tmp/download.out ; \ - exit_code=$${PIPESTATUS[0]} ; \ - if [[ "$$exit_code" != "0" ]]; then \ - if grep -q "Imposm config file cannot be generated from this source" /tmp/download.out; then \ - echo "WARNING: $(IMPOSM_CONFIG_FILE) could not be generated, but it is only needed to apply updates." ; \ - else \ - exit $$exit_code ; \ - fi ; \ + if [[ "$$DIFF_MODE" == "true" ]]; then \ + download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \ + --imposm-cfg "$(IMPOSM_CONFIG_FILE)" \ + --output "$(PBF_FILE)" ; \ + else \ + download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \ + --output "$(PBF_FILE)" ; \ fi' @echo "" else + @echo "Data files $(PBF_FILE) already exists, skipping the download." +endif + +.PHONY: generate-dc-config +generate-dc-config: + @$(assert_area_is_given) ifeq (,$(wildcard $(AREA_DC_CONFIG_FILE))) - @echo "Data file $(PBF_FILE) already exists, but the $(AREA_DC_CONFIG_FILE) is not, generating..." @$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \ - download-osm make-dc $(PBF_FILE) \ - --minzoom $$QUICKSTART_MIN_ZOOM \ - --maxzoom $$QUICKSTART_MAX_ZOOM \ - --make-dc $(AREA_DC_CONFIG_FILE) \ + download-osm make-dc "$(PBF_FILE)" \ + --make-dc "$(AREA_DC_CONFIG_FILE)" \ --id "$(area)"' else - @echo "Data files $(PBF_FILE) and $(AREA_DC_CONFIG_FILE) already exists, skipping the download." -endif + @echo "Configuration file $(AREA_DC_CONFIG_FILE) already exists, no need to regenerate." endif .PHONY: psql diff --git a/QUICKSTART.md b/QUICKSTART.md index a8b5f9d2..46927f82 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -384,20 +384,17 @@ and the generated maps are going to be available in webbrowser on [localhost:808 This is only a quick preview, because your mbtiles only generated to zoom level 7 ! -### Change MIN_ZOOM and MAX_ZOOM +### Set which zooms to generate -modify the settings in the `.env` file, the defaults : -* QUICKSTART_MIN_ZOOM=0 -* QUICKSTART_MAX_ZOOM=7 +modify the settings in the `.env` file, the defaults: +* `MIN_ZOOM=0` +* `MAX_ZOOM=7` -and re-start `./quickstart.sh ` -* the new config file re-generating to here ./data/docker-compose-config.yml -* Known problems: - * If you use same area - then the ./data/docker-compose-config.yml not re-generating, so you have to modify by hand! +Delete the `./data/.dc-config.yml` file, and re-start `./quickstart.sh ` Hints: -* Small increments! Never starts with the MAX_ZOOM = 14 -* The suggested MAX_ZOOM = 14 - use only with small extracts +* Small increments! Never starts with the `MAX_ZOOM = 14` +* The suggested `MAX_ZOOM = 14` - use only with small extracts ### Check other commands diff --git a/README.md b/README.md index a33f7f9b..1285b62b 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), make import-data ``` -Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. Use `download-geofabrik`, `download-bbbike`, or `download-osmfr` for a specific source. Use `download area=planet` for the entire OSM dataset (very large). Note that if you have more than one `data/*.osm.pbf` file, every `make` command will require `area=...` parameter (or you can just `export area=...` first) +Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. To use a specific download source, use `download-geofabrik`, `download-bbbike`, or `download-osmfr`, or use `download` to make it auto-pick the area. You can use `area=planet` for the entire OSM dataset (very large). Note that if you have more than one `data/*.osm.pbf` file, every `make` command will always require `area=...` parameter (or you can just `export area=...` first). ```bash make download area=albania @@ -135,11 +135,11 @@ make make import-sql ``` -Now you are ready to **generate the vector tiles**. Using environment variables -you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`). +Now you are ready to **generate the vector tiles**. By default, `./.env` specifies the entire planet BBOX for zooms 0-7, but running `generate-dc-config` will analyze the data file and set the `BBOX` param to limit tile generation. It will also modify `MIN_ZOOM` and `MAX_ZOOM` values based on the .env, but can be changed. ``` -make generate-tiles +make generate-dc-config # compute data bbox -- not needed for the whole planet +make generate-tiles # generate tiles ``` ## License diff --git a/quickstart.sh b/quickstart.sh index 3c5513b6..6b0e90c2 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -33,7 +33,7 @@ set -o nounset # ./quickstart.sh Adelaide bbbike # .... # -# to list geofabrik areas: make download-geofabrik-list +# to list geofabrik areas: make list-geofabrik or make list-bbbike # see more QUICKSTART.md # @@ -261,6 +261,15 @@ echo "-------------------------------------------------------------------------- echo "====> : Testing PostgreSQL tables to match layer definitions metadata" make test-perf-null +echo " " +echo "-------------------------------------------------------------------------------------" +if [[ "$area" != "planet" ]]; then + echo "====> : Compute bounding box for tile generation" + make generate-dc-config +else + echo "====> : Skipping bbox calculation when generating the entire planet" +fi + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "