New generate-dc-config target, rm QUICKSTART_MIN/MAX_ZOOM (#915)

* Set `MAX_ZOOM` to 7 by default.
* Remove `QUICKSTART_MIN/MAX_ZOOM` - unneeded complexity with two env vars. We can just use `MIN_ZOOM` and `MAX_ZOOM`. See also #261
* Generate dc-config yaml file with a new `make generate-dc-config` step. It will compute BBOX based on the downloaded data file. This step is not needed for planet generation.
* Generate Imposm replication file only when `DIFF_MODE` is `true`. Not needed otherwise. If the data source does not support it, it will throw an error.

.env

@@ -10,13 +10,16 @@ PGPASSWORD=openmaptiles
 PGHOST=postgres
 PGPORT=5432
 
-QUICKSTART_MIN_ZOOM=0
-QUICKSTART_MAX_ZOOM=7
-DIFF_MODE=false
-
+# BBOX may get overwritten by the computed bbox of the specific area:
+#   make generate-dc-config
 BBOX=-180.0,-85.0511,180.0,85.0511
+
+# Which zooms to generate in   make generate-tiles
 MIN_ZOOM=0
-MAX_ZOOM=14
+MAX_ZOOM=7
+
+# Use  true  (case sensitive) to allow data updates
+DIFF_MODE=false
 
 # Hide some output from Mapnik tile generation for clarity
 FILTER_MAPNIK_OUTPUT=1

.github/workflows/tests.yml

@@ -20,12 +20,9 @@ jobs:
       - name: Run quickstart for a small area
         env:
           area: monaco
-          QUICKSTART_MIN_ZOOM: 0
-          QUICKSTART_MAX_ZOOM: 14
+          MIN_ZOOM: 0
+          MAX_ZOOM: 14
         run: |
-          # For now, change the quickstart values directly in the .env file
-          # TODO: We should probably use env vars instead
-          sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
           export QUIET=1
           ./quickstart.sh $area
 

Makefile

@@ -51,6 +51,12 @@ endif
 # Set OpenMapTiles host
 OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
 
+# This defines an easy $(newline) value to act as a "\n". Make sure to keep exactly two empty lines after newline.
+define newline
+
+
+endef
+
 
 #
 # Determine area to work on
@@ -161,6 +167,7 @@ help:
 	@echo "  make download-geofabrik area=albania # download OSM data from geofabrik.de     and create config file"
 	@echo "  make download-osmfr area=asia/qatar  # download OSM data from openstreetmap.fr and create config file"
 	@echo "  make download-bbbike area=Amsterdam  # download OSM data from bbbike.org       and create config file"
+	@echo "  make generate-dc-config              # scan data file and generate tile generation config file with bbox"
 	@echo "  make psql                            # start PostgreSQL console"
 	@echo "  make psql-list-tables                # list all PostgreSQL tables"
 	@echo "  make vacuum-db                       # PostgreSQL: VACUUM ANALYZE"
@@ -255,41 +262,35 @@ OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@))
 .PHONY: $(ALL_DOWNLOADS)
 $(ALL_DOWNLOADS): init-dirs
 	@$(assert_area_is_given)
-ifeq (,$(wildcard $(PBF_FILE)))
 ifneq ($(strip $(url)),)
-	$(if $(OSM_SERVER),$(error url parameter can only be used with the 'make download area=... url=...'))
+	$(if $(OSM_SERVER),$(error url parameter can only be used with non-specific download target:$(newline)       make download area=$(area) url="$(url)"$(newline)))
 endif
-	@echo "Downloading $(area) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+ifeq (,$(wildcard $(PBF_FILE)))
+	@echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
 	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
-		download-osm $(OSM_SERVER) $(DOWNLOAD_AREA) \
-			--minzoom $$QUICKSTART_MIN_ZOOM \
-			--maxzoom $$QUICKSTART_MAX_ZOOM \
-			--make-dc $(AREA_DC_CONFIG_FILE) \
-			--imposm-cfg $(IMPOSM_CONFIG_FILE) \
-			--output $(PBF_FILE) \
-			2>&1 \
-			| tee /tmp/download.out ; \
-		exit_code=$${PIPESTATUS[0]} ; \
-		if [[ "$$exit_code" != "0" ]]; then \
-			if grep -q "Imposm config file cannot be generated from this source" /tmp/download.out; then \
-				echo "WARNING: $(IMPOSM_CONFIG_FILE) could not be generated, but it is only needed to apply updates." ; \
-			else \
-				exit $$exit_code ; \
-			fi ; \
+		if [[ "$$DIFF_MODE" == "true" ]]; then \
+			download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--imposm-cfg "$(IMPOSM_CONFIG_FILE)" \
+				--output "$(PBF_FILE)" ; \
+		else \
+			download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--output "$(PBF_FILE)" ; \
 		fi'
 	@echo ""
 else
+	@echo "Data files $(PBF_FILE) already exists, skipping the download."
+endif
+
+.PHONY: generate-dc-config
+generate-dc-config:
+	@$(assert_area_is_given)
 ifeq (,$(wildcard $(AREA_DC_CONFIG_FILE)))
-	@echo "Data file $(PBF_FILE) already exists, but the $(AREA_DC_CONFIG_FILE) is not, generating..."
 	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
-		download-osm make-dc $(PBF_FILE) \
-			--minzoom $$QUICKSTART_MIN_ZOOM \
-			--maxzoom $$QUICKSTART_MAX_ZOOM \
-			--make-dc $(AREA_DC_CONFIG_FILE) \
+		download-osm make-dc "$(PBF_FILE)" \
+			--make-dc "$(AREA_DC_CONFIG_FILE)" \
 			--id "$(area)"'
 else
-	@echo "Data files $(PBF_FILE) and $(AREA_DC_CONFIG_FILE) already exists, skipping the download."
-endif
+	@echo "Configuration file $(AREA_DC_CONFIG_FILE) already exists, no need to regenerate."
 endif
 
 .PHONY: psql

QUICKSTART.md

@@ -384,20 +384,17 @@ and the generated maps are going to be available in webbrowser on [localhost:808
 This is only a quick preview, because your mbtiles only generated to zoom level 7 !
 
 
-### Change MIN_ZOOM and MAX_ZOOM
+### Set which zooms to generate
 
-modify the settings in the `.env`  file, the defaults :
-* QUICKSTART_MIN_ZOOM=0
-* QUICKSTART_MAX_ZOOM=7
+modify the settings in the `.env` file, the defaults:
+* `MIN_ZOOM=0`
+* `MAX_ZOOM=7`
 
-and re-start  `./quickstart.sh `
-*  the new config file re-generating to here  ./data/docker-compose-config.yml
-*  Known problems:
-    * If you use same area - then the ./data/docker-compose-config.yml not re-generating, so you have to modify by hand!
+Delete the `./data/<area>.dc-config.yml` file, and re-start `./quickstart.sh <area>`
 
 Hints:
-* Small increments! Never starts with the MAX_ZOOM = 14
-* The suggested  MAX_ZOOM = 14  - use only with small extracts
+* Small increments! Never starts with the `MAX_ZOOM = 14`
+* The suggested  `MAX_ZOOM = 14`  - use only with small extracts
 
 ### Check other commands
 

README.md

@@ -103,7 +103,7 @@ Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/),
 make import-data
 ```
 
-Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. Use `download-geofabrik`, `download-bbbike`, or `download-osmfr` for a specific source. Use `download area=planet` for the entire OSM dataset (very large).  Note that if you have more than one `data/*.osm.pbf` file, every `make` command will require `area=...` parameter (or you can just `export area=...` first)
+Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. To use a specific download source, use `download-geofabrik`, `download-bbbike`, or `download-osmfr`, or use `download` to make it auto-pick the area. You can use `area=planet` for the entire OSM dataset (very large).  Note that if you have more than one `data/*.osm.pbf` file, every `make` command will always require `area=...` parameter (or you can just `export area=...` first).
 
 ```bash
 make download area=albania
@@ -135,11 +135,11 @@ make
 make import-sql
 ```
 
-Now you are ready to **generate the vector tiles**. Using environment variables
-you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`).
+Now you are ready to **generate the vector tiles**. By default, `./.env` specifies the entire planet BBOX for zooms 0-7, but running `generate-dc-config` will analyze the data file and set the `BBOX` param to limit tile generation. It will also modify `MIN_ZOOM` and `MAX_ZOOM` values based on the .env, but can be changed.
 
 ```
-make generate-tiles
+make generate-dc-config  # compute data bbox -- not needed for the whole planet
+make generate-tiles      # generate tiles
 ```
 
 ## License

quickstart.sh

@@ -33,7 +33,7 @@ set -o nounset
 # ./quickstart.sh Adelaide bbbike
 # ....
 #
-# to list geofabrik areas:  make download-geofabrik-list
+# to list geofabrik areas:  make list-geofabrik or make list-bbbike
 # see more QUICKSTART.md
 #
 
@@ -261,6 +261,15 @@ echo "--------------------------------------------------------------------------
 echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
 make test-perf-null
 
+echo " "
+echo "-------------------------------------------------------------------------------------"
+if [[ "$area" != "planet" ]]; then
+  echo "====> : Compute bounding box for tile generation"
+  make generate-dc-config
+else
+  echo "====> : Skipping bbox calculation when generating the entire planet"
+fi
+
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "