Makefile target cleanup, minor improvements (#821)

* allow postgres image to be overwritten with an env var
* allow DIFF_MODE var to be overwritten with an env var
* add /mapping and /cache dirs into tools image
* make `build-sql` target explicit rather than relying on a filename
* `tools-dev` will open a shell in a docker to experiment and debug (instead of `import-sql-dev` and `import-osm-dev`)
* separated `start-maputnik` from `start-postserve`
* renamed `clean-docker` into `db-destroy` to make it more explicit
* cleaner `db-start`, `db-stop`, `db-destroy` targets
* better output messages

Makefile

@@ -24,7 +24,7 @@ else
 endif
 
 .PHONY: all
-all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
+all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
 
 # Set OpenMapTiles host
 OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
@@ -38,8 +38,9 @@ help:
 	@echo "  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar "
 	@echo " "
 	@echo "Hints for designers:"
-	@echo "  make start-postserve                 # start Postserver + Maputnik Editor [ see $(OMT_HOST):8088 ] "
+	@echo "  make maputnik-start                  # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]"
-	@echo "  make start-tileserver                # start klokantech/tileserver-gl     [ see $(OMT_HOST):8080 ] "
+	@echo "  make postserve-start                 # start dynamic tile server [ see $(OMT_HOST):8088 ]"
+	@echo "  make tileserver-start                # start klokantech/tileserver-gl     [ see $(OMT_HOST):8080 ]"
 	@echo " "
 	@echo "Hints for developers:"
 	@echo "  make                                 # build source code"
@@ -53,10 +54,12 @@ help:
 	@echo "  make psql-analyze                    # PostgreSQL: ANALYZE"
 	@echo "  make generate-qareports              # generate reports                                [./build/qareports]"
 	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers [./layers/...]"
+	@echo "  make tools-dev                       # start openmaptiles-tools /bin/bash terminal"
+	@echo "  make db-destroy                      # remove docker containers and PostgreSQL data volume"
+	@echo "  make db-start                        # start PostgreSQL, creating it if it doesn't exist"
+	@echo "  make db-stop                         # stop PostgreSQL database without destroying the data"
 	@echo "  make import-sql-dev                  # start import-sql /bin/bash terminal"
 	@echo "  make import-osm-dev                  # start import-osm /bin/bash terminal (imposm3)"
-	@echo "  make clean-docker                    # remove docker containers, PG data volume"
-	@echo "  make forced-clean-sql                # drop all PostgreSQL tables for clean environment"
 	@echo "  make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)"
 	@echo "  make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB"
 	@echo "  make remove-docker-images            # remove openmaptiles docker images"
@@ -80,15 +83,16 @@ build/openmaptiles.tm2source/data.yml: init-dirs
 build/mapping.yaml: init-dirs
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@
 
-build/tileset.sql: init-dirs
+.PHONY: build-sql
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@
+build-sql: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql
 
 .PHONY: clean
 clean:
 	rm -rf build
 
-.PHONY: clean-docker
+.PHONY: db-destroy
-clean-docker:
+db-destroy:
 	$(DOCKER_COMPOSE) down -v --remove-orphans
 	$(DOCKER_COMPOSE) rm -fv
 	docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm
@@ -105,6 +109,10 @@ db-stop:
 	@echo "Stopping PostgreSQL..."
 	$(DOCKER_COMPOSE) stop postgres
 
+.PHONY: list-geofabrik
+list-geofabrik:
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik
+
 OSM_SERVERS:=geofabrik osmfr bbbike
 ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS))
 OSM_SERVER=$(patsubst download-%,%,$@)
@@ -138,10 +146,6 @@ psql: db-start
 import-osm: db-start all
 	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm
 
-.PHONY: import-sql
-import-sql: db-start all
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
-
 .PHONY: import-osmsql
 import-osmsql: db-start all import-osm import-sql
 
@@ -153,17 +157,23 @@ import-data: db-start
 import-borders: db-start
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
 
+.PHONY: import-sql
+import-sql: db-start all
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
+
 .PHONY: generate-tiles
 ifneq ($(wildcard data/docker-compose-config.yml),)
   DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
 endif
 generate-tiles: init-dirs db-start all
 	rm -rf data/tiles.mbtiles
+	echo "Generating tiles ..."; \
 	$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
+	@echo "Updating generated tile metadata ..."
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles
 
-.PHONY: start-tileserver
+.PHONY: tileserver-start
-start-tileserver: init-dirs
+tileserver-start: init-dirs
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
@@ -183,29 +193,42 @@ start-tileserver: init-dirs
 	@echo " "
 	docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:8080 klokantech/tileserver-gl --port 8080
 
-.PHONY: start-postserve
+.PHONY: postserve-start
-start-postserve: db-start
+postserve-start: db-start
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Bring up postserve at $(OMT_HOST):8090"
+	@echo "*     --> can view it locally (use make maputnik-start)"
+	@echo "*     --> or can use https://maputnik.github.io/editor"
+	@echo "* "
+	@echo "*  set data source / TileJSON URL to http://localhost:8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
 	$(DOCKER_COMPOSE) up -d postserve
-	docker pull maputnik/editor
+
+.PHONY: postserve-stop
+postserve-stop:
+	$(DOCKER_COMPOSE) stop postserve
+
+.PHONY: maputnik-start
+maputnik-start: maputnik-stop postserve-start
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Start maputnik/editor "
 	@echo "*       ---> go to http://$(OMT_HOST):8088 "
-	@echo "*       ---> set 'data source' to http://$(OMT_HOST):8090"
+	@echo "*       ---> set data source / TileJSON URL to http://$(OMT_HOST):8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
-	-docker rm -f maputnik_editor
 	docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor
 
+.PHONY: maputnik-stop
+maputnik-stop:
+	-docker rm -f maputnik_editor
+
 .PHONY: generate-qareports
 generate-qareports:
 	./qa/run.sh
@@ -218,8 +241,8 @@ generate-devdoc: init-dirs
 			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
 			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
 
-.PHONY: import-sql-dev
+.PHONY: tools-dev
-import-sql-dev:
+tools-dev:
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash
 
 .PHONY: import-osm-dev
@@ -228,7 +251,7 @@ import-osm-dev:
 
 .PHONY: import-wikidata
 import-wikidata:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml
 
 .PHONY: psql-pg-stat-reset
 psql-pg-stat-reset:
@@ -272,7 +295,13 @@ list-docker-images:
 
 .PHONY: refresh-docker-images
 refresh-docker-images:
-	$(DOCKER_COMPOSE) pull --ignore-pull-failures
+	@if test "$(NO_REFRESH)"; then \
+		echo "Skipping docker image refresh" ;\
+	else \
+		echo "" ;\
+		echo "Refreshing docker images... Use NO_REFRESH=1 to skip." ;\
+		$(DOCKER_COMPOSE) pull --ignore-pull-failures ;\
+	fi
 
 .PHONY: remove-docker-images
 remove-docker-images:

docker-compose.yml

@@ -11,7 +11,7 @@ networks:
 services:
 
   postgres:
-    image: "openmaptiles/postgis:${TOOLS_VERSION}"
+    image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
     volumes:
       - pgdata:/var/lib/postgresql/data
     networks:
@@ -71,12 +71,16 @@ services:
       # Must match the version of this file (first line)
       # download-osm will use it when generating a composer file
       MAKE_DC_VERSION: "2.3"
+      # Allow DIFF_MODE to be overwritten from shell
+      DIFF_MODE: ${DIFF_MODE}
     networks:
       - postgres_conn
     volumes:
       - .:/tileset
       - ./data:/import
       - ./build:/sql
+      - ./build:/mapping
+      - ./cache:/cache
 
   generate-changed-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"

quickstart.sh

@@ -116,7 +116,7 @@ fi
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Stopping running services & removing old containers"
-make clean-docker
+make db-destroy
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -298,8 +298,8 @@ echo "We saved the log file to $log_file  ( for debugging ) You can compare with
 echo " "
 echo "Start experimenting! And check the QUICKSTART.MD file!"
 echo " "
-echo "*  Use   make start-postserve    to explore tile generation on request"
+echo "*  Use   make maputnik-start     to explore tile generation on request"
-echo "*  Use   make start-tileserver   to view pre-generated tiles"
+echo "*  Use   make tileserver-start   to view pre-generated tiles"
 echo " "
 echo "Available help commands (make help)  "
 make help