From 2ca55abb7d8b2be322cd5fdab50966e6ec1872fb Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan <yuriastrakhan@gmail.com>
Date: Wed, 22 Apr 2020 02:48:57 -0400
Subject: [PATCH 1/6] Use new import-data image (#818)

This is a partial migration of https://github.com/openmaptiles/openmaptiles/pull/785

* Use `import-data` instead of `import-lakelines`, `import-water`, and `import-natural-earth`
* Upgrade docker-compose.yml to version 2.3 (allows some extra env var usage in yaml file itself)
* Remove `openmaptiles-tools:latest` usage -- no longer needed, can use current version 4.1
* `db-start` does not do a container recreation in case docker-compose.yml definition has changed.
* a few minor cleanups in quickstart.sh
---
 .env                         |  6 +++-
 .github/workflows/omt_ci.yml | 10 +++----
 Makefile                     | 25 +++++++----------
 README.md                    |  6 ++--
 docker-compose.yml           | 51 +++++++++++++++-------------------
 quickstart.sh                | 54 +++++++++++++-----------------------
 6 files changed, 65 insertions(+), 87 deletions(-)

diff --git a/.env b/.env
index aa66d0c5..70d706a0 100644
--- a/.env
+++ b/.env
@@ -1,12 +1,16 @@
+# This file defines default environment variables for all images
+
+TOOLS_VERSION=4.1.0
+
 POSTGRES_DB=openmaptiles
 POSTGRES_USER=openmaptiles
 POSTGRES_PASSWORD=openmaptiles
 POSTGRES_HOST=postgres
 POSTGRES_PORT=5432
+
 QUICKSTART_MIN_ZOOM=0
 QUICKSTART_MAX_ZOOM=7
 DIFF_MODE=false
-TOOLS_VERSION=4.1.0
 
 BBOX=-180.0,-85.0511,180.0,85.0511
 MIN_ZOOM=0
diff --git a/.github/workflows/omt_ci.yml b/.github/workflows/omt_ci.yml
index a2230d5f..8728bf0a 100644
--- a/.github/workflows/omt_ci.yml
+++ b/.github/workflows/omt_ci.yml
@@ -1,4 +1,4 @@
-# MapTiler OpenMapTiles 
+# MapTiler OpenMapTiles
 #######################
 
 # Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
@@ -28,16 +28,16 @@ jobs:
     # Named steps
     - name: generate all zooms
       run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
-    
-    # Runs quickstart  
+
+    # Runs quickstart
     - name: quickstart
       env:
         area: northamptonshire
       run:  bash ./quickstart.sh $area
-      
+
     - name: generate devdoc
       run: TEST_MODE=yes make generate-devdoc
-    
+
     # todo: use artifact to store result of tests
     #- uses: actions/upload-artifact@v1
     #  with:
diff --git a/Makefile b/Makefile
index 34d323ca..5462580e 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,9 @@ help:
 
 .PHONY: init-dirs
 init-dirs:
-	mkdir -p build && mkdir -p data && mkdir -p cache
+	@mkdir -p build
+	@mkdir -p data
+	@mkdir -p cache
 
 build/openmaptiles.tm2source/data.yml: init-dirs
 	mkdir -p build/openmaptiles.tm2source
@@ -94,12 +96,13 @@ clean-docker:
 
 .PHONY: db-start
 db-start:
-	$(DOCKER_COMPOSE) up -d postgres
+	$(DOCKER_COMPOSE) up --no-recreate -d postgres
 	@echo "Wait for PostgreSQL to start..."
 	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh
 
 .PHONY: db-stop
 db-stop:
+	@echo "Stopping PostgreSQL..."
 	$(DOCKER_COMPOSE) stop postgres
 
 OSM_SERVERS:=geofabrik osmfr bbbike
@@ -142,22 +145,14 @@ import-sql: db-start all
 .PHONY: import-osmsql
 import-osmsql: db-start all import-osm import-sql
 
+.PHONY: import-data
+import-data: db-start
+	$(DOCKER_COMPOSE) run $(DC_OPTS) import-data
+
 .PHONY: import-borders
 import-borders: db-start
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
 
-.PHONY: import-water
-import-water: db-start
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-water
-
-.PHONY: import-natural-earth
-import-natural-earth: db-start
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth
-
-.PHONY: import-lakelines
-import-lakelines: db-start
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines
-
 .PHONY: generate-tiles
 ifneq ($(wildcard data/docker-compose-config.yml),)
   DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
@@ -219,7 +214,7 @@ generate-qareports:
 .PHONY: generate-devdoc
 generate-devdoc: init-dirs
 	mkdir -p ./build/devdoc && \
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
 			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
 			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
 
diff --git a/README.md b/README.md
index 1da2a97a..5e34b48e 100644
--- a/README.md
+++ b/README.md
@@ -100,9 +100,7 @@ make db-start
 Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).
 
 ```bash
-make import-water
-make import-natural-earth
-make import-lakelines
+make import-data
 ```
 
 [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
@@ -119,7 +117,7 @@ make download-geofabrik area=albania
 ```
 
 [Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
-`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton.  Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
+`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition.  Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
 
 ```bash
 make import-osm
diff --git a/docker-compose.yml b/docker-compose.yml
index 41087d41..5e0552cc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,7 +1,15 @@
-version: "2"
+# This version must match the MAKE_DC_VERSION value below
+version: "2.3"
+
 volumes:
   pgdata:
+
+networks:
+  postgres_conn:
+    driver: bridge
+
 services:
+
   postgres:
     image: "openmaptiles/postgis:${TOOLS_VERSION}"
     volumes:
@@ -11,21 +19,13 @@ services:
     ports:
       - "5432"
     env_file: .env
-  import-natural-earth:
-    image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-water:
-    image: "openmaptiles/import-water:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-lakelines:
-    image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
+
+  import-data:
+    image: "openmaptiles/import-data:${TOOLS_VERSION}"
     env_file: .env
     networks:
       - postgres_conn
+
   import-osm:
     image: "openmaptiles/import-osm:${TOOLS_VERSION}"
     env_file: .env
@@ -37,6 +37,7 @@ services:
       - ./data:/import
       - ./build:/mapping
       - ./cache:/cache
+
   import-osm-diff:
     image: "openmaptiles/import-osm:${TOOLS_VERSION}"
     env_file: .env
@@ -49,6 +50,7 @@ services:
       - ./data:/import
       - ./build:/mapping
       - ./cache:/cache
+
   update-osm:
     image: "openmaptiles/import-osm:${TOOLS_VERSION}"
     env_file: .env
@@ -61,26 +63,21 @@ services:
       - ./data:/import
       - ./build:/mapping
       - ./cache:/cache
+
   openmaptiles-tools:
     image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
     env_file: .env
+    environment:
+      # Must match the version of this file (first line)
+      # download-osm will use it when generating a composer file
+      MAKE_DC_VERSION: "2.3"
     networks:
       - postgres_conn
     volumes:
       - .:/tileset
       - ./data:/import
       - ./build:/sql
-  openmaptiles-tools-latest:
-    # This target exists for experimental tools that have not yet been published.
-    # Do not use this for production.
-    image: "openmaptiles/openmaptiles-tools:latest"
-    env_file: .env
-    networks:
-      - postgres_conn
-    volumes:
-      - .:/tileset
-      - ./data:/import
-      - ./build:/sql
+
   generate-changed-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
     command: ./export-list.sh
@@ -90,6 +87,7 @@ services:
     networks:
       - postgres_conn
     env_file: .env
+
   generate-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
     volumes:
@@ -102,6 +100,7 @@ services:
       BBOX: ${BBOX}
       MIN_ZOOM: ${MIN_ZOOM}
       MAX_ZOOM: ${MAX_ZOOM}
+
   postserve:
     image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
     command: postserve openmaptiles.yaml --verbose
@@ -112,7 +111,3 @@ services:
       - "8090:8090"
     volumes:
       - .:/tileset
-
-networks:
-  postgres_conn:
-    driver: bridge
diff --git a/quickstart.sh b/quickstart.sh
index 4af930b6..4e158769 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc  )
     echo "system memory (GB): ${mem}"
     grep SwapTotal /proc/meminfo
-    echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)"
-    cat /proc/meminfo  | grep Free
+    echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
+    grep Free /proc/meminfo
 else
     echo " "
     echo "Warning : Platforms other than Linux are less tested"
@@ -120,7 +120,7 @@ make clean-docker
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Checking OpenMapTiles docker images "
+echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")"
 docker images | grep openmaptiles
 
 echo " "
@@ -162,7 +162,7 @@ echo "--------------------------------------------------------------------------
 echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
 echo "      : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
 echo "      : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
-make
+make all
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL  public schema "
 # This adds an extra safety belt if the user modifies the docker volume settings
 make forced-clean-sql
 
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL "
-echo "      : Source code:  https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water "
-echo "      : Data license: https://osmdata.openstreetmap.de/info/license.html "
-echo "      : Thank you: https://osmdata.openstreetmap.de/info/ "
-make import-water
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing  http://www.naturalearthdata.com  into PostgreSQL "
-echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth "
-echo "      : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use  "
-echo "      : Thank you: Natural Earth Contributors! "
-make import-natural-earth
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing OpenStreetMap Lakelines data "
-echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines "
-echo "      :              https://github.com/lukasmartinelli/osm-lakelines "
-echo "      : Data license: .. "
-make import-lakelines
+echo "====> : Importing all the data:"
+echo "      : * Water data from http://osmdata.openstreetmap.de"
+echo "      :   Data license: https://osmdata.openstreetmap.de/info/license.html"
+echo "      : * Natural Earth from http://www.naturalearthdata.com"
+echo "      :   Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
+echo "      : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
+echo "      :"
+echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
+echo "      :   includes all data from the import-data image"
+echo "      :"
+echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
+make import-data
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -269,16 +258,13 @@ echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Inputs - Outputs md5sum for debugging "
 rm -f ./data/quickstart_checklist.chk
-md5sum build/mapping.yaml                     >> ./data/quickstart_checklist.chk
-md5sum build/tileset.sql                      >> ./data/quickstart_checklist.chk
-md5sum build/openmaptiles.tm2source/data.yml  >> ./data/quickstart_checklist.chk
-md5sum "./data/${testdata}"                   >> ./data/quickstart_checklist.chk
-md5sum ./data/tiles.mbtiles                   >> ./data/quickstart_checklist.chk
-md5sum ./data/docker-compose-config.yml       >> ./data/quickstart_checklist.chk
+{
+  find build -type f | sort | xargs md5sum ;
+  find data -type f | sort | xargs md5sum ;
+} >> ./data/quickstart_checklist.chk
 cat ./data/quickstart_checklist.chk
 
 ENDTIME=$(date +%s)
-ENDDATE=$(date +"%Y-%m-%dT%H:%M%z")
 if stat --help >/dev/null 2>&1; then
   MODDATE=$(stat -c %y "./data/${testdata}" )
 else

From 8f9770e54692d84695b9b1daf6b90d3e92d2e226 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= <fred.rodrigo@gmail.com>
Date: Wed, 22 Apr 2020 09:03:10 +0200
Subject: [PATCH 2/6] Reorder POI data update and trigger creation to avoid
 refresh of materialized view on initial import (#780)

Reorder POI data update and trigger creation to avoid refresh of materialized view after update done by initial import.

I checked the other updates and there are OK.
---
 layers/poi/poi.yaml             | 4 ++--
 layers/poi/poi_stop_agg.sql     | 3 ---
 layers/poi/update_poi_point.sql | 3 +++
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/layers/poi/poi.yaml b/layers/poi/poi.yaml
index d678bd65..ed49fea7 100644
--- a/layers/poi/poi.yaml
+++ b/layers/poi/poi.yaml
@@ -146,10 +146,10 @@ layer:
     query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, subclass, agg_stop, layer, level, indoor, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
 schema:
   - ./public_transport_stop_type.sql
-  - ./update_poi_polygon.sql
-  - ./update_poi_point.sql
   - ./class.sql
   - ./poi_stop_agg.sql
+  - ./update_poi_polygon.sql
+  - ./update_poi_point.sql
   - ./layer.sql
 datasources:
   - type: imposm3
diff --git a/layers/poi/poi_stop_agg.sql b/layers/poi/poi_stop_agg.sql
index 788bcda6..4e4f40ac 100644
--- a/layers/poi/poi_stop_agg.sql
+++ b/layers/poi/poi_stop_agg.sql
@@ -33,6 +33,3 @@ CREATE MATERIALIZED VIEW osm_poi_stop_rank AS (
 		subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway')
 	ORDER BY p.uic_ref, rk
 ) /* DELAY_MATERIALIZED_VIEW_CREATION */;
-
-ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL;
-SELECT update_osm_poi_point_agg();
diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql
index dddec1d8..bd6aeb41 100644
--- a/layers/poi/update_poi_point.sql
+++ b/layers/poi/update_poi_point.sql
@@ -43,6 +43,9 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
+ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL;
+SELECT update_osm_poi_point_agg();
+
 -- Handle updates
 
 CREATE SCHEMA IF NOT EXISTS poi_point;

From 21053e4a5fcc116346f9f97489026c5bf786b461 Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan <yuriastrakhan@gmail.com>
Date: Wed, 22 Apr 2020 11:15:20 -0400
Subject: [PATCH 3/6] Makefile target cleanup, minor improvements (#821)

* allow postgres image to be overwritten with an env var
* allow DIFF_MODE var to be overwritten with an env var
* add /mapping and /cache dirs into tools image
* make `build-sql` target explicit rather than relying on a filename
* `tools-dev` will open a shell in a docker to experiment and debug (instead of `import-sql-dev` and `import-osm-dev`)
* separated `start-maputnik` from `start-postserve`
* renamed `clean-docker` into `db-destroy` to make it more explicit
* cleaner `db-start`, `db-stop`, `db-destroy` targets
* better output messages
---
 Makefile           | 77 +++++++++++++++++++++++++++++++---------------
 docker-compose.yml |  6 +++-
 quickstart.sh      |  6 ++--
 3 files changed, 61 insertions(+), 28 deletions(-)

diff --git a/Makefile b/Makefile
index 5462580e..83753140 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ else
 endif
 
 .PHONY: all
-all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
+all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
 
 # Set OpenMapTiles host
 OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
@@ -38,8 +38,9 @@ help:
 	@echo "  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar "
 	@echo " "
 	@echo "Hints for designers:"
-	@echo "  make start-postserve                 # start Postserver + Maputnik Editor [ see $(OMT_HOST):8088 ] "
-	@echo "  make start-tileserver                # start klokantech/tileserver-gl     [ see $(OMT_HOST):8080 ] "
+	@echo "  make maputnik-start                  # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]"
+	@echo "  make postserve-start                 # start dynamic tile server [ see $(OMT_HOST):8088 ]"
+	@echo "  make tileserver-start                # start klokantech/tileserver-gl     [ see $(OMT_HOST):8080 ]"
 	@echo " "
 	@echo "Hints for developers:"
 	@echo "  make                                 # build source code"
@@ -53,10 +54,12 @@ help:
 	@echo "  make psql-analyze                    # PostgreSQL: ANALYZE"
 	@echo "  make generate-qareports              # generate reports                                [./build/qareports]"
 	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers [./layers/...]"
+	@echo "  make tools-dev                       # start openmaptiles-tools /bin/bash terminal"
+	@echo "  make db-destroy                      # remove docker containers and PostgreSQL data volume"
+	@echo "  make db-start                        # start PostgreSQL, creating it if it doesn't exist"
+	@echo "  make db-stop                         # stop PostgreSQL database without destroying the data"
 	@echo "  make import-sql-dev                  # start import-sql /bin/bash terminal"
 	@echo "  make import-osm-dev                  # start import-osm /bin/bash terminal (imposm3)"
-	@echo "  make clean-docker                    # remove docker containers, PG data volume"
-	@echo "  make forced-clean-sql                # drop all PostgreSQL tables for clean environment"
 	@echo "  make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)"
 	@echo "  make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB"
 	@echo "  make remove-docker-images            # remove openmaptiles docker images"
@@ -80,15 +83,16 @@ build/openmaptiles.tm2source/data.yml: init-dirs
 build/mapping.yaml: init-dirs
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@
 
-build/tileset.sql: init-dirs
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@
+.PHONY: build-sql
+build-sql: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql
 
 .PHONY: clean
 clean:
 	rm -rf build
 
-.PHONY: clean-docker
-clean-docker:
+.PHONY: db-destroy
+db-destroy:
 	$(DOCKER_COMPOSE) down -v --remove-orphans
 	$(DOCKER_COMPOSE) rm -fv
 	docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm
@@ -105,6 +109,10 @@ db-stop:
 	@echo "Stopping PostgreSQL..."
 	$(DOCKER_COMPOSE) stop postgres
 
+.PHONY: list-geofabrik
+list-geofabrik:
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik
+
 OSM_SERVERS:=geofabrik osmfr bbbike
 ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS))
 OSM_SERVER=$(patsubst download-%,%,$@)
@@ -138,10 +146,6 @@ psql: db-start
 import-osm: db-start all
 	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm
 
-.PHONY: import-sql
-import-sql: db-start all
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
-
 .PHONY: import-osmsql
 import-osmsql: db-start all import-osm import-sql
 
@@ -153,17 +157,23 @@ import-data: db-start
 import-borders: db-start
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
 
+.PHONY: import-sql
+import-sql: db-start all
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
+
 .PHONY: generate-tiles
 ifneq ($(wildcard data/docker-compose-config.yml),)
   DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
 endif
 generate-tiles: init-dirs db-start all
 	rm -rf data/tiles.mbtiles
+	echo "Generating tiles ..."; \
 	$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
+	@echo "Updating generated tile metadata ..."
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles
 
-.PHONY: start-tileserver
-start-tileserver: init-dirs
+.PHONY: tileserver-start
+tileserver-start: init-dirs
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
@@ -183,29 +193,42 @@ start-tileserver: init-dirs
 	@echo " "
 	docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:8080 klokantech/tileserver-gl --port 8080
 
-.PHONY: start-postserve
-start-postserve: db-start
+.PHONY: postserve-start
+postserve-start: db-start
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Bring up postserve at $(OMT_HOST):8090"
+	@echo "*     --> can view it locally (use make maputnik-start)"
+	@echo "*     --> or can use https://maputnik.github.io/editor"
+	@echo "* "
+	@echo "*  set data source / TileJSON URL to http://localhost:8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
 	$(DOCKER_COMPOSE) up -d postserve
-	docker pull maputnik/editor
+
+.PHONY: postserve-stop
+postserve-stop:
+	$(DOCKER_COMPOSE) stop postserve
+
+.PHONY: maputnik-start
+maputnik-start: maputnik-stop postserve-start
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Start maputnik/editor "
 	@echo "*       ---> go to http://$(OMT_HOST):8088 "
-	@echo "*       ---> set 'data source' to http://$(OMT_HOST):8090"
+	@echo "*       ---> set data source / TileJSON URL to http://$(OMT_HOST):8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
-	-docker rm -f maputnik_editor
 	docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor
 
+.PHONY: maputnik-stop
+maputnik-stop:
+	-docker rm -f maputnik_editor
+
 .PHONY: generate-qareports
 generate-qareports:
 	./qa/run.sh
@@ -218,8 +241,8 @@ generate-devdoc: init-dirs
 			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
 			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
 
-.PHONY: import-sql-dev
-import-sql-dev:
+.PHONY: tools-dev
+tools-dev:
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash
 
 .PHONY: import-osm-dev
@@ -228,7 +251,7 @@ import-osm-dev:
 
 .PHONY: import-wikidata
 import-wikidata:
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml
 
 .PHONY: psql-pg-stat-reset
 psql-pg-stat-reset:
@@ -272,7 +295,13 @@ list-docker-images:
 
 .PHONY: refresh-docker-images
 refresh-docker-images:
-	$(DOCKER_COMPOSE) pull --ignore-pull-failures
+	@if test "$(NO_REFRESH)"; then \
+		echo "Skipping docker image refresh" ;\
+	else \
+		echo "" ;\
+		echo "Refreshing docker images... Use NO_REFRESH=1 to skip." ;\
+		$(DOCKER_COMPOSE) pull --ignore-pull-failures ;\
+	fi
 
 .PHONY: remove-docker-images
 remove-docker-images:
diff --git a/docker-compose.yml b/docker-compose.yml
index 5e0552cc..75c479f1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,7 +11,7 @@ networks:
 services:
 
   postgres:
-    image: "openmaptiles/postgis:${TOOLS_VERSION}"
+    image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
     volumes:
       - pgdata:/var/lib/postgresql/data
     networks:
@@ -71,12 +71,16 @@ services:
       # Must match the version of this file (first line)
       # download-osm will use it when generating a composer file
       MAKE_DC_VERSION: "2.3"
+      # Allow DIFF_MODE to be overwritten from shell
+      DIFF_MODE: ${DIFF_MODE}
     networks:
       - postgres_conn
     volumes:
       - .:/tileset
       - ./data:/import
       - ./build:/sql
+      - ./build:/mapping
+      - ./cache:/cache
 
   generate-changed-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
diff --git a/quickstart.sh b/quickstart.sh
index 4e158769..01083438 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -116,7 +116,7 @@ fi
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Stopping running services & removing old containers"
-make clean-docker
+make db-destroy
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -298,8 +298,8 @@ echo "We saved the log file to $log_file  ( for debugging ) You can compare with
 echo " "
 echo "Start experimenting! And check the QUICKSTART.MD file!"
 echo " "
-echo "*  Use   make start-postserve    to explore tile generation on request"
-echo "*  Use   make start-tileserver   to view pre-generated tiles"
+echo "*  Use   make maputnik-start     to explore tile generation on request"
+echo "*  Use   make tileserver-start   to view pre-generated tiles"
 echo " "
 echo "Available help commands (make help)  "
 make help

From 95ddc342957db33276d367d9826049824fdeee14 Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan <yuriastrakhan@gmail.com>
Date: Wed, 22 Apr 2020 14:55:13 -0400
Subject: [PATCH 4/6] Minor makefile cleanup followup #821 (#823)

Per @zstadler recommendations in
https://github.com/openmaptiles/openmaptiles/commit/21053e4a5fcc116346f9f97489026c5bf786b461
---
 Makefile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 83753140..626395c8 100644
--- a/Makefile
+++ b/Makefile
@@ -202,7 +202,7 @@ postserve-start: db-start
 	@echo "*     --> can view it locally (use make maputnik-start)"
 	@echo "*     --> or can use https://maputnik.github.io/editor"
 	@echo "* "
-	@echo "*  set data source / TileJSON URL to http://localhost:8090"
+	@echo "*  set data source / TileJSON URL to http://$(OMT_HOST):8090"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
@@ -295,13 +295,13 @@ list-docker-images:
 
 .PHONY: refresh-docker-images
 refresh-docker-images:
-	@if test "$(NO_REFRESH)"; then \
-		echo "Skipping docker image refresh" ;\
-	else \
-		echo "" ;\
-		echo "Refreshing docker images... Use NO_REFRESH=1 to skip." ;\
-		$(DOCKER_COMPOSE) pull --ignore-pull-failures ;\
-	fi
+ifneq ($(strip $(NO_REFRESH)),)
+	@echo "Skipping docker image refresh"
+else
+	@echo ""
+	@echo "Refreshing docker images... Use NO_REFRESH=1 to skip."
+	$(DOCKER_COMPOSE) pull --ignore-pull-failures
+endif
 
 .PHONY: remove-docker-images
 remove-docker-images:

From 5da406f4a291bd07c52da9b1211a6aa10f42812e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= <fred.rodrigo@gmail.com>
Date: Thu, 23 Apr 2020 08:23:52 +0200
Subject: [PATCH 5/6] Filter osm_building_polygon on polygon (#758)

---
 layers/building/building.sql | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/layers/building/building.sql b/layers/building/building.sql
index 95f729c6..81f39fdb 100644
--- a/layers/building/building.sql
+++ b/layers/building/building.sql
@@ -72,7 +72,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS (
                   FALSE as hide_3d
          FROM
          osm_building_polygon obp
-         WHERE osm_id < 0
+         -- OSM mulipolygons once imported can give unique postgis polygons with holes, or multi parts polygons
+         WHERE osm_id < 0 AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon')
 
          UNION ALL
          -- etldoc: osm_building_polygon -> layer_building:z14_
@@ -88,7 +89,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS (
          FROM
          osm_building_polygon obp
            LEFT JOIN osm_building_relation obr ON (obr.member = obp.osm_id)
-         WHERE obp.osm_id >= 0
+         -- Only check for ST_Polygon as we exclude buildings from relations keeping only positive ids
+         WHERE obp.osm_id >= 0 AND ST_GeometryType(obp.geometry) = 'ST_Polygon'
 );
 
 CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int)

From d7873db0c0dae07cea6c47e60190a3b950740aab Mon Sep 17 00:00:00 2001
From: Yuri Astrakhan <yuriastrakhan@gmail.com>
Date: Fri, 24 Apr 2020 09:20:59 -0400
Subject: [PATCH 6/6] Implement PR performance testing (#824)

* On pull request and on commit, run base test followed by the test of the change,
comparing the results, and publishing the results to the Pull Request.
If the pull request is updated, the resulting comment will be updated.

* also save quickstart.log as an artifact

Note that due to GitHub workflow security restrictions, it is not possible to post PR comments if the change originated from a fork.  I am still looking for workarounds.

To view what would have been posted, in the build results at the bottom, open `PR performance` details, and expand the ` Comment on Pull Request` (and its subitem).

Optimizations: the process keeps two caches -- one for the data test file, and one for the results of the performance run for the "base" revision.  If this or other PR has been executed for the same revision and the same test data, performance test will only run for the proposed changes, not for the base.

Co-authored-by: Tomas Pohanka <TomPohys@gmail.com>
---
 .github/workflows/omt_ci.yml |  45 --------
 .github/workflows/tests.yml  | 200 +++++++++++++++++++++++++++++++++++
 2 files changed, 200 insertions(+), 45 deletions(-)
 delete mode 100644 .github/workflows/omt_ci.yml
 create mode 100644 .github/workflows/tests.yml

diff --git a/.github/workflows/omt_ci.yml b/.github/workflows/omt_ci.yml
deleted file mode 100644
index 8728bf0a..00000000
--- a/.github/workflows/omt_ci.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-# MapTiler OpenMapTiles
-#######################
-
-# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
-
-name: OMT_CI
-
-# Controls when the action will run. Triggers the workflow on push and pull request
-# events but only for the master branch
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-# jobs can run parallel
-jobs:
-  # This workflow contains a single job called "build"
-  build:
-    # runs on ubuntu (can run on windows and mac os)
-    runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/checkout@v2
-
-    # Named steps
-    - name: generate all zooms
-      run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
-
-    # Runs quickstart
-    - name: quickstart
-      env:
-        area: northamptonshire
-      run:  bash ./quickstart.sh $area
-
-    - name: generate devdoc
-      run: TEST_MODE=yes make generate-devdoc
-
-    # todo: use artifact to store result of tests
-    #- uses: actions/upload-artifact@v1
-    #  with:
-    #    name: quickstart log file
-    #    path: quickstart.log
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 00000000..886ec472
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,200 @@
+# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+
+  integrity_test:
+    name: Run integrity test
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout the changes
+        uses: actions/checkout@v2
+
+      - name: Run quickstart for a small area
+        env:
+          area: monaco
+        run: |
+          # For now, change the quickstart values directly in the .env file
+          # TODO: We should probably use env vars instead
+          sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
+          ./quickstart.sh $area
+
+      - name: Save quickstart.log
+        uses: actions/upload-artifact@v1
+        with:
+          name: quickstart.log
+          path: quickstart.log
+
+      - name: Test etldoc images
+        run: |
+          export TEST_MODE=yes
+          make generate-devdoc
+
+  performance:
+    name: Evaluate performance
+    runs-on: ubuntu-latest
+    # Even though we technically don't need to wait for integrity test to finish,
+    # there is no point to run long perf test until we know the code is OK
+    needs: integrity_test
+    env:
+      # Smaller tests (runs everything in about 30 minutes)
+      TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test equatorial-guinea --test liechtenstein"
+      TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=12vw07f9W0MiAHIqMztRiIMwahJfqTi21"
+
+        ## Large test data -- we should switch to it after everything is working ok
+        # TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test hungary --test isle-of-man"
+        # TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=1kw7XPDPd1Rc-Zi2XxGLTXdinUSq-S4pT"
+    steps:
+      - name: Cache test data download
+        id: cache-testdata
+        uses: actions/cache@v1
+        with:
+          path: ci_cache
+          key: "${{ env.TEST_DATA_URL }}"
+
+      - name: Download test data on cache miss
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        run: |
+          echo "Data file does not exist, downloading $TEST_DATA_URL"
+          mkdir -p ci_cache
+          curl --silent --show-error --location --output ci_cache/perf-test-areas-latest.osm.pbf "$TEST_DATA_URL"
+
+      - name: Get code
+        uses: actions/checkout@v2
+        with:
+          # Fetch the last two commits in case this is a PR,
+          # and we need to profile the base branch first
+          fetch-depth: 2
+          path: code
+
+      - name: Compute git revision hash to cache
+        id: calc
+        run: |
+          # If this is a pull request, we should cache the parent (base) revision
+          # Otherwise cache the current one
+          cd code
+          REV_HASH=$(git log -1 --format="%H")
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            # Take the first parent of the grafted commit (cannot use HEAD^1 with shallow clones)
+            REV_HASH=$(git cat-file -p $REV_HASH | awk 'NR > 1 {if(/^parent/){print $2; exit}}')
+          fi
+          echo "::set-output name=hash::$REV_HASH"
+
+      - name: Set up caching for the performance results
+        uses: actions/cache@v1
+        with:
+          path: perf_cache
+          key: "${{ steps.calc.outputs.hash }}-${{ env.TEST_DATA_URL }}"
+
+      - name: Load test data into DB and run performance test
+        id: main
+        env:
+          CACHE_SHA: "${{ steps.calc.outputs.hash }}"
+        run: |
+          create_db() {
+            make clean
+            make init-dirs
+            cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas-latest.osm.pbf
+            make db-destroy
+            make all
+            make db-start
+            time make import-data
+            time make import-osm
+            time make import-borders
+            time make import-wikidata
+            time make import-sql
+          }
+
+          mkdir -p perf_cache
+          mkdir -p artifacts
+          cd code
+
+          CURRENT_SHA=$(git log -1 --format="%H")
+
+          if [ ! -f ../perf_cache/results.json ]; then
+            echo "We do not have cached performance results, create them..."
+            if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+              git reset --hard ${CURRENT_SHA}^1
+            fi
+
+            create_db
+
+            # Use latest tools version because these specific tests do not yet exist in the 4.1 tools version
+            # Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version
+            TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \
+              test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
+              --record /tileset/results.json
+            mv results.json ../perf_cache
+
+            if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+              # For pull requests, restore to the PR version before continuing
+              git reset --hard ${CURRENT_SHA}
+            fi
+          else
+            echo "Found cached performance results"
+          fi
+
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            cp ../perf_cache/results.json ../artifacts/base-results.json
+          else
+            cp ../perf_cache/results.json ../artifacts/results.json
+          fi
+
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            echo "Comparing pull request results with the base..."
+
+            create_db
+
+            # Use latest tools version because these specific tests do not yet exist in the 4.1 tools version
+            # Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version
+            cp ../perf_cache/results.json .
+            OUTPUT="$(TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \
+                        test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
+                        --compare /tileset/results.json --record /tileset/pr-results.json)"
+            rm results.json
+            mv pr-results.json ../artifacts/
+
+            # Convert multiline output into a single long string.
+            # See https://github.community/t5/GitHub-Actions/set-output-Truncates-Multiline-Strings/td-p/37870
+            OUTPUT="${OUTPUT//'%'/'%25'}"
+            OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+            OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+
+            # Split into two parts -- before and after the ===== SUMMARY =====
+            echo "::set-output name=summary::${OUTPUT##*========}"
+            echo "::set-output name=details::${OUTPUT%%========*}"
+          fi
+
+      - name: Save artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: performance_results
+          path: artifacts
+
+      - name: Post a comment on Pull Request
+        if: "github.event_name == 'pull_request'"
+        uses: marocchino/sticky-pull-request-comment@v1
+        timeout-minutes: 1
+        continue-on-error: true
+        with:
+          message: |-
+            ```
+            ${{ steps.main.outputs.summary }}
+            ```
+
+            <details>
+            <summary>expand for details...</summary>
+
+            ```
+            ${{ steps.main.outputs.details }}
+            ```
+
+            </details>
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}