Use v4.1, move import-wikidata before import-sql (#805)

import-wikidata should run before import-sql, and v4.1 allows for that.
Also it optimizes the wd_names table to just be a simple Wikidata ID -> labels lookup, with a proper index.

Minor other changes:
* `test-perf-null` target is now part of the Makefile
* `./data/osmstat.txt` is no longer created
* area download file now in this format:   `${osm_area}-latest.osm.pbf`

.env

@@ -6,7 +6,7 @@ POSTGRES_PORT=5432
 QUICKSTART_MIN_ZOOM=0
 QUICKSTART_MAX_ZOOM=7
 DIFF_MODE=false
-TOOLS_VERSION=4.0.0
+TOOLS_VERSION=4.1.0
 
 BBOX=-180.0,-85.0511,180.0,85.0511
 MIN_ZOOM=0

Makefile

@@ -20,16 +20,16 @@ help:
 	@echo "=============================================================================="
 	@echo " OpenMapTiles  https://github.com/openmaptiles/openmaptiles "
 	@echo "Hints for testing areas                "
-	@echo "  make download-geofabrik-list         # list actual geofabrik OSM extracts for download -> <<your-area>> "
+	@echo "  make list-geofabrik                  # list actual geofabrik OSM extracts for download -> <<your-area>> "
-	@echo "  make list                            # list actual geofabrik OSM extracts for download -> <<your-area>> "
 	@echo "  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar "
-	@echo "  "
+	@echo " "
 	@echo "Hints for designers:"
 	@echo "  make start-postserve                 # start Postserver + Maputnik Editor [ see localhost:8088 ] "
 	@echo "  make start-tileserver                # start klokantech/tileserver-gl [ see localhost:8080 ] "
-	@echo "  "
+	@echo " "
 	@echo "Hints for developers:"
 	@echo "  make                                 # build source code"
+	@echo "  make list-geofabrik                  # list actual geofabrik OSM extracts for download"
 	@echo "  make download-geofabrik area=albania # download OSM data from geofabrik, and create config file"
 	@echo "  make psql                            # start PostgreSQL console"
 	@echo "  make psql-list-tables                # list all PostgreSQL tables"
@@ -88,16 +88,26 @@ db-stop:
 
 .PHONY: download-geofabrik
 download-geofabrik: init-dirs
-	@echo =============== download-geofabrik =======================
+	@if ! test "$(area)"; then \
-	@echo Download area:   $(area)
+		echo "" ;\
-	@echo [[ example: make download-geofabrik area=albania ]]
+		echo "ERROR: Unable to download an area if area is not given." ;\
-	@echo [[ list areas:  make download-geofabrik-list     ]]
+		echo "Usage:" ;\
-	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area)
+		echo "  make download-geofabrik area=<area-id>" ;\
-	ls -la ./data/$(area).*
+		echo "" ;\
-	@echo "Generated config file: ./data/docker-compose-config.yml"
+		echo "Use   make list-geofabrik   to get a list of all available areas" ;\
-	@echo " "
+		echo "" ;\
-	cat ./data/docker-compose-config.yml
+		exit 1 ;\
-	@echo " "
+	else \
+		echo "=============== download-geofabrik =======================" ;\
+		echo "Download area: $(area)" ;\
+		docker-compose run $(DC_OPTS) openmaptiles-tools bash -c \
+			'download-osm geofabrik $(area) \
+				--minzoom $$QUICKSTART_MIN_ZOOM \
+				--maxzoom $$QUICKSTART_MAX_ZOOM \
+				--make-dc /import/docker-compose-config.yml -- -d /import' ;\
+		ls -la ./data/$(area)* ;\
+		echo " " ;\
+	fi
 
 .PHONY: psql
 psql: db-start
@@ -205,14 +215,6 @@ import-sql-dev:
 import-osm-dev:
 	docker-compose run $(DC_OPTS) import-osm /bin/bash
 
-# the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command
-.PHONY: list
-list: download-geofabrik-list
-
-.PHONY: download-geofabrik-list
-download-geofabrik-list:
-	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh
-
 .PHONY: import-wikidata
 import-wikidata:
 	docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml
@@ -275,3 +277,7 @@ docker-unnecessary-clean:
 	@docker ps -a  | grep Exited | awk -F" " '{print $$1}' | xargs  --no-run-if-empty docker rm
 	@echo "Deleting unnecessary image(s)..."
 	@docker images | grep \<none\> | awk -F" " '{print $$3}' | xargs  --no-run-if-empty  docker rmi
+
+.PHONY: test-perf-null
+test-perf-null:
+	docker-compose run $(DC_OPTS) openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color

openmaptiles.yaml

@@ -30,7 +30,7 @@ tileset:
     - am # Amharic
     - ar # Arabic
     - az # Azerbaijani, Latin
-    - be # Belarusian
+    - be # Belorussian
     - bg # Bulgarian
     - br # Breton, Latin
     - bs # Bosnian, Latin

quickstart.sh

@@ -27,7 +27,7 @@ if [ $# -eq 0 ]; then
 else
     osm_area=$1
 fi
-testdata="${osm_area}.osm.pbf"
+testdata="${osm_area}-latest.osm.pbf"
 
 ##  Min versions ...
 MIN_COMPOSE_VER=1.7.1
@@ -36,11 +36,6 @@ STARTTIME=$(date +%s)
 STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
 githash=$( git rev-parse HEAD )
 
-# Options to run with docker and docker-compose - ensure the container is destroyed on exit,
-# as well as pass any other common parameters.
-# In the future this should use -u $(id -u "$USER"):$(id -g "$USER") instead of running docker as root.
-DC_OPTS="--rm -u $(id -u "$USER"):$(id -g "$USER")"
-
 log_file=./quickstart.log
 rm -f $log_file
 echo " "
@@ -138,21 +133,12 @@ echo "--------------------------------------------------------------------------
 echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) "
 rm -f ./data/*.mbtiles
 
-if [ !  -f "./data/${testdata}" ]; then
+if [[ ! -f "./data/${testdata}" || ! -f "./data/docker-compose-config.yml" ]]; then
     echo " "
     echo "-------------------------------------------------------------------------------------"
-    echo "====> : Downloading testdata $testdata"
+    echo "====> : Downloading ${osm_area} from Geofabrik..."
-    rm -f ./data/*
+    rm -rf ./data/*
-    #wget $testdataurl  -P ./data
     make download-geofabrik "area=${osm_area}"
-    echo " "
-    echo "-------------------------------------------------------------------------------------"
-    echo "====> : Osm metadata : $testdata"
-    cat ./data/osmstat.txt
-    echo " "
-    echo "-------------------------------------------------------------------------------------"
-    echo "====> : Generated docker-compose config"
-    cat ./data/docker-compose-config.yml
 else
     echo " "
     echo "-------------------------------------------------------------------------------------"
@@ -236,7 +222,14 @@ make import-borders
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Start SQL postprocessing:  ./build/tileset.sql -> PostgreSQL "
+echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
+echo "      : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
+echo "      : Thank you Wikidata Contributors ! "
+make import-wikidata
+
+echo " "
+echo "-------------------------------------------------------------------------------------"
+echo "====> : Start SQL postprocessing:  ./build/sql/* -> PostgreSQL "
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql"
 # If the output contains a WARNING, stop further processing
 # Adapted from https://unix.stackexchange.com/questions/307562
@@ -248,17 +241,10 @@ echo "--------------------------------------------------------------------------
 echo "====> : Analyze PostgreSQL tables"
 make psql-analyze
 
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
-echo "      : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
-echo "      : Thank you Wikidata Contributors ! "
-make import-wikidata
-
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
-docker-compose run $DC_OPTS openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color
+make test-perf-null
 
 echo " "
 echo "-------------------------------------------------------------------------------------"
@@ -289,7 +275,6 @@ md5sum build/openmaptiles.tm2source/data.yml  >> ./data/quickstart_checklist.chk
 md5sum "./data/${testdata}"                   >> ./data/quickstart_checklist.chk
 md5sum ./data/tiles.mbtiles                   >> ./data/quickstart_checklist.chk
 md5sum ./data/docker-compose-config.yml       >> ./data/quickstart_checklist.chk
-md5sum ./data/osmstat.txt                     >> ./data/quickstart_checklist.chk
 cat ./data/quickstart_checklist.chk
 
 ENDTIME=$(date +%s)