Use v4.1, move import-wikidata before import-sql (#805)

import-wikidata should run before import-sql, and v4.1 allows for that. Also it optimizes the wd_names table to just be a simple Wikidata ID -> labels lookup, with a proper index. Minor other changes: * `test-perf-null` target is now part of the Makefile * `./data/osmstat.txt` is no longer created * area download file now in this format: `${osm_area}-latest.osm.pbf`
2020-04-09 10:49:35 -04:00 · 2020-04-09 10:49:35 -04:00 · 4f5ea5f26e
commit 4f5ea5f26e
--- a/.env
+++ b/.env
@ -6,7 +6,7 @@ POSTGRES_PORT=5432
 QUICKSTART_MIN_ZOOM=0
 QUICKSTART_MAX_ZOOM=7
 DIFF_MODE=false
-TOOLS_VERSION=4.0.0
+TOOLS_VERSION=4.1.0

 BBOX=-180.0,-85.0511,180.0,85.0511
 MIN_ZOOM=0
--- a/50
+++ b/50
@ -20,16 +20,16 @@ help:
 	@echo "=============================================================================="
 	@echo " OpenMapTiles  https://github.com/openmaptiles/openmaptiles "
 	@echo "Hints for testing areas                "
-	@echo "  make download-geofabrik-list         # list actual geofabrik OSM extracts for download -> <<your-area>> "
-	@echo "  make list                            # list actual geofabrik OSM extracts for download -> <<your-area>> "
+	@echo "  make list-geofabrik                  # list actual geofabrik OSM extracts for download -> <<your-area>> "
 	@echo "  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar "
-	@echo "  "
+	@echo " "
 	@echo "Hints for designers:"
 	@echo "  make start-postserve                 # start Postserver + Maputnik Editor [ see localhost:8088 ] "
 	@echo "  make start-tileserver                # start klokantech/tileserver-gl [ see localhost:8080 ] "
-	@echo "  "
+	@echo " "
 	@echo "Hints for developers:"
 	@echo "  make                                 # build source code"
+	@echo "  make list-geofabrik                  # list actual geofabrik OSM extracts for download"
 	@echo "  make download-geofabrik area=albania # download OSM data from geofabrik, and create config file"
 	@echo "  make psql                            # start PostgreSQL console"
 	@echo "  make psql-list-tables                # list all PostgreSQL tables"
@ -88,16 +88,26 @@ db-stop:

 .PHONY: download-geofabrik
 download-geofabrik: init-dirs
-	@echo =============== download-geofabrik =======================
-	@echo Download area:   $(area)
-	@echo [[ example: make download-geofabrik area=albania ]]
-	@echo [[ list areas:  make download-geofabrik-list     ]]
-	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area)
-	ls -la ./data/$(area).*
-	@echo "Generated config file: ./data/docker-compose-config.yml"
-	@echo " "
-	cat ./data/docker-compose-config.yml
-	@echo " "
+	@if ! test "$(area)"; then \
+		echo "" ;\
+		echo "ERROR: Unable to download an area if area is not given." ;\
+		echo "Usage:" ;\
+		echo "  make download-geofabrik area=<area-id>" ;\
+		echo "" ;\
+		echo "Use   make list-geofabrik   to get a list of all available areas" ;\
+		echo "" ;\
+		exit 1 ;\
+	else \
+		echo "=============== download-geofabrik =======================" ;\
+		echo "Download area: $(area)" ;\
+		docker-compose run $(DC_OPTS) openmaptiles-tools bash -c \
+			'download-osm geofabrik $(area) \
+				--minzoom $$QUICKSTART_MIN_ZOOM \
+				--maxzoom $$QUICKSTART_MAX_ZOOM \
+				--make-dc /import/docker-compose-config.yml -- -d /import' ;\
+		ls -la ./data/$(area)* ;\
+		echo " " ;\
+	fi

 .PHONY: psql
 psql: db-start
@ -205,14 +215,6 @@ import-sql-dev:
 import-osm-dev:
 	docker-compose run $(DC_OPTS) import-osm /bin/bash

-# the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command
-.PHONY: list
-list: download-geofabrik-list
-
-.PHONY: download-geofabrik-list
-download-geofabrik-list:
-	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh
-
 .PHONY: import-wikidata
 import-wikidata:
 	docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml
@ -275,3 +277,7 @@ docker-unnecessary-clean:
 	@docker ps -a  | grep Exited | awk -F" " '{print $$1}' | xargs  --no-run-if-empty docker rm
 	@echo "Deleting unnecessary image(s)..."
 	@docker images | grep \<none\> | awk -F" " '{print $$3}' | xargs  --no-run-if-empty  docker rmi
+
+.PHONY: test-perf-null
+test-perf-null:
+	docker-compose run $(DC_OPTS) openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color
--- a/openmaptiles.yaml
+++ b/openmaptiles.yaml
@ -30,7 +30,7 @@ tileset:
    - am # Amharic
    - ar # Arabic
    - az # Azerbaijani, Latin
-    - be # Belarusian
+    - be # Belorussian
    - bg # Bulgarian
    - br # Breton, Latin
    - bs # Bosnian, Latin
--- a/quickstart.sh
+++ b/quickstart.sh
@ -27,7 +27,7 @@ if [ $# -eq 0 ]; then
 else
    osm_area=$1
 fi
-testdata="${osm_area}.osm.pbf"
+testdata="${osm_area}-latest.osm.pbf"

 ##  Min versions ...
 MIN_COMPOSE_VER=1.7.1
@ -36,11 +36,6 @@ STARTTIME=$(date +%s)
 STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
 githash=$( git rev-parse HEAD )

-# Options to run with docker and docker-compose - ensure the container is destroyed on exit,
-# as well as pass any other common parameters.
-# In the future this should use -u $(id -u "$USER"):$(id -g "$USER") instead of running docker as root.
-DC_OPTS="--rm -u $(id -u "$USER"):$(id -g "$USER")"
-
 log_file=./quickstart.log
 rm -f $log_file
 echo " "
@ -138,21 +133,12 @@ echo "--------------------------------------------------------------------------
 echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) "
 rm -f ./data/*.mbtiles

-if [ !  -f "./data/${testdata}" ]; then
+if [[ ! -f "./data/${testdata}" || ! -f "./data/docker-compose-config.yml" ]]; then
    echo " "
    echo "-------------------------------------------------------------------------------------"
-    echo "====> : Downloading testdata $testdata"
-    rm -f ./data/*
-    #wget $testdataurl  -P ./data
+    echo "====> : Downloading ${osm_area} from Geofabrik..."
+    rm -rf ./data/*
    make download-geofabrik "area=${osm_area}"
-    echo " "
-    echo "-------------------------------------------------------------------------------------"
-    echo "====> : Osm metadata : $testdata"
-    cat ./data/osmstat.txt
-    echo " "
-    echo "-------------------------------------------------------------------------------------"
-    echo "====> : Generated docker-compose config"
-    cat ./data/docker-compose-config.yml
 else
    echo " "
    echo "-------------------------------------------------------------------------------------"
@ -236,7 +222,14 @@ make import-borders

 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Start SQL postprocessing:  ./build/tileset.sql -> PostgreSQL "
+echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
+echo "      : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
+echo "      : Thank you Wikidata Contributors ! "
+make import-wikidata
+
+echo " "
+echo "-------------------------------------------------------------------------------------"
+echo "====> : Start SQL postprocessing:  ./build/sql/* -> PostgreSQL "
 echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql"
 # If the output contains a WARNING, stop further processing
 # Adapted from https://unix.stackexchange.com/questions/307562
@ -248,17 +241,10 @@ echo "--------------------------------------------------------------------------
 echo "====> : Analyze PostgreSQL tables"
 make psql-analyze

-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
-echo "      : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
-echo "      : Thank you Wikidata Contributors ! "
-make import-wikidata
-
 echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
-docker-compose run $DC_OPTS openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color
+make test-perf-null

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -289,7 +275,6 @@ md5sum build/openmaptiles.tm2source/data.yml  >> ./data/quickstart_checklist.chk
 md5sum "./data/${testdata}"                   >> ./data/quickstart_checklist.chk
 md5sum ./data/tiles.mbtiles                   >> ./data/quickstart_checklist.chk
 md5sum ./data/docker-compose-config.yml       >> ./data/quickstart_checklist.chk
-md5sum ./data/osmstat.txt                     >> ./data/quickstart_checklist.chk
 cat ./data/quickstart_checklist.chk

 ENDTIME=$(date +%s)