diff --git a/.env b/.env index 2e1d7997..c4156760 100644 --- a/.env +++ b/.env @@ -24,3 +24,14 @@ FILTER_MAPNIK_OUTPUT=1 # Some area data like openstreetmap.fr can contain invalid references # that must be cleaned up before using it for borders -- set it to true. BORDERS_CLEANUP=false + +# The current setup assumes this file is placed inside the data/ dir +MBTILES_FILE=tiles.mbtiles +# This is the current repl_config.json location, pre-configured in the tools Dockerfile +# Makefile and quickstart replace it with the dynamically generated one, but we keep it here in case some other method is used to run. +IMPOSM_CONFIG_FILE=/usr/src/app/config/repl_config.json + +# import-borders temp files - set them here to defaults, and override in the Makefile based on the area +BORDERS_CLEANUP_FILE=data/borders/cleanup.pbf +BORDERS_PBF_FILE=data/borders/filtered.pbf +BORDERS_CSV_FILE=data/borders/lines.csv diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8075731..abb258c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -173,7 +173,7 @@ jobs: create_db() { make clean make init-dirs - cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas-latest.osm.pbf + cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas.osm.pbf make destroy-db make all make start-db diff --git a/Makefile b/Makefile index 9e3aeeca..2e044fc5 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,6 @@ DC_OPTS ?= --rm -u $(shell id -u):$(shell id -g) # If set to a non-empty value, will use postgis-preloaded instead of postgis docker image USE_PRELOADED_IMAGE ?= -# If set, this file will be imported in the import-osm target -PBF_FILE?= - # Local port to use with postserve PPORT ?= 8090 export PPORT @@ -55,6 +52,87 @@ endif OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost) +# +# Determine area to work on +# If $(area) parameter is not set and data/*.osm.pbf finds only one file, use it as $(area). +# Otherwise all make targets requiring area param will show an error. +# Note: If there are no data files, and user calls make download area=... once, +# they will not need to use area= parameter after that because there will be just a single file. +# + +# historically we have been using $(area) rather than $(AREA), so make both work +area ?= $(AREA) +# Ensure the $(AREA) param is set, or try to automatically determine it based on available data files +ifeq ($(strip $(area)),) + # if $area is not set. set it to the name of the *.osm.pbf file, but only if there is only one + data_files := $(wildcard data/*.osm.pbf) + ifneq ($(word 2,$(data_files)),) + AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are more than one data/*.osm.pbf files. Set area to one of these IDs, or a new one: $(patsubst data/%.osm.pbf,'%',$(data_files)) + else + ifeq ($(word 1,$(data_files)),) + AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are no data/*.osm.pbf files + else + # Keep just the name of the data file, without the .osm.pbf extension + area := $(strip $(basename $(basename $(notdir $(data_files))))) + # Rename area-latest.osm.pbf to area.osm.pbf + # TODO: This if statement could be removed in a few months once everyone is using the file without the `-latest`? + ifneq ($(area),$(area:-latest=)) + $(shell mv "data/$(area).osm.pbf" "data/$(area:-latest=).osm.pbf") + area := $(area:-latest=) + $(warning ATTENTION: File data/$(area)-latest.osm.pbf was renamed to $(area).osm.pbf.) + AREA_INFO := Detected area=$(area) based on the found data/$(area)-latest.osm.pbf (renamed to $(area).osm.pbf). Use 'area' parameter (or env var) to override. + else + AREA_INFO := Detected area=$(area) based on the found data/ pbf file. Use 'area' parameter (or env var) to override. + endif + endif + endif +endif + +# If set, this file will be downloaded in download-osm and imported in the import-osm targets +PBF_FILE ?= data/$(area).osm.pbf + +# For download-osm, allow URL parameter to download file from a given URL. Area param must still be provided. +ifneq ($(strip $(url)),) + DOWNLOAD_AREA := $(url) +else + DOWNLOAD_AREA := $(area) +endif + +# import-borders uses these temp files during border parsing/import +export BORDERS_CLEANUP_FILE ?= data/borders/$(area).cleanup.pbf +export BORDERS_PBF_FILE ?= data/borders/$(area).filtered.pbf +export BORDERS_CSV_FILE ?= data/borders/$(area).lines.csv + +# The file is placed into the $EXPORT_DIR=/export (mapped to ./data) +export MBTILES_FILE ?= $(area).mbtiles +MBTILES_LOCAL_FILE = data/$(MBTILES_FILE) + +# Location of the dynamically-generated imposm config file +export IMPOSM_CONFIG_FILE ?= data/$(area).repl.json + +# download-osm generates this file with metadata about the file +AREA_DC_CONFIG_FILE ?= data/$(area).dc-config.yml + +ifeq ($(strip $(area)),) + define assert_area_is_given + @echo "ERROR: $(AREA_ERROR)" + @echo "" + @echo " make $@ area=" + @echo "" + @echo "To download an area, use make download " + @echo "To list downloadable areas, use make list-geofabrik and/or make list-bbbike" + @exit 1 + endef +else + ifneq ($(strip $(AREA_INFO)),) + define assert_area_is_given + @echo "$(AREA_INFO)" + endef + endif +endif + + + # # TARGETS # @@ -78,6 +156,8 @@ help: @echo "Hints for developers:" @echo " make # build source code" @echo " make list-geofabrik # list actual geofabrik OSM extracts for download" + @echo " make list-bbbike # list actual BBBike OSM extracts for download" + @echo " make download area=albania # download OSM data from any source and create config file" @echo " make download-geofabrik area=albania # download OSM data from geofabrik.de and create config file" @echo " make download-osmfr area=asia/qatar # download OSM data from openstreetmap.fr and create config file" @echo " make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file" @@ -105,7 +185,7 @@ help: .PHONY: init-dirs init-dirs: @mkdir -p build/sql - @mkdir -p data + @mkdir -p data/borders @mkdir -p cache build/openmaptiles.tm2source/data.yml: init-dirs @@ -165,29 +245,51 @@ list-geofabrik: init-dirs list-bbbike: init-dirs $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike +# +# download, download-geofabrik, download-osmfr, and download-bbbike are handled here +# The --imposm-cfg will fail for some of the sources, but we ignore that error -- only needed for diff mode +# OSM_SERVERS := geofabrik osmfr bbbike -ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) -OSM_SERVER=$(patsubst download-%,%,$@) +ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) download +OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@)) .PHONY: $(ALL_DOWNLOADS) $(ALL_DOWNLOADS): init-dirs -ifeq ($(strip $(area)),) - @echo "" - @echo "ERROR: Unable to download an area if area is not given." - @echo "Usage:" - @echo " make download-$(OSM_SERVER) area=" - @echo "" - $(if $(filter %-geofabrik,$@),@echo "Use make list-geofabrik to get a list of all available areas";echo "") - @exit 1 -else - @echo "=============== download-$(OSM_SERVER) =======================" - @echo "Download area: $(area)" - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \ - 'download-osm $(OSM_SERVER) $(area) \ + @$(assert_area_is_given) +ifeq (,$(wildcard $(PBF_FILE))) +ifneq ($(strip $(url)),) + $(if $(OSM_SERVER),$(error url parameter can only be used with the 'make download area=... url=...')) +endif + @echo "Downloading $(area) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)" + @$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \ + download-osm $(OSM_SERVER) $(DOWNLOAD_AREA) \ --minzoom $$QUICKSTART_MIN_ZOOM \ --maxzoom $$QUICKSTART_MAX_ZOOM \ - --make-dc /import/docker-compose-config.yml -- -d /import' - ls -la ./data/$(notdir $(area))* + --make-dc $(AREA_DC_CONFIG_FILE) \ + --imposm-cfg $(IMPOSM_CONFIG_FILE) \ + --output $(PBF_FILE) \ + 2>&1 \ + | tee /tmp/download.out ; \ + exit_code=$${PIPESTATUS[0]} ; \ + if [[ "$$exit_code" != "0" ]]; then \ + if grep -q "Imposm config file cannot be generated from this source" /tmp/download.out; then \ + echo "WARNING: $(IMPOSM_CONFIG_FILE) could not be generated, but it is only needed to apply updates." ; \ + else \ + exit $$exit_code ; \ + fi ; \ + fi' @echo "" +else +ifeq (,$(wildcard $(AREA_DC_CONFIG_FILE))) + @echo "Data file $(PBF_FILE) already exists, but the $(AREA_DC_CONFIG_FILE) is not, generating..." + @$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \ + download-osm make-dc $(PBF_FILE) \ + --minzoom $$QUICKSTART_MIN_ZOOM \ + --maxzoom $$QUICKSTART_MAX_ZOOM \ + --make-dc $(AREA_DC_CONFIG_FILE) \ + --id "$(area)"' +else + @echo "Data files $(PBF_FILE) and $(AREA_DC_CONFIG_FILE) already exists, skipping the download." +endif endif .PHONY: psql @@ -196,6 +298,7 @@ psql: start-db-nowait .PHONY: import-osm import-osm: all start-db-nowait + @$(assert_area_is_given) $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)' .PHONY: update-osm @@ -212,24 +315,28 @@ import-data: start-db .PHONY: import-borders import-borders: start-db-nowait - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-borders $(PBF_FILE)' + @$(assert_area_is_given) + # If CSV borders file already exists, use it without re-parsing + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \ + 'pgwait && import-borders $$([ -f "$(BORDERS_CSV_FILE)" ] && echo 'load' || echo 'import') $(PBF_FILE)' .PHONY: import-sql import-sql: all start-db-nowait $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \ awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1' -.PHONY: generate-tiles -ifneq ($(wildcard data/docker-compose-config.yml),) - DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml +ifneq ($(wildcard $(AREA_DC_CONFIG_FILE)),) + DC_CONFIG_TILES := -f docker-compose.yml -f $(AREA_DC_CONFIG_FILE) endif +.PHONY: generate-tiles generate-tiles: all start-db - rm -rf data/tiles.mbtiles - echo "Generating tiles ..."; \ + @$(assert_area_is_given) + @echo "Generating tiles into $(MBTILES_LOCAL_FILE) (will delete if already exists)..." + @rm -rf "$(MBTILES_LOCAL_FILE)" $(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles @echo "Updating generated tile metadata ..." $(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) openmaptiles-tools \ - mbtiles-tools meta-generate ./data/tiles.mbtiles ./openmaptiles.yaml --auto-minmax --show-ranges + mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" ./openmaptiles.yaml --auto-minmax --show-ranges .PHONY: start-tileserver start-tileserver: init-dirs diff --git a/README.md b/README.md index 2ebb322e..a33f7f9b 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,10 @@ make ``` You can execute the following manual steps (for better understanding) -or use the provided `quickstart.sh` script. +or use the provided `quickstart.sh` script to automatically download and import given area. If area is not given, albania will be imported. ``` -./quickstart.sh +./quickstart.sh ``` ### Prepare the Database @@ -103,10 +103,10 @@ Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), make import-data ``` -[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory. +Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. Use `download-geofabrik`, `download-bbbike`, or `download-osmfr` for a specific source. Use `download area=planet` for the entire OSM dataset (very large). Note that if you have more than one `data/*.osm.pbf` file, every `make` command will require `area=...` parameter (or you can just `export area=...` first) ```bash -make download-geofabrik area=albania +make download area=albania ``` [Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from diff --git a/docker-compose.yml b/docker-compose.yml index 45db0890..f586e1c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,10 @@ services: MAKE_DC_VERSION: "2.3" # Allow DIFF_MODE to be overwritten from shell DIFF_MODE: ${DIFF_MODE} + # Which files to use during import-borders processing + BORDERS_CLEANUP_FILE: ${BORDERS_CLEANUP_FILE} + BORDERS_PBF_FILE: ${BORDERS_PBF_FILE} + BORDERS_CSV_FILE: ${BORDERS_CSV_FILE} networks: - postgres_conn volumes: @@ -54,6 +58,10 @@ services: networks: - postgres_conn env_file: .env + environment: + FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT} + MBTILES_NAME: ${MBTILES_FILE} + IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE} generate-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" @@ -64,10 +72,12 @@ services: - postgres_conn env_file: .env environment: + FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT} + MBTILES_NAME: ${MBTILES_FILE} + IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE} BBOX: ${BBOX} MIN_ZOOM: ${MIN_ZOOM} MAX_ZOOM: ${MAX_ZOOM} - FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT} postserve: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" diff --git a/quickstart.sh b/quickstart.sh index dd44a256..3c5513b6 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -46,26 +46,17 @@ else fi if [ $# -eq 0 ]; then - osm_area=albania # default test country - echo "No parameter - set area=$osm_area " + # default test area + export area=albania + echo "No parameter - set area=$area " else - osm_area=$1 + export area=$1 fi if [ $# -eq 2 ]; then - osm_server=$2 -else - if [[ ${osm_area} == [[:upper:]]* ]]; then - # Only bbbike area names are capitalized - osm_server=bbbike - else - # default OSM server - osm_server=geofabrik - fi + osm_server=$2 fi -pbf_file="./data/${osm_area##*/}-latest.osm.pbf" - ## Min versions ... MIN_COMPOSE_VER=1.7.1 MIN_DOCKER_VER=1.12.3 @@ -117,8 +108,8 @@ echo " Start processing echo "-------------------------------------------------------------------------------------" echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " echo " : This will be logged to the $log_file file (for debugging) and to the screen" -echo " : Area : $osm_area " -echo " : Download Server : $osm_server " +echo " : Area : $area " +echo " : Download Server : ${osm_server:-unset (automatic)} " echo " : Preloaded Image : $USE_PRELOADED_IMAGE " echo " : Git version : $(git rev-parse HEAD) " echo " : Started : $STARTDATE " @@ -140,7 +131,7 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then exit 1 fi echo " : --- Memory, CPU info ---- " - mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) + mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) echo "system memory (GB): ${mem}" grep SwapTotal /proc/meminfo echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)" @@ -168,27 +159,13 @@ make init-dirs echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " -rm -f ./data/*.mbtiles +echo "====> : Removing old MBTILES if exists ( ./data/${area}.mbtiles ) " +rm -f "./data/${area}.mbtiles" -if [[ ! -f "${pbf_file}" || ! -f "./data/docker-compose-config.yml" ]]; then - echo " " - echo "-------------------------------------------------------------------------------------" - echo "====> : Downloading ${osm_area} from ${osm_server}..." - rm -rf ./data/* - make download-${osm_server} "area=${osm_area}" -else - echo " " - echo "-------------------------------------------------------------------------------------" - echo "====> : The pbf file $pbf_file exists, we don't need to download!" -fi - - -if [ ! -f "${pbf_file}" ]; then - echo " " - echo "Missing $pbf_file , Download or Parameter error? " - exit 1 -fi +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Downloading ${area} from ${osm_server:-any source}..." +make "download${osm_server:+-${osm_server}}" echo " " echo "-------------------------------------------------------------------------------------" @@ -243,17 +220,17 @@ fi echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing OpenStreetMap data: ${pbf_file} -> imposm3[./build/mapping.yaml] -> PostgreSQL" +echo "====> : Start importing OpenStreetMap data: ${area} -> imposm3[./build/mapping.yaml] -> PostgreSQL" echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html " echo " : Thank you Omniscale! " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm " echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " echo " : Thank you OpenStreetMap Contributors ! " -make import-osm "PBF_FILE=${pbf_file}" +make import-osm echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing border data from ${pbf_file} into PostgreSQL using osmborder" +echo "====> : Start importing border ${area} data into PostgreSQL using osmborder" echo " : Source code: https://github.com/pnorman/osmborder" echo " : Data license: http://www.openstreetmap.org/copyright" echo " : Thank you: Paul Norman" @@ -288,7 +265,7 @@ echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " -echo " : Output MBTiles: ./data/tiles.mbtiles " +echo " : Output MBTiles: ./data/${area}.mbtiles " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles " echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " echo " : Thank you https://www.mapbox.com !" @@ -314,11 +291,6 @@ rm -f ./data/quickstart_checklist.chk cat ./data/quickstart_checklist.chk ENDTIME=$(date +%s) -if stat --help >/dev/null 2>&1; then - MODDATE=$(stat -c %y "${pbf_file}" ) -else - MODDATE=$(stat -f%Sm -t '%F %T %z' "${pbf_file}" ) -fi echo " " echo " " @@ -333,15 +305,14 @@ docker images | grep openmaptiles echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) " +echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) " echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file." -echo " : Created from $pbf_file ( file moddate: $MODDATE ) " -echo " : Size: " -ls -la ./data/*.mbtiles +echo " : Data directory content:" +ls -la ./data echo " " echo "-------------------------------------------------------------------------------------" -echo "The ./quickstart.sh $osm_area is finished! " +echo "The ./quickstart.sh $area is finished! " echo "It takes $((ENDTIME - STARTTIME)) seconds to complete" echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" echo " "