kopia lustrzana https://github.com/openmaptiles/openmaptiles
Rework download area support (#908)
Closes #904 * Make all data-related targets like `download*`, `import-osm`, `import-borders`, and `generate-tiles` into `area`-aware -- making it possible for multiple data files to coexist inside the `./data` dir. * Add `make download area=... [url=...]` command to automatically download any kind of area by checking Geofabrik, BBBike, and OSM.fr, optionally from a custom URL. Supports `area=planet` too. * Do not re-download area with `make download-*` if it already exists. * Automatically rename `<area>-latest.osm.pbf` into `<area>.osm.pbf` * If `area=...` parameter is not given to `make`, see if there is exactly one `*.osm.pbf` file, and if so, use `*` as the `area`. * Configure many variables in the .env file, overriding the defaults in tools * If `<area>.osm.pbf` exists, but `<area>.dc-config.pbf` is missing, generate it using `download-osm make-dc` command. Also: * closes #614 * closes #647 * partially addresses #261pull/916/head
rodzic
fca53ef0ee
commit
1486b7e0cb
11
.env
11
.env
|
@ -24,3 +24,14 @@ FILTER_MAPNIK_OUTPUT=1
|
||||||
# Some area data like openstreetmap.fr can contain invalid references
|
# Some area data like openstreetmap.fr can contain invalid references
|
||||||
# that must be cleaned up before using it for borders -- set it to true.
|
# that must be cleaned up before using it for borders -- set it to true.
|
||||||
BORDERS_CLEANUP=false
|
BORDERS_CLEANUP=false
|
||||||
|
|
||||||
|
# The current setup assumes this file is placed inside the data/ dir
|
||||||
|
MBTILES_FILE=tiles.mbtiles
|
||||||
|
# This is the current repl_config.json location, pre-configured in the tools Dockerfile
|
||||||
|
# Makefile and quickstart replace it with the dynamically generated one, but we keep it here in case some other method is used to run.
|
||||||
|
IMPOSM_CONFIG_FILE=/usr/src/app/config/repl_config.json
|
||||||
|
|
||||||
|
# import-borders temp files - set them here to defaults, and override in the Makefile based on the area
|
||||||
|
BORDERS_CLEANUP_FILE=data/borders/cleanup.pbf
|
||||||
|
BORDERS_PBF_FILE=data/borders/filtered.pbf
|
||||||
|
BORDERS_CSV_FILE=data/borders/lines.csv
|
||||||
|
|
|
@ -173,7 +173,7 @@ jobs:
|
||||||
create_db() {
|
create_db() {
|
||||||
make clean
|
make clean
|
||||||
make init-dirs
|
make init-dirs
|
||||||
cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas-latest.osm.pbf
|
cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas.osm.pbf
|
||||||
make destroy-db
|
make destroy-db
|
||||||
make all
|
make all
|
||||||
make start-db
|
make start-db
|
||||||
|
|
163
Makefile
163
Makefile
|
@ -13,9 +13,6 @@ DC_OPTS ?= --rm -u $(shell id -u):$(shell id -g)
|
||||||
# If set to a non-empty value, will use postgis-preloaded instead of postgis docker image
|
# If set to a non-empty value, will use postgis-preloaded instead of postgis docker image
|
||||||
USE_PRELOADED_IMAGE ?=
|
USE_PRELOADED_IMAGE ?=
|
||||||
|
|
||||||
# If set, this file will be imported in the import-osm target
|
|
||||||
PBF_FILE?=
|
|
||||||
|
|
||||||
# Local port to use with postserve
|
# Local port to use with postserve
|
||||||
PPORT ?= 8090
|
PPORT ?= 8090
|
||||||
export PPORT
|
export PPORT
|
||||||
|
@ -55,6 +52,87 @@ endif
|
||||||
OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
|
OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
# Determine area to work on
|
||||||
|
# If $(area) parameter is not set and data/*.osm.pbf finds only one file, use it as $(area).
|
||||||
|
# Otherwise all make targets requiring area param will show an error.
|
||||||
|
# Note: If there are no data files, and user calls make download area=... once,
|
||||||
|
# they will not need to use area= parameter after that because there will be just a single file.
|
||||||
|
#
|
||||||
|
|
||||||
|
# historically we have been using $(area) rather than $(AREA), so make both work
|
||||||
|
area ?= $(AREA)
|
||||||
|
# Ensure the $(AREA) param is set, or try to automatically determine it based on available data files
|
||||||
|
ifeq ($(strip $(area)),)
|
||||||
|
# if $area is not set. set it to the name of the *.osm.pbf file, but only if there is only one
|
||||||
|
data_files := $(wildcard data/*.osm.pbf)
|
||||||
|
ifneq ($(word 2,$(data_files)),)
|
||||||
|
AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are more than one data/*.osm.pbf files. Set area to one of these IDs, or a new one: $(patsubst data/%.osm.pbf,'%',$(data_files))
|
||||||
|
else
|
||||||
|
ifeq ($(word 1,$(data_files)),)
|
||||||
|
AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are no data/*.osm.pbf files
|
||||||
|
else
|
||||||
|
# Keep just the name of the data file, without the .osm.pbf extension
|
||||||
|
area := $(strip $(basename $(basename $(notdir $(data_files)))))
|
||||||
|
# Rename area-latest.osm.pbf to area.osm.pbf
|
||||||
|
# TODO: This if statement could be removed in a few months once everyone is using the file without the `-latest`?
|
||||||
|
ifneq ($(area),$(area:-latest=))
|
||||||
|
$(shell mv "data/$(area).osm.pbf" "data/$(area:-latest=).osm.pbf")
|
||||||
|
area := $(area:-latest=)
|
||||||
|
$(warning ATTENTION: File data/$(area)-latest.osm.pbf was renamed to $(area).osm.pbf.)
|
||||||
|
AREA_INFO := Detected area=$(area) based on the found data/$(area)-latest.osm.pbf (renamed to $(area).osm.pbf). Use 'area' parameter (or env var) to override.
|
||||||
|
else
|
||||||
|
AREA_INFO := Detected area=$(area) based on the found data/ pbf file. Use 'area' parameter (or env var) to override.
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
|
# If set, this file will be downloaded in download-osm and imported in the import-osm targets
|
||||||
|
PBF_FILE ?= data/$(area).osm.pbf
|
||||||
|
|
||||||
|
# For download-osm, allow URL parameter to download file from a given URL. Area param must still be provided.
|
||||||
|
ifneq ($(strip $(url)),)
|
||||||
|
DOWNLOAD_AREA := $(url)
|
||||||
|
else
|
||||||
|
DOWNLOAD_AREA := $(area)
|
||||||
|
endif
|
||||||
|
|
||||||
|
# import-borders uses these temp files during border parsing/import
|
||||||
|
export BORDERS_CLEANUP_FILE ?= data/borders/$(area).cleanup.pbf
|
||||||
|
export BORDERS_PBF_FILE ?= data/borders/$(area).filtered.pbf
|
||||||
|
export BORDERS_CSV_FILE ?= data/borders/$(area).lines.csv
|
||||||
|
|
||||||
|
# The file is placed into the $EXPORT_DIR=/export (mapped to ./data)
|
||||||
|
export MBTILES_FILE ?= $(area).mbtiles
|
||||||
|
MBTILES_LOCAL_FILE = data/$(MBTILES_FILE)
|
||||||
|
|
||||||
|
# Location of the dynamically-generated imposm config file
|
||||||
|
export IMPOSM_CONFIG_FILE ?= data/$(area).repl.json
|
||||||
|
|
||||||
|
# download-osm generates this file with metadata about the file
|
||||||
|
AREA_DC_CONFIG_FILE ?= data/$(area).dc-config.yml
|
||||||
|
|
||||||
|
ifeq ($(strip $(area)),)
|
||||||
|
define assert_area_is_given
|
||||||
|
@echo "ERROR: $(AREA_ERROR)"
|
||||||
|
@echo ""
|
||||||
|
@echo " make $@ area=<area-id>"
|
||||||
|
@echo ""
|
||||||
|
@echo "To download an area, use make download <area-id>"
|
||||||
|
@echo "To list downloadable areas, use make list-geofabrik and/or make list-bbbike"
|
||||||
|
@exit 1
|
||||||
|
endef
|
||||||
|
else
|
||||||
|
ifneq ($(strip $(AREA_INFO)),)
|
||||||
|
define assert_area_is_given
|
||||||
|
@echo "$(AREA_INFO)"
|
||||||
|
endef
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# TARGETS
|
# TARGETS
|
||||||
#
|
#
|
||||||
|
@ -78,6 +156,8 @@ help:
|
||||||
@echo "Hints for developers:"
|
@echo "Hints for developers:"
|
||||||
@echo " make # build source code"
|
@echo " make # build source code"
|
||||||
@echo " make list-geofabrik # list actual geofabrik OSM extracts for download"
|
@echo " make list-geofabrik # list actual geofabrik OSM extracts for download"
|
||||||
|
@echo " make list-bbbike # list actual BBBike OSM extracts for download"
|
||||||
|
@echo " make download area=albania # download OSM data from any source and create config file"
|
||||||
@echo " make download-geofabrik area=albania # download OSM data from geofabrik.de and create config file"
|
@echo " make download-geofabrik area=albania # download OSM data from geofabrik.de and create config file"
|
||||||
@echo " make download-osmfr area=asia/qatar # download OSM data from openstreetmap.fr and create config file"
|
@echo " make download-osmfr area=asia/qatar # download OSM data from openstreetmap.fr and create config file"
|
||||||
@echo " make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file"
|
@echo " make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file"
|
||||||
|
@ -105,7 +185,7 @@ help:
|
||||||
.PHONY: init-dirs
|
.PHONY: init-dirs
|
||||||
init-dirs:
|
init-dirs:
|
||||||
@mkdir -p build/sql
|
@mkdir -p build/sql
|
||||||
@mkdir -p data
|
@mkdir -p data/borders
|
||||||
@mkdir -p cache
|
@mkdir -p cache
|
||||||
|
|
||||||
build/openmaptiles.tm2source/data.yml: init-dirs
|
build/openmaptiles.tm2source/data.yml: init-dirs
|
||||||
|
@ -165,29 +245,51 @@ list-geofabrik: init-dirs
|
||||||
list-bbbike: init-dirs
|
list-bbbike: init-dirs
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike
|
||||||
|
|
||||||
|
#
|
||||||
|
# download, download-geofabrik, download-osmfr, and download-bbbike are handled here
|
||||||
|
# The --imposm-cfg will fail for some of the sources, but we ignore that error -- only needed for diff mode
|
||||||
|
#
|
||||||
OSM_SERVERS := geofabrik osmfr bbbike
|
OSM_SERVERS := geofabrik osmfr bbbike
|
||||||
ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS))
|
ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) download
|
||||||
OSM_SERVER=$(patsubst download-%,%,$@)
|
OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@))
|
||||||
.PHONY: $(ALL_DOWNLOADS)
|
.PHONY: $(ALL_DOWNLOADS)
|
||||||
$(ALL_DOWNLOADS): init-dirs
|
$(ALL_DOWNLOADS): init-dirs
|
||||||
ifeq ($(strip $(area)),)
|
@$(assert_area_is_given)
|
||||||
@echo ""
|
ifeq (,$(wildcard $(PBF_FILE)))
|
||||||
@echo "ERROR: Unable to download an area if area is not given."
|
ifneq ($(strip $(url)),)
|
||||||
@echo "Usage:"
|
$(if $(OSM_SERVER),$(error url parameter can only be used with the 'make download area=... url=...'))
|
||||||
@echo " make download-$(OSM_SERVER) area=<area-id>"
|
endif
|
||||||
@echo ""
|
@echo "Downloading $(area) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
|
||||||
$(if $(filter %-geofabrik,$@),@echo "Use make list-geofabrik to get a list of all available areas";echo "")
|
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
|
||||||
@exit 1
|
download-osm $(OSM_SERVER) $(DOWNLOAD_AREA) \
|
||||||
else
|
|
||||||
@echo "=============== download-$(OSM_SERVER) ======================="
|
|
||||||
@echo "Download area: $(area)"
|
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
|
|
||||||
'download-osm $(OSM_SERVER) $(area) \
|
|
||||||
--minzoom $$QUICKSTART_MIN_ZOOM \
|
--minzoom $$QUICKSTART_MIN_ZOOM \
|
||||||
--maxzoom $$QUICKSTART_MAX_ZOOM \
|
--maxzoom $$QUICKSTART_MAX_ZOOM \
|
||||||
--make-dc /import/docker-compose-config.yml -- -d /import'
|
--make-dc $(AREA_DC_CONFIG_FILE) \
|
||||||
ls -la ./data/$(notdir $(area))*
|
--imposm-cfg $(IMPOSM_CONFIG_FILE) \
|
||||||
|
--output $(PBF_FILE) \
|
||||||
|
2>&1 \
|
||||||
|
| tee /tmp/download.out ; \
|
||||||
|
exit_code=$${PIPESTATUS[0]} ; \
|
||||||
|
if [[ "$$exit_code" != "0" ]]; then \
|
||||||
|
if grep -q "Imposm config file cannot be generated from this source" /tmp/download.out; then \
|
||||||
|
echo "WARNING: $(IMPOSM_CONFIG_FILE) could not be generated, but it is only needed to apply updates." ; \
|
||||||
|
else \
|
||||||
|
exit $$exit_code ; \
|
||||||
|
fi ; \
|
||||||
|
fi'
|
||||||
@echo ""
|
@echo ""
|
||||||
|
else
|
||||||
|
ifeq (,$(wildcard $(AREA_DC_CONFIG_FILE)))
|
||||||
|
@echo "Data file $(PBF_FILE) already exists, but the $(AREA_DC_CONFIG_FILE) is not, generating..."
|
||||||
|
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
|
||||||
|
download-osm make-dc $(PBF_FILE) \
|
||||||
|
--minzoom $$QUICKSTART_MIN_ZOOM \
|
||||||
|
--maxzoom $$QUICKSTART_MAX_ZOOM \
|
||||||
|
--make-dc $(AREA_DC_CONFIG_FILE) \
|
||||||
|
--id "$(area)"'
|
||||||
|
else
|
||||||
|
@echo "Data files $(PBF_FILE) and $(AREA_DC_CONFIG_FILE) already exists, skipping the download."
|
||||||
|
endif
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: psql
|
.PHONY: psql
|
||||||
|
@ -196,6 +298,7 @@ psql: start-db-nowait
|
||||||
|
|
||||||
.PHONY: import-osm
|
.PHONY: import-osm
|
||||||
import-osm: all start-db-nowait
|
import-osm: all start-db-nowait
|
||||||
|
@$(assert_area_is_given)
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)'
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)'
|
||||||
|
|
||||||
.PHONY: update-osm
|
.PHONY: update-osm
|
||||||
|
@ -212,24 +315,28 @@ import-data: start-db
|
||||||
|
|
||||||
.PHONY: import-borders
|
.PHONY: import-borders
|
||||||
import-borders: start-db-nowait
|
import-borders: start-db-nowait
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-borders $(PBF_FILE)'
|
@$(assert_area_is_given)
|
||||||
|
# If CSV borders file already exists, use it without re-parsing
|
||||||
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
|
||||||
|
'pgwait && import-borders $$([ -f "$(BORDERS_CSV_FILE)" ] && echo 'load' || echo 'import') $(PBF_FILE)'
|
||||||
|
|
||||||
.PHONY: import-sql
|
.PHONY: import-sql
|
||||||
import-sql: all start-db-nowait
|
import-sql: all start-db-nowait
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
|
||||||
awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1'
|
awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1'
|
||||||
|
|
||||||
.PHONY: generate-tiles
|
ifneq ($(wildcard $(AREA_DC_CONFIG_FILE)),)
|
||||||
ifneq ($(wildcard data/docker-compose-config.yml),)
|
DC_CONFIG_TILES := -f docker-compose.yml -f $(AREA_DC_CONFIG_FILE)
|
||||||
DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
|
|
||||||
endif
|
endif
|
||||||
|
.PHONY: generate-tiles
|
||||||
generate-tiles: all start-db
|
generate-tiles: all start-db
|
||||||
rm -rf data/tiles.mbtiles
|
@$(assert_area_is_given)
|
||||||
echo "Generating tiles ..."; \
|
@echo "Generating tiles into $(MBTILES_LOCAL_FILE) (will delete if already exists)..."
|
||||||
|
@rm -rf "$(MBTILES_LOCAL_FILE)"
|
||||||
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
|
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
|
||||||
@echo "Updating generated tile metadata ..."
|
@echo "Updating generated tile metadata ..."
|
||||||
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) openmaptiles-tools \
|
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) openmaptiles-tools \
|
||||||
mbtiles-tools meta-generate ./data/tiles.mbtiles ./openmaptiles.yaml --auto-minmax --show-ranges
|
mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" ./openmaptiles.yaml --auto-minmax --show-ranges
|
||||||
|
|
||||||
.PHONY: start-tileserver
|
.PHONY: start-tileserver
|
||||||
start-tileserver: init-dirs
|
start-tileserver: init-dirs
|
||||||
|
|
|
@ -83,10 +83,10 @@ make
|
||||||
```
|
```
|
||||||
|
|
||||||
You can execute the following manual steps (for better understanding)
|
You can execute the following manual steps (for better understanding)
|
||||||
or use the provided `quickstart.sh` script.
|
or use the provided `quickstart.sh` script to automatically download and import given area. If area is not given, albania will be imported.
|
||||||
|
|
||||||
```
|
```
|
||||||
./quickstart.sh
|
./quickstart.sh <area>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Prepare the Database
|
### Prepare the Database
|
||||||
|
@ -103,10 +103,10 @@ Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/),
|
||||||
make import-data
|
make import-data
|
||||||
```
|
```
|
||||||
|
|
||||||
[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
|
Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. Use `download-geofabrik`, `download-bbbike`, or `download-osmfr` for a specific source. Use `download area=planet` for the entire OSM dataset (very large). Note that if you have more than one `data/*.osm.pbf` file, every `make` command will require `area=...` parameter (or you can just `export area=...` first)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make download-geofabrik area=albania
|
make download area=albania
|
||||||
```
|
```
|
||||||
|
|
||||||
[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
|
[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
|
||||||
|
|
|
@ -36,6 +36,10 @@ services:
|
||||||
MAKE_DC_VERSION: "2.3"
|
MAKE_DC_VERSION: "2.3"
|
||||||
# Allow DIFF_MODE to be overwritten from shell
|
# Allow DIFF_MODE to be overwritten from shell
|
||||||
DIFF_MODE: ${DIFF_MODE}
|
DIFF_MODE: ${DIFF_MODE}
|
||||||
|
# Which files to use during import-borders processing
|
||||||
|
BORDERS_CLEANUP_FILE: ${BORDERS_CLEANUP_FILE}
|
||||||
|
BORDERS_PBF_FILE: ${BORDERS_PBF_FILE}
|
||||||
|
BORDERS_CSV_FILE: ${BORDERS_CSV_FILE}
|
||||||
networks:
|
networks:
|
||||||
- postgres_conn
|
- postgres_conn
|
||||||
volumes:
|
volumes:
|
||||||
|
@ -54,6 +58,10 @@ services:
|
||||||
networks:
|
networks:
|
||||||
- postgres_conn
|
- postgres_conn
|
||||||
env_file: .env
|
env_file: .env
|
||||||
|
environment:
|
||||||
|
FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT}
|
||||||
|
MBTILES_NAME: ${MBTILES_FILE}
|
||||||
|
IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE}
|
||||||
|
|
||||||
generate-vectortiles:
|
generate-vectortiles:
|
||||||
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
|
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
|
||||||
|
@ -64,10 +72,12 @@ services:
|
||||||
- postgres_conn
|
- postgres_conn
|
||||||
env_file: .env
|
env_file: .env
|
||||||
environment:
|
environment:
|
||||||
|
FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT}
|
||||||
|
MBTILES_NAME: ${MBTILES_FILE}
|
||||||
|
IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE}
|
||||||
BBOX: ${BBOX}
|
BBOX: ${BBOX}
|
||||||
MIN_ZOOM: ${MIN_ZOOM}
|
MIN_ZOOM: ${MIN_ZOOM}
|
||||||
MAX_ZOOM: ${MAX_ZOOM}
|
MAX_ZOOM: ${MAX_ZOOM}
|
||||||
FILTER_MAPNIK_OUTPUT: ${FILTER_MAPNIK_OUTPUT}
|
|
||||||
|
|
||||||
postserve:
|
postserve:
|
||||||
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
|
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
|
||||||
|
|
|
@ -46,26 +46,17 @@ else
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ $# -eq 0 ]; then
|
if [ $# -eq 0 ]; then
|
||||||
osm_area=albania # default test country
|
# default test area
|
||||||
echo "No parameter - set area=$osm_area "
|
export area=albania
|
||||||
|
echo "No parameter - set area=$area "
|
||||||
else
|
else
|
||||||
osm_area=$1
|
export area=$1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ $# -eq 2 ]; then
|
if [ $# -eq 2 ]; then
|
||||||
osm_server=$2
|
osm_server=$2
|
||||||
else
|
|
||||||
if [[ ${osm_area} == [[:upper:]]* ]]; then
|
|
||||||
# Only bbbike area names are capitalized
|
|
||||||
osm_server=bbbike
|
|
||||||
else
|
|
||||||
# default OSM server
|
|
||||||
osm_server=geofabrik
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pbf_file="./data/${osm_area##*/}-latest.osm.pbf"
|
|
||||||
|
|
||||||
## Min versions ...
|
## Min versions ...
|
||||||
MIN_COMPOSE_VER=1.7.1
|
MIN_COMPOSE_VER=1.7.1
|
||||||
MIN_DOCKER_VER=1.12.3
|
MIN_DOCKER_VER=1.12.3
|
||||||
|
@ -117,8 +108,8 @@ echo " Start processing
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] "
|
echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] "
|
||||||
echo " : This will be logged to the $log_file file (for debugging) and to the screen"
|
echo " : This will be logged to the $log_file file (for debugging) and to the screen"
|
||||||
echo " : Area : $osm_area "
|
echo " : Area : $area "
|
||||||
echo " : Download Server : $osm_server "
|
echo " : Download Server : ${osm_server:-unset (automatic)} "
|
||||||
echo " : Preloaded Image : $USE_PRELOADED_IMAGE "
|
echo " : Preloaded Image : $USE_PRELOADED_IMAGE "
|
||||||
echo " : Git version : $(git rev-parse HEAD) "
|
echo " : Git version : $(git rev-parse HEAD) "
|
||||||
echo " : Started : $STARTDATE "
|
echo " : Started : $STARTDATE "
|
||||||
|
@ -140,7 +131,7 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo " : --- Memory, CPU info ---- "
|
echo " : --- Memory, CPU info ---- "
|
||||||
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
|
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
|
||||||
echo "system memory (GB): ${mem}"
|
echo "system memory (GB): ${mem}"
|
||||||
grep SwapTotal /proc/meminfo
|
grep SwapTotal /proc/meminfo
|
||||||
echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
|
echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
|
||||||
|
@ -168,27 +159,13 @@ make init-dirs
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) "
|
echo "====> : Removing old MBTILES if exists ( ./data/${area}.mbtiles ) "
|
||||||
rm -f ./data/*.mbtiles
|
rm -f "./data/${area}.mbtiles"
|
||||||
|
|
||||||
if [[ ! -f "${pbf_file}" || ! -f "./data/docker-compose-config.yml" ]]; then
|
echo " "
|
||||||
echo " "
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "====> : Downloading ${area} from ${osm_server:-any source}..."
|
||||||
echo "====> : Downloading ${osm_area} from ${osm_server}..."
|
make "download${osm_server:+-${osm_server}}"
|
||||||
rm -rf ./data/*
|
|
||||||
make download-${osm_server} "area=${osm_area}"
|
|
||||||
else
|
|
||||||
echo " "
|
|
||||||
echo "-------------------------------------------------------------------------------------"
|
|
||||||
echo "====> : The pbf file $pbf_file exists, we don't need to download!"
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
if [ ! -f "${pbf_file}" ]; then
|
|
||||||
echo " "
|
|
||||||
echo "Missing $pbf_file , Download or Parameter error? "
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
|
@ -243,17 +220,17 @@ fi
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : Start importing OpenStreetMap data: ${pbf_file} -> imposm3[./build/mapping.yaml] -> PostgreSQL"
|
echo "====> : Start importing OpenStreetMap data: ${area} -> imposm3[./build/mapping.yaml] -> PostgreSQL"
|
||||||
echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html "
|
echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html "
|
||||||
echo " : Thank you Omniscale! "
|
echo " : Thank you Omniscale! "
|
||||||
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm "
|
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm "
|
||||||
echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) "
|
echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) "
|
||||||
echo " : Thank you OpenStreetMap Contributors ! "
|
echo " : Thank you OpenStreetMap Contributors ! "
|
||||||
make import-osm "PBF_FILE=${pbf_file}"
|
make import-osm
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : Start importing border data from ${pbf_file} into PostgreSQL using osmborder"
|
echo "====> : Start importing border ${area} data into PostgreSQL using osmborder"
|
||||||
echo " : Source code: https://github.com/pnorman/osmborder"
|
echo " : Source code: https://github.com/pnorman/osmborder"
|
||||||
echo " : Data license: http://www.openstreetmap.org/copyright"
|
echo " : Data license: http://www.openstreetmap.org/copyright"
|
||||||
echo " : Thank you: Paul Norman"
|
echo " : Thank you: Paul Norman"
|
||||||
|
@ -288,7 +265,7 @@ echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "
|
echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "
|
||||||
echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml "
|
echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml "
|
||||||
echo " : Output MBTiles: ./data/tiles.mbtiles "
|
echo " : Output MBTiles: ./data/${area}.mbtiles "
|
||||||
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles "
|
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles "
|
||||||
echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox "
|
echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox "
|
||||||
echo " : Thank you https://www.mapbox.com !"
|
echo " : Thank you https://www.mapbox.com !"
|
||||||
|
@ -314,11 +291,6 @@ rm -f ./data/quickstart_checklist.chk
|
||||||
cat ./data/quickstart_checklist.chk
|
cat ./data/quickstart_checklist.chk
|
||||||
|
|
||||||
ENDTIME=$(date +%s)
|
ENDTIME=$(date +%s)
|
||||||
if stat --help >/dev/null 2>&1; then
|
|
||||||
MODDATE=$(stat -c %y "${pbf_file}" )
|
|
||||||
else
|
|
||||||
MODDATE=$(stat -f%Sm -t '%F %T %z' "${pbf_file}" )
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo " "
|
echo " "
|
||||||
|
@ -333,15 +305,14 @@ docker images | grep openmaptiles
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) "
|
echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) "
|
||||||
echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file."
|
echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file."
|
||||||
echo " : Created from $pbf_file ( file moddate: $MODDATE ) "
|
echo " : Data directory content:"
|
||||||
echo " : Size: "
|
ls -la ./data
|
||||||
ls -la ./data/*.mbtiles
|
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "The ./quickstart.sh $osm_area is finished! "
|
echo "The ./quickstart.sh $area is finished! "
|
||||||
echo "It takes $((ENDTIME - STARTTIME)) seconds to complete"
|
echo "It takes $((ENDTIME - STARTTIME)) seconds to complete"
|
||||||
echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !"
|
echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !"
|
||||||
echo " "
|
echo " "
|
||||||
|
|
Ładowanie…
Reference in New Issue