kopia lustrzana https://github.com/openmaptiles/openmaptiles
update makefile
sdirollo
rodzic
6c4b90806a
commit
92591158e2
356
Makefile
356
Makefile
|
|
@ -47,14 +47,204 @@ XARGS := xargs $(shell xargs --no-run-if-empty </dev/null 2>/dev/null && echo --
|
|||
# If running in the test mode, compare files rather than copy them
|
||||
TEST_MODE?=no
|
||||
ifeq ($(TEST_MODE),yes)
|
||||
COPY_TO_GIT=diff
|
||||
# create images in ./build/devdoc and compare them to ./layers
|
||||
GRAPH_PARAMS=./build/devdoc ./layers
|
||||
else
|
||||
COPY_TO_GIT=cp
|
||||
# update graphs in the ./layers dir
|
||||
GRAPH_PARAMS=./layers
|
||||
endif
|
||||
|
||||
# Set OpenMapTiles host
|
||||
OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
|
||||
export OMT_HOST
|
||||
|
||||
# This defines an easy $(newline) value to act as a "\n". Make sure to keep exactly two empty lines after newline.
|
||||
define newline
|
||||
|
||||
|
||||
endef
|
||||
|
||||
# use the old postgres connection values if they are existing
|
||||
PGHOST := $(or $(POSTGRES_HOST),$(PGHOST))
|
||||
PGPORT := $(or $(POSTGRES_PORT),$(PGPORT))
|
||||
PGDATABASE := $(or $(POSTGRES_DB),$(PGDATABASE))
|
||||
PGUSER := $(or $(POSTGRES_USER),$(PGUSER))
|
||||
PGPASSWORD := $(or $(POSTGRES_PASSWORD),$(PGPASSWORD))
|
||||
|
||||
#
|
||||
# Determine area to work on
|
||||
# If $(area) parameter is not set, and only one *.osm.pbf file is found in ./data, use it as $(area).
|
||||
# Otherwise, all make targets requiring an area will show an error.
|
||||
# Note: If no *.osm.pbf files are found, once the users call "make download area=..."
|
||||
# they will not need to use an "area=" parameter again because there will be just a single file.
|
||||
#
|
||||
|
||||
# historically we have been using $(area) rather than $(AREA), so make both work
|
||||
area ?= $(AREA)
|
||||
# Ensure the $(area) param is set, or try to automatically determine it based on available data files
|
||||
ifeq ($(strip $(area)),)
|
||||
# An $(area) parameter is not set. If only one *.osm.pbf file is found in ./data, use it as $(area).
|
||||
data_files := $(shell find data -name '*.osm.pbf' 2>/dev/null)
|
||||
ifneq ($(word 2,$(data_files)),)
|
||||
define assert_area_is_given
|
||||
@echo ""
|
||||
@echo "ERROR: The 'area' parameter or environment variable have not been set, and there several 'area' options:"
|
||||
@$(patsubst data/%.osm.pbf,echo " '%'";,$(data_files))
|
||||
@echo ""
|
||||
@echo "To specify an area use:"
|
||||
@echo " make $@ area=<area-id>"
|
||||
@echo ""
|
||||
@exit 1
|
||||
endef
|
||||
else
|
||||
ifeq ($(word 1,$(data_files)),)
|
||||
define assert_area_is_given
|
||||
@echo ""
|
||||
@echo "ERROR: The 'area' parameter (or env var) has not been set, and there are no data/*.osm.pbf files"
|
||||
@echo ""
|
||||
@echo "To specify an area use"
|
||||
@echo " make $@ area=<area-id>"
|
||||
@echo ""
|
||||
@echo "To download an area, use make download area=<area-id>"
|
||||
@echo "To list downloadable areas, use make list-geofabrik and/or make list-bbbike"
|
||||
@exit 1
|
||||
@echo ""
|
||||
endef
|
||||
else
|
||||
# Keep just the name of the data file, without the .osm.pbf extension
|
||||
area := $(patsubst data/%.osm.pbf,%,$(data_files))
|
||||
# Rename area-latest.osm.pbf to area.osm.pbf
|
||||
# TODO: This if statement could be removed in a few months once everyone is using the file without the `-latest`?
|
||||
ifneq ($(area),$(area:-latest=))
|
||||
$(shell mv "data/$(area).osm.pbf" "data/$(area:-latest=).osm.pbf")
|
||||
area := $(area:-latest=)
|
||||
$(warning ATTENTION: File data/$(area)-latest.osm.pbf was renamed to $(area).osm.pbf.)
|
||||
AREA_INFO := Detected area=$(area) based on finding a 'data/$(area)-latest.osm.pbf' file - renamed to '$(area).osm.pbf'. Use 'area' parameter or environment variable to override.
|
||||
else
|
||||
AREA_INFO := Detected area=$(area) based on finding a 'data/$(area).osm.pbf' file. Use 'area' parameter or environment variable to override.
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
ifneq ($(strip $(AREA_INFO)),)
|
||||
define assert_area_is_given
|
||||
@echo "$(AREA_INFO)"
|
||||
endef
|
||||
endif
|
||||
|
||||
# If set, this file will be downloaded in download-osm and imported in the import-osm targets
|
||||
PBF_FILE ?= data/$(area).osm.pbf
|
||||
|
||||
# For download-osm, allow URL parameter to download file from a given URL. Area param must still be provided.
|
||||
ifneq ($(strip $(url)),)
|
||||
DOWNLOAD_AREA := $(url)
|
||||
else
|
||||
DOWNLOAD_AREA := $(area)
|
||||
endif
|
||||
|
||||
# The file is placed into the $EXPORT_DIR=/export (mapped to ./data)
|
||||
export MBTILES_FILE ?= $(area).mbtiles
|
||||
MBTILES_LOCAL_FILE = data/$(MBTILES_FILE)
|
||||
|
||||
ifeq ($(strip $(DIFF_MODE)),true)
|
||||
# import-osm implementation requires IMPOSM_CONFIG_FILE to be set to a valid file
|
||||
# For static (no-updates) import, we don't need to override the default value
|
||||
# For the update mode, set location of the dynamically-generated area-based config file
|
||||
export IMPOSM_CONFIG_FILE = data/$(area).repl.json
|
||||
endif
|
||||
|
||||
# Load area-specific bbox file that gets generated by the download-osm --bbox
|
||||
AREA_BBOX_FILE ?= data/$(area).bbox
|
||||
ifneq (,$(wildcard $(AREA_BBOX_FILE)))
|
||||
cat := $(if $(filter $(OS),Windows_NT),type,cat)
|
||||
BBOX := $(shell $(cat) ${AREA_BBOX_FILE})
|
||||
export BBOX
|
||||
endif
|
||||
|
||||
define HELP_MESSAGE
|
||||
==============================================================================
|
||||
OpenMapTiles https://github.com/openmaptiles/openmaptiles
|
||||
|
||||
Hints for testing areas
|
||||
make list-geofabrik # list actual geofabrik OSM extracts for download -> <<your-area>>
|
||||
./quickstart.sh <<your-area>> # example: ./quickstart.sh madagascar
|
||||
|
||||
Hints for designers:
|
||||
make start-maputnik # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]
|
||||
make start-postserve # start dynamic tile server [ see $(OMT_HOST):$(PPORT) ]
|
||||
make stop-postserve # stop dynamic tile server
|
||||
make start-tileserver # start maptiler/tileserver-gl [ see $(OMT_HOST):$(TPORT) ]
|
||||
|
||||
Hints for developers:
|
||||
make # build source code
|
||||
make bash # start openmaptiles-tools /bin/bash terminal
|
||||
make generate-bbox-file # compute bounding box of a data file and store it in a file
|
||||
make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]
|
||||
make generate-qa # statistics for a given layer's field
|
||||
make generate-tiles-pg # generate vector tiles based on .env settings using PostGIS ST_MVT()
|
||||
make generate-tiles # generate vector tiles based on .env settings using Mapnik (obsolete)
|
||||
make test-sql # run unit tests on the OpenMapTiles SQL schema
|
||||
cat .env # list PG database and MIN_ZOOM and MAX_ZOOM information
|
||||
cat quickstart.log # transcript of the last ./quickstart.sh run
|
||||
make help # help about available commands
|
||||
|
||||
Hints for downloading & importing data:
|
||||
make list-geofabrik # list actual geofabrik OSM extracts for download
|
||||
make list-bbbike # list actual BBBike OSM extracts for download
|
||||
make download area=albania # download OSM data from any source and create config file
|
||||
make download-geofabrik area=albania # download OSM data from geofabrik.de and create config file
|
||||
make download-osmfr area=asia/qatar # download OSM data from openstreetmap.fr and create config file
|
||||
make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file
|
||||
make import-data # Import data from OpenStreetMapData, Natural Earth and OSM Lake Labels.
|
||||
make import-osm # Import OSM data with the mapping rules from build/mapping.yaml
|
||||
make import-wikidata # Import labels from Wikidata
|
||||
make import-sql # Import layers (run this after modifying layer SQL)
|
||||
|
||||
Hints for database management:
|
||||
make psql # start PostgreSQL console
|
||||
make psql-list-tables # list all PostgreSQL tables
|
||||
make list-views # list PostgreSQL public schema views
|
||||
make list-tables # list PostgreSQL public schema tables
|
||||
make vacuum-db # PostgreSQL: VACUUM ANALYZE
|
||||
make analyze-db # PostgreSQL: ANALYZE
|
||||
make destroy-db # remove docker containers and PostgreSQL data volume
|
||||
make start-db # start PostgreSQL, creating it if it doesn't exist
|
||||
make start-db-preloaded # start PostgreSQL, creating data-prepopulated one if it doesn't exist
|
||||
make stop-db # stop PostgreSQL database without destroying the data
|
||||
|
||||
Hints for Docker management:
|
||||
make clean-unnecessary-docker # clean unnecessary docker image(s) and container(s)
|
||||
make refresh-docker-images # refresh openmaptiles docker images from Docker HUB
|
||||
make remove-docker-images # remove openmaptiles docker images
|
||||
make list-docker-images # show a list of available docker images
|
||||
==============================================================================
|
||||
endef
|
||||
export HELP_MESSAGE
|
||||
|
||||
#
|
||||
# TARGETS
|
||||
#
|
||||
|
||||
.PHONY: all
|
||||
all: init-dirs build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "$$HELP_MESSAGE" | less
|
||||
|
||||
define win_fs_error
|
||||
( \
|
||||
echo "" ;\
|
||||
echo "ERROR: Windows native filesystem" ;\
|
||||
echo "" ;\
|
||||
echo "Please avoid running OpenMapTiles in a Windows filesystem." ;\
|
||||
echo "See https://github.com/openmaptiles/openmaptiles/issues/1095#issuecomment-817095465" ;\
|
||||
echo "" ;\
|
||||
exit 1 ;\
|
||||
)
|
||||
endef
|
||||
|
||||
.PHONY: init-dirs
|
||||
init-dirs:
|
||||
@mkdir -p build/sql/parallel
|
||||
|
|
@ -63,68 +253,47 @@ init-dirs:
|
|||
@mkdir -p cache
|
||||
@ ! ($(DOCKER_COMPOSE) 2>/dev/null run $(DC_OPTS) openmaptiles-tools df --output=fstype /tileset| grep -q 9p) < /dev/null || ($(win_fs_error))
|
||||
|
||||
build/openmaptiles.tm2source/data.yml: init-dirs
|
||||
ifeq (,$(wildcard build/openmaptiles.tm2source/data.yml))
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
|
||||
'generate-tm2source $(TILESET_FILE) --host="$(PGHOST)" --port=$(PGPORT) --database="$(PGDATABASE)" --user="$(PGUSER)" --password="$(PGPASSWORD)" > $@'
|
||||
endif
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "=============================================================================="
|
||||
@echo " OpenMapTiles https://github.com/openmaptiles/openmaptiles "
|
||||
@echo "Hints for testing areas "
|
||||
@echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <<your-area>> "
|
||||
@echo " make list # list actual geofabrik OSM extracts for download -> <<your-area>> "
|
||||
@echo " ./quickstart.sh <<your-area>> # example: ./quickstart.sh madagascar "
|
||||
@echo " "
|
||||
@echo "Hints for designers:"
|
||||
@echo " make start-postserve # start Postserver + Maputnik Editor [ see localhost:8088 ] "
|
||||
@echo " make start-tileserver # start klokantech/tileserver-gl [ see localhost:8080 ] "
|
||||
@echo " "
|
||||
@echo "Hints for developers:"
|
||||
@echo " make # build source code"
|
||||
@echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file"
|
||||
@echo " make psql # start PostgreSQL console"
|
||||
@echo " make psql-list-tables # list all PostgreSQL tables"
|
||||
@echo " make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE"
|
||||
@echo " make psql-analyze # PostgreSQL: ANALYZE"
|
||||
@echo " make generate-qareports # generate reports [./build/qareports]"
|
||||
@echo " make generate-devdoc # generate devdoc including graphs for all layers [./build/devdoc]"
|
||||
@echo " make etl-graph # hint for generating a single etl graph"
|
||||
@echo " make mapping-graph # hint for generating a single mapping graph"
|
||||
@echo " make import-sql-dev # start import-sql /bin/bash terminal"
|
||||
@echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)"
|
||||
@echo " make clean-docker # remove docker containers, PG data volume"
|
||||
@echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment"
|
||||
@echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)"
|
||||
@echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB"
|
||||
@echo " make remove-docker-images # remove openmaptiles docker images"
|
||||
@echo " make pgclimb-list-views # list PostgreSQL public schema views"
|
||||
@echo " make pgclimb-list-tables # list PostgreSQL public schema tables"
|
||||
@echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM information"
|
||||
@echo " cat quickstart.log # backup of the last ./quickstart.sh"
|
||||
@echo " make help # help about available commands"
|
||||
@echo "=============================================================================="
|
||||
build/mapping.yaml: init-dirs
|
||||
ifeq (,$(wildcard build/mapping.yaml))
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
|
||||
'generate-imposm3 $(TILESET_FILE) > $@'
|
||||
endif
|
||||
|
||||
.PHONY: build
|
||||
build:
|
||||
mkdir -p build
|
||||
|
||||
build/openmaptiles.tm2source/data.yml: build
|
||||
mkdir -p build/openmaptiles.tm2source
|
||||
docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml
|
||||
|
||||
build/mapping.yaml: build
|
||||
docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > build/mapping.yaml
|
||||
|
||||
build/tileset.sql: build
|
||||
docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql
|
||||
.PHONY: build-sql
|
||||
build-sql: init-dirs
|
||||
ifeq (,$(wildcard build/sql/run_last.sql))
|
||||
@mkdir -p build/sql/parallel
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
|
||||
'generate-sql $(TILESET_FILE) --dir ./build/sql \
|
||||
&& generate-sqltomvt $(TILESET_FILE) \
|
||||
--key --gzip --postgis-ver 3.0.1 \
|
||||
--function --fname=getmvt >> ./build/sql/run_last.sql'
|
||||
endif
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql
|
||||
clean: clean-test-data
|
||||
rm -rf build
|
||||
|
||||
.PHONY: clean-docker
|
||||
clean-docker:
|
||||
docker-compose down -v --remove-orphans
|
||||
docker-compose rm -fv
|
||||
docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true
|
||||
clean-test-data:
|
||||
rm -rf data/changes.state.txt
|
||||
rm -rf data/last.state.txt
|
||||
rm -rf data/changes.repl.json
|
||||
|
||||
.PHONY: destroy-db
|
||||
# TODO: Use https://stackoverflow.com/a/27852388/177275
|
||||
destroy-db: DC_PROJECT := $(shell echo $(DC_PROJECT) | tr A-Z a-z)
|
||||
destroy-db:
|
||||
$(DOCKER_COMPOSE) down -v --remove-orphans
|
||||
$(DOCKER_COMPOSE) rm -fv
|
||||
docker volume ls -q -f "name=^$(DC_PROJECT)_" | $(XARGS) docker volume rm
|
||||
rm -rf cache
|
||||
mkdir cache
|
||||
|
||||
.PHONY: start-db-nowait
|
||||
start-db-nowait: init-dirs
|
||||
|
|
@ -151,18 +320,63 @@ stop-db:
|
|||
list-geofabrik: init-dirs
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik
|
||||
|
||||
.PHONY: download-geofabrik
|
||||
download-geofabrik:
|
||||
@echo =============== download-geofabrik =======================
|
||||
@echo Download area : $(area)
|
||||
@echo [[ example: make download-geofabrik area=albania ]]
|
||||
@echo [[ list areas: make download-geofabrik-list ]]
|
||||
docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area)
|
||||
ls -la ./data/$(area).*
|
||||
@echo "Generated config file: ./data/docker-compose-config.yml"
|
||||
@echo " "
|
||||
cat ./data/docker-compose-config.yml
|
||||
@echo " "
|
||||
.PHONY: list-bbbike
|
||||
list-bbbike: init-dirs
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike
|
||||
|
||||
#
|
||||
# download, download-geofabrik, download-osmfr, and download-bbbike are handled here
|
||||
# The --imposm-cfg will fail for some of the sources, but we ignore that error -- only needed for diff mode
|
||||
#
|
||||
OSM_SERVERS := geofabrik osmfr bbbike
|
||||
ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) download
|
||||
OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@))
|
||||
.PHONY: $(ALL_DOWNLOADS)
|
||||
$(ALL_DOWNLOADS): init-dirs
|
||||
@$(assert_area_is_given)
|
||||
ifneq ($(strip $(url)),)
|
||||
$(if $(OSM_SERVER),$(error url parameter can only be used with non-specific download target:$(newline) make download area=$(area) url="$(url)"$(newline)))
|
||||
endif
|
||||
ifeq (,$(wildcard $(PBF_FILE)))
|
||||
ifeq ($(strip $(DIFF_MODE)),true)
|
||||
@echo "Downloading $(DOWNLOAD_AREA) with replication support into $(PBF_FILE) and $(IMPOSM_CONFIG_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
|
||||
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
|
||||
--imposm-cfg "$(IMPOSM_CONFIG_FILE)" \
|
||||
--bbox "$(AREA_BBOX_FILE)" \
|
||||
--output "$(PBF_FILE)"
|
||||
else
|
||||
@echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
|
||||
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
|
||||
--bbox "$(AREA_BBOX_FILE)" \
|
||||
--output "$(PBF_FILE)"
|
||||
endif
|
||||
@echo ""
|
||||
else
|
||||
ifeq ($(strip $(DIFF_MODE)),true)
|
||||
ifeq (,$(wildcard $(IMPOSM_CONFIG_FILE)))
|
||||
$(error \
|
||||
$(newline) Data files $(PBF_FILE) already exists, but $(IMPOSM_CONFIG_FILE) does not. \
|
||||
$(newline) You probably downloaded the data file before setting DIFF_MODE=true. \
|
||||
$(newline) You can delete the data file $(PBF_FILE) and re-run make download \
|
||||
$(newline) to re-download and generate config, or manually create $(IMPOSM_CONFIG_FILE) \
|
||||
$(newline) See example https://github.com/openmaptiles/openmaptiles-tools/blob/v5.2/bin/config/repl_config.json \
|
||||
$(newline))
|
||||
else
|
||||
@echo "Data files $(PBF_FILE) and replication config $(IMPOSM_CONFIG_FILE) already exists, skipping the download."
|
||||
endif
|
||||
else
|
||||
@echo "Data files $(PBF_FILE) already exists, skipping the download."
|
||||
endif
|
||||
endif
|
||||
|
||||
.PHONY: generate-bbox-file
|
||||
generate-bbox-file:
|
||||
@$(assert_area_is_given)
|
||||
ifeq (,$(wildcard $(AREA_BBOX_FILE)))
|
||||
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm bbox "$(PBF_FILE)" "$(AREA_BBOX_FILE)"
|
||||
else
|
||||
@echo "Configuration file $(AREA_BBOX_FILE) already exists, no need to regenerate. BBOX=$(BBOX)"
|
||||
endif
|
||||
|
||||
.PHONY: psql
|
||||
psql: start-db-nowait
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue