kopia lustrzana https://github.com/openmaptiles/openmaptiles
Use new import-data image (#818)
This is a partial migration of https://github.com/openmaptiles/openmaptiles/pull/785 * Use `import-data` instead of `import-lakelines`, `import-water`, and `import-natural-earth` * Upgrade docker-compose.yml to version 2.3 (allows some extra env var usage in yaml file itself) * Remove `openmaptiles-tools:latest` usage -- no longer needed, can use current version 4.1 * `db-start` does not do a container recreation in case docker-compose.yml definition has changed. * a few minor cleanups in quickstart.shpull/821/head
rodzic
a26ccedf08
commit
2ca55abb7d
6
.env
6
.env
|
@ -1,12 +1,16 @@
|
||||||
|
# This file defines default environment variables for all images
|
||||||
|
|
||||||
|
TOOLS_VERSION=4.1.0
|
||||||
|
|
||||||
POSTGRES_DB=openmaptiles
|
POSTGRES_DB=openmaptiles
|
||||||
POSTGRES_USER=openmaptiles
|
POSTGRES_USER=openmaptiles
|
||||||
POSTGRES_PASSWORD=openmaptiles
|
POSTGRES_PASSWORD=openmaptiles
|
||||||
POSTGRES_HOST=postgres
|
POSTGRES_HOST=postgres
|
||||||
POSTGRES_PORT=5432
|
POSTGRES_PORT=5432
|
||||||
|
|
||||||
QUICKSTART_MIN_ZOOM=0
|
QUICKSTART_MIN_ZOOM=0
|
||||||
QUICKSTART_MAX_ZOOM=7
|
QUICKSTART_MAX_ZOOM=7
|
||||||
DIFF_MODE=false
|
DIFF_MODE=false
|
||||||
TOOLS_VERSION=4.1.0
|
|
||||||
|
|
||||||
BBOX=-180.0,-85.0511,180.0,85.0511
|
BBOX=-180.0,-85.0511,180.0,85.0511
|
||||||
MIN_ZOOM=0
|
MIN_ZOOM=0
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# MapTiler OpenMapTiles
|
# MapTiler OpenMapTiles
|
||||||
#######################
|
#######################
|
||||||
|
|
||||||
# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
|
# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
|
||||||
|
@ -28,16 +28,16 @@ jobs:
|
||||||
# Named steps
|
# Named steps
|
||||||
- name: generate all zooms
|
- name: generate all zooms
|
||||||
run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
|
run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
|
||||||
|
|
||||||
# Runs quickstart
|
# Runs quickstart
|
||||||
- name: quickstart
|
- name: quickstart
|
||||||
env:
|
env:
|
||||||
area: northamptonshire
|
area: northamptonshire
|
||||||
run: bash ./quickstart.sh $area
|
run: bash ./quickstart.sh $area
|
||||||
|
|
||||||
- name: generate devdoc
|
- name: generate devdoc
|
||||||
run: TEST_MODE=yes make generate-devdoc
|
run: TEST_MODE=yes make generate-devdoc
|
||||||
|
|
||||||
# todo: use artifact to store result of tests
|
# todo: use artifact to store result of tests
|
||||||
#- uses: actions/upload-artifact@v1
|
#- uses: actions/upload-artifact@v1
|
||||||
# with:
|
# with:
|
||||||
|
|
25
Makefile
25
Makefile
|
@ -69,7 +69,9 @@ help:
|
||||||
|
|
||||||
.PHONY: init-dirs
|
.PHONY: init-dirs
|
||||||
init-dirs:
|
init-dirs:
|
||||||
mkdir -p build && mkdir -p data && mkdir -p cache
|
@mkdir -p build
|
||||||
|
@mkdir -p data
|
||||||
|
@mkdir -p cache
|
||||||
|
|
||||||
build/openmaptiles.tm2source/data.yml: init-dirs
|
build/openmaptiles.tm2source/data.yml: init-dirs
|
||||||
mkdir -p build/openmaptiles.tm2source
|
mkdir -p build/openmaptiles.tm2source
|
||||||
|
@ -94,12 +96,13 @@ clean-docker:
|
||||||
|
|
||||||
.PHONY: db-start
|
.PHONY: db-start
|
||||||
db-start:
|
db-start:
|
||||||
$(DOCKER_COMPOSE) up -d postgres
|
$(DOCKER_COMPOSE) up --no-recreate -d postgres
|
||||||
@echo "Wait for PostgreSQL to start..."
|
@echo "Wait for PostgreSQL to start..."
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh
|
$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh
|
||||||
|
|
||||||
.PHONY: db-stop
|
.PHONY: db-stop
|
||||||
db-stop:
|
db-stop:
|
||||||
|
@echo "Stopping PostgreSQL..."
|
||||||
$(DOCKER_COMPOSE) stop postgres
|
$(DOCKER_COMPOSE) stop postgres
|
||||||
|
|
||||||
OSM_SERVERS:=geofabrik osmfr bbbike
|
OSM_SERVERS:=geofabrik osmfr bbbike
|
||||||
|
@ -142,22 +145,14 @@ import-sql: db-start all
|
||||||
.PHONY: import-osmsql
|
.PHONY: import-osmsql
|
||||||
import-osmsql: db-start all import-osm import-sql
|
import-osmsql: db-start all import-osm import-sql
|
||||||
|
|
||||||
|
.PHONY: import-data
|
||||||
|
import-data: db-start
|
||||||
|
$(DOCKER_COMPOSE) run $(DC_OPTS) import-data
|
||||||
|
|
||||||
.PHONY: import-borders
|
.PHONY: import-borders
|
||||||
import-borders: db-start
|
import-borders: db-start
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
|
||||||
|
|
||||||
.PHONY: import-water
|
|
||||||
import-water: db-start
|
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) import-water
|
|
||||||
|
|
||||||
.PHONY: import-natural-earth
|
|
||||||
import-natural-earth: db-start
|
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth
|
|
||||||
|
|
||||||
.PHONY: import-lakelines
|
|
||||||
import-lakelines: db-start
|
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines
|
|
||||||
|
|
||||||
.PHONY: generate-tiles
|
.PHONY: generate-tiles
|
||||||
ifneq ($(wildcard data/docker-compose-config.yml),)
|
ifneq ($(wildcard data/docker-compose-config.yml),)
|
||||||
DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
|
DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
|
||||||
|
@ -219,7 +214,7 @@ generate-qareports:
|
||||||
.PHONY: generate-devdoc
|
.PHONY: generate-devdoc
|
||||||
generate-devdoc: init-dirs
|
generate-devdoc: init-dirs
|
||||||
mkdir -p ./build/devdoc && \
|
mkdir -p ./build/devdoc && \
|
||||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
|
||||||
'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
|
'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
|
||||||
generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
|
generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
|
||||||
|
|
||||||
|
|
|
@ -100,9 +100,7 @@ make db-start
|
||||||
Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).
|
Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make import-water
|
make import-data
|
||||||
make import-natural-earth
|
|
||||||
make import-lakelines
|
|
||||||
```
|
```
|
||||||
|
|
||||||
[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
|
[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
|
||||||
|
@ -119,7 +117,7 @@ make download-geofabrik area=albania
|
||||||
```
|
```
|
||||||
|
|
||||||
[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
|
[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
|
||||||
`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
|
`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make import-osm
|
make import-osm
|
||||||
|
|
|
@ -1,7 +1,15 @@
|
||||||
version: "2"
|
# This version must match the MAKE_DC_VERSION value below
|
||||||
|
version: "2.3"
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
pgdata:
|
pgdata:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
postgres_conn:
|
||||||
|
driver: bridge
|
||||||
|
|
||||||
services:
|
services:
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: "openmaptiles/postgis:${TOOLS_VERSION}"
|
image: "openmaptiles/postgis:${TOOLS_VERSION}"
|
||||||
volumes:
|
volumes:
|
||||||
|
@ -11,21 +19,13 @@ services:
|
||||||
ports:
|
ports:
|
||||||
- "5432"
|
- "5432"
|
||||||
env_file: .env
|
env_file: .env
|
||||||
import-natural-earth:
|
|
||||||
image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
|
import-data:
|
||||||
env_file: .env
|
image: "openmaptiles/import-data:${TOOLS_VERSION}"
|
||||||
networks:
|
|
||||||
- postgres_conn
|
|
||||||
import-water:
|
|
||||||
image: "openmaptiles/import-water:${TOOLS_VERSION}"
|
|
||||||
env_file: .env
|
|
||||||
networks:
|
|
||||||
- postgres_conn
|
|
||||||
import-lakelines:
|
|
||||||
image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
|
|
||||||
env_file: .env
|
env_file: .env
|
||||||
networks:
|
networks:
|
||||||
- postgres_conn
|
- postgres_conn
|
||||||
|
|
||||||
import-osm:
|
import-osm:
|
||||||
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
|
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
|
||||||
env_file: .env
|
env_file: .env
|
||||||
|
@ -37,6 +37,7 @@ services:
|
||||||
- ./data:/import
|
- ./data:/import
|
||||||
- ./build:/mapping
|
- ./build:/mapping
|
||||||
- ./cache:/cache
|
- ./cache:/cache
|
||||||
|
|
||||||
import-osm-diff:
|
import-osm-diff:
|
||||||
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
|
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
|
||||||
env_file: .env
|
env_file: .env
|
||||||
|
@ -49,6 +50,7 @@ services:
|
||||||
- ./data:/import
|
- ./data:/import
|
||||||
- ./build:/mapping
|
- ./build:/mapping
|
||||||
- ./cache:/cache
|
- ./cache:/cache
|
||||||
|
|
||||||
update-osm:
|
update-osm:
|
||||||
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
|
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
|
||||||
env_file: .env
|
env_file: .env
|
||||||
|
@ -61,26 +63,21 @@ services:
|
||||||
- ./data:/import
|
- ./data:/import
|
||||||
- ./build:/mapping
|
- ./build:/mapping
|
||||||
- ./cache:/cache
|
- ./cache:/cache
|
||||||
|
|
||||||
openmaptiles-tools:
|
openmaptiles-tools:
|
||||||
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
|
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
|
||||||
env_file: .env
|
env_file: .env
|
||||||
|
environment:
|
||||||
|
# Must match the version of this file (first line)
|
||||||
|
# download-osm will use it when generating a composer file
|
||||||
|
MAKE_DC_VERSION: "2.3"
|
||||||
networks:
|
networks:
|
||||||
- postgres_conn
|
- postgres_conn
|
||||||
volumes:
|
volumes:
|
||||||
- .:/tileset
|
- .:/tileset
|
||||||
- ./data:/import
|
- ./data:/import
|
||||||
- ./build:/sql
|
- ./build:/sql
|
||||||
openmaptiles-tools-latest:
|
|
||||||
# This target exists for experimental tools that have not yet been published.
|
|
||||||
# Do not use this for production.
|
|
||||||
image: "openmaptiles/openmaptiles-tools:latest"
|
|
||||||
env_file: .env
|
|
||||||
networks:
|
|
||||||
- postgres_conn
|
|
||||||
volumes:
|
|
||||||
- .:/tileset
|
|
||||||
- ./data:/import
|
|
||||||
- ./build:/sql
|
|
||||||
generate-changed-vectortiles:
|
generate-changed-vectortiles:
|
||||||
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
|
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
|
||||||
command: ./export-list.sh
|
command: ./export-list.sh
|
||||||
|
@ -90,6 +87,7 @@ services:
|
||||||
networks:
|
networks:
|
||||||
- postgres_conn
|
- postgres_conn
|
||||||
env_file: .env
|
env_file: .env
|
||||||
|
|
||||||
generate-vectortiles:
|
generate-vectortiles:
|
||||||
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
|
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
|
||||||
volumes:
|
volumes:
|
||||||
|
@ -102,6 +100,7 @@ services:
|
||||||
BBOX: ${BBOX}
|
BBOX: ${BBOX}
|
||||||
MIN_ZOOM: ${MIN_ZOOM}
|
MIN_ZOOM: ${MIN_ZOOM}
|
||||||
MAX_ZOOM: ${MAX_ZOOM}
|
MAX_ZOOM: ${MAX_ZOOM}
|
||||||
|
|
||||||
postserve:
|
postserve:
|
||||||
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
|
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
|
||||||
command: postserve openmaptiles.yaml --verbose
|
command: postserve openmaptiles.yaml --verbose
|
||||||
|
@ -112,7 +111,3 @@ services:
|
||||||
- "8090:8090"
|
- "8090:8090"
|
||||||
volumes:
|
volumes:
|
||||||
- .:/tileset
|
- .:/tileset
|
||||||
|
|
||||||
networks:
|
|
||||||
postgres_conn:
|
|
||||||
driver: bridge
|
|
||||||
|
|
|
@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
|
||||||
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
|
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
|
||||||
echo "system memory (GB): ${mem}"
|
echo "system memory (GB): ${mem}"
|
||||||
grep SwapTotal /proc/meminfo
|
grep SwapTotal /proc/meminfo
|
||||||
echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)"
|
echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
|
||||||
cat /proc/meminfo | grep Free
|
grep Free /proc/meminfo
|
||||||
else
|
else
|
||||||
echo " "
|
echo " "
|
||||||
echo "Warning : Platforms other than Linux are less tested"
|
echo "Warning : Platforms other than Linux are less tested"
|
||||||
|
@ -120,7 +120,7 @@ make clean-docker
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : Checking OpenMapTiles docker images "
|
echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")"
|
||||||
docker images | grep openmaptiles
|
docker images | grep openmaptiles
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
|
@ -162,7 +162,7 @@ echo "--------------------------------------------------------------------------
|
||||||
echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
|
echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
|
||||||
echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
|
echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
|
||||||
echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
|
echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
|
||||||
make
|
make all
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
|
@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL public schema "
|
||||||
# This adds an extra safety belt if the user modifies the docker volume settings
|
# This adds an extra safety belt if the user modifies the docker volume settings
|
||||||
make forced-clean-sql
|
make forced-clean-sql
|
||||||
|
|
||||||
echo " "
|
echo "====> : Importing all the data:"
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo " : * Water data from http://osmdata.openstreetmap.de"
|
||||||
echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL "
|
echo " : Data license: https://osmdata.openstreetmap.de/info/license.html"
|
||||||
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water "
|
echo " : * Natural Earth from http://www.naturalearthdata.com"
|
||||||
echo " : Data license: https://osmdata.openstreetmap.de/info/license.html "
|
echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
|
||||||
echo " : Thank you: https://osmdata.openstreetmap.de/info/ "
|
echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
|
||||||
make import-water
|
echo " :"
|
||||||
|
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
|
||||||
echo " "
|
echo " : includes all data from the import-data image"
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo " :"
|
||||||
echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL "
|
echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !"
|
||||||
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth "
|
make import-data
|
||||||
echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use "
|
|
||||||
echo " : Thank you: Natural Earth Contributors! "
|
|
||||||
make import-natural-earth
|
|
||||||
|
|
||||||
echo " "
|
|
||||||
echo "-------------------------------------------------------------------------------------"
|
|
||||||
echo "====> : Start importing OpenStreetMap Lakelines data "
|
|
||||||
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines "
|
|
||||||
echo " : https://github.com/lukasmartinelli/osm-lakelines "
|
|
||||||
echo " : Data license: .. "
|
|
||||||
make import-lakelines
|
|
||||||
|
|
||||||
echo " "
|
echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
|
@ -269,16 +258,13 @@ echo " "
|
||||||
echo "-------------------------------------------------------------------------------------"
|
echo "-------------------------------------------------------------------------------------"
|
||||||
echo "====> : Inputs - Outputs md5sum for debugging "
|
echo "====> : Inputs - Outputs md5sum for debugging "
|
||||||
rm -f ./data/quickstart_checklist.chk
|
rm -f ./data/quickstart_checklist.chk
|
||||||
md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk
|
{
|
||||||
md5sum build/tileset.sql >> ./data/quickstart_checklist.chk
|
find build -type f | sort | xargs md5sum ;
|
||||||
md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk
|
find data -type f | sort | xargs md5sum ;
|
||||||
md5sum "./data/${testdata}" >> ./data/quickstart_checklist.chk
|
} >> ./data/quickstart_checklist.chk
|
||||||
md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk
|
|
||||||
md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk
|
|
||||||
cat ./data/quickstart_checklist.chk
|
cat ./data/quickstart_checklist.chk
|
||||||
|
|
||||||
ENDTIME=$(date +%s)
|
ENDTIME=$(date +%s)
|
||||||
ENDDATE=$(date +"%Y-%m-%dT%H:%M%z")
|
|
||||||
if stat --help >/dev/null 2>&1; then
|
if stat --help >/dev/null 2>&1; then
|
||||||
MODDATE=$(stat -c %y "./data/${testdata}" )
|
MODDATE=$(stat -c %y "./data/${testdata}" )
|
||||||
else
|
else
|
||||||
|
|
Ładowanie…
Reference in New Issue