mirror
/
openmaptiles
kopia lustrzana https://github.com/openmaptiles/openmaptiles
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'master' into maptiler-change

pull/828/head
Yuri Astrakhan 2020-04-24 14:08:36 -04:00
rodzic 3f2f400450 d7873db0c0
commit 79be9bb0bc
11 zmienionych plików z 328 dodań i 160 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
.env
Wyświetl plik

@ -1,12 +1,16 @@
# This file defines default environment variables for all images
TOOLS_VERSION=4.1.0
POSTGRES_DB=openmaptiles POSTGRES_DB=openmaptiles
POSTGRES_USER=openmaptiles POSTGRES_USER=openmaptiles
POSTGRES_PASSWORD=openmaptiles POSTGRES_PASSWORD=openmaptiles
POSTGRES_HOST=postgres POSTGRES_HOST=postgres
POSTGRES_PORT=5432 POSTGRES_PORT=5432
QUICKSTART_MIN_ZOOM=0 QUICKSTART_MIN_ZOOM=0
QUICKSTART_MAX_ZOOM=7 QUICKSTART_MAX_ZOOM=7
DIFF_MODE=false DIFF_MODE=false
TOOLS_VERSION=4.1.0
BBOX=-180.0,-85.0511,180.0,85.0511 BBOX=-180.0,-85.0511,180.0,85.0511
MIN_ZOOM=0 MIN_ZOOM=0

45
.github/workflows/omt_ci.yml vendored
Wyświetl plik

@ -1,45 +0,0 @@
# MapTiler OpenMapTiles
#######################
# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
name: OMT_CI
# Controls when the action will run. Triggers the workflow on push and pull request
# events but only for the master branch
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
# jobs can run parallel
jobs:
# This workflow contains a single job called "build"
build:
# runs on ubuntu (can run on windows and mac os)
runs-on: ubuntu-latest
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
# Named steps
- name: generate all zooms
run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
# Runs quickstart
- name: quickstart
env:
area: northamptonshire
run: bash ./quickstart.sh $area
- name: generate devdoc
run: TEST_MODE=yes make generate-devdoc
# todo: use artifact to store result of tests
#- uses: actions/upload-artifact@v1
# with:
# name: quickstart log file
# path: quickstart.log

200
.github/workflows/tests.yml vendored 100644
Wyświetl plik

@ -0,0 +1,200 @@
# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
name: OpenMapTiles CI
on:
push:
branches: [ master ]
pull_request:
jobs:
integrity_test:
name: Run integrity test
runs-on: ubuntu-latest
steps:
- name: Checkout the changes
uses: actions/checkout@v2
- name: Run quickstart for a small area
env:
area: monaco
run: |
# For now, change the quickstart values directly in the .env file
# TODO: We should probably use env vars instead
sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
./quickstart.sh $area
- name: Save quickstart.log
uses: actions/upload-artifact@v1
with:
name: quickstart.log
path: quickstart.log
- name: Test etldoc images
run: |
export TEST_MODE=yes
make generate-devdoc
performance:
name: Evaluate performance
runs-on: ubuntu-latest
# Even though we technically don't need to wait for integrity test to finish,
# there is no point to run long perf test until we know the code is OK
needs: integrity_test
env:
# Smaller tests (runs everything in about 30 minutes)
TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test equatorial-guinea --test liechtenstein"
TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=12vw07f9W0MiAHIqMztRiIMwahJfqTi21"
## Large test data -- we should switch to it after everything is working ok
# TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test hungary --test isle-of-man"
# TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=1kw7XPDPd1Rc-Zi2XxGLTXdinUSq-S4pT"
steps:
- name: Cache test data download
id: cache-testdata
uses: actions/cache@v1
with:
path: ci_cache
key: "${{ env.TEST_DATA_URL }}"
- name: Download test data on cache miss
if: steps.cache-testdata.outputs.cache-hit != 'true'
run: |
echo "Data file does not exist, downloading $TEST_DATA_URL"
mkdir -p ci_cache
curl --silent --show-error --location --output ci_cache/perf-test-areas-latest.osm.pbf "$TEST_DATA_URL"
- name: Get code
uses: actions/checkout@v2
with:
# Fetch the last two commits in case this is a PR,
# and we need to profile the base branch first
fetch-depth: 2
path: code
- name: Compute git revision hash to cache
id: calc
run: |
# If this is a pull request, we should cache the parent (base) revision
# Otherwise cache the current one
cd code
REV_HASH=$(git log -1 --format="%H")
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
# Take the first parent of the grafted commit (cannot use HEAD^1 with shallow clones)
REV_HASH=$(git cat-file -p $REV_HASH | awk 'NR > 1 {if(/^parent/){print $2; exit}}')
fi
echo "::set-output name=hash::$REV_HASH"
- name: Set up caching for the performance results
uses: actions/cache@v1
with:
path: perf_cache
key: "${{ steps.calc.outputs.hash }}-${{ env.TEST_DATA_URL }}"
- name: Load test data into DB and run performance test
id: main
env:
CACHE_SHA: "${{ steps.calc.outputs.hash }}"
run: |
create_db() {
make clean
make init-dirs
cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas-latest.osm.pbf
make db-destroy
make all
make db-start
time make import-data
time make import-osm
time make import-borders
time make import-wikidata
time make import-sql
}
mkdir -p perf_cache
mkdir -p artifacts
cd code
CURRENT_SHA=$(git log -1 --format="%H")
if [ ! -f ../perf_cache/results.json ]; then
echo "We do not have cached performance results, create them..."
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
git reset --hard ${CURRENT_SHA}^1
fi
create_db
# Use latest tools version because these specific tests do not yet exist in the 4.1 tools version
# Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version
TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \
test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
--record /tileset/results.json
mv results.json ../perf_cache
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
# For pull requests, restore to the PR version before continuing
git reset --hard ${CURRENT_SHA}
fi
else
echo "Found cached performance results"
fi
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
cp ../perf_cache/results.json ../artifacts/base-results.json
else
cp ../perf_cache/results.json ../artifacts/results.json
fi
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
echo "Comparing pull request results with the base..."
create_db
# Use latest tools version because these specific tests do not yet exist in the 4.1 tools version
# Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version
cp ../perf_cache/results.json .
OUTPUT="$(TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \
test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
--compare /tileset/results.json --record /tileset/pr-results.json)"
rm results.json
mv pr-results.json ../artifacts/
# Convert multiline output into a single long string.
# See https://github.community/t5/GitHub-Actions/set-output-Truncates-Multiline-Strings/td-p/37870
OUTPUT="${OUTPUT//'%'/'%25'}"
OUTPUT="${OUTPUT//$'\n'/'%0A'}"
OUTPUT="${OUTPUT//$'\r'/'%0D'}"
# Split into two parts -- before and after the ===== SUMMARY =====
echo "::set-output name=summary::${OUTPUT##*========}"
echo "::set-output name=details::${OUTPUT%%========*}"
fi
- name: Save artifacts
uses: actions/upload-artifact@v1
with:
name: performance_results
path: artifacts
- name: Post a comment on Pull Request
if: "github.event_name == 'pull_request'"
uses: marocchino/sticky-pull-request-comment@v1
timeout-minutes: 1
continue-on-error: true
with:
message: |-
```
${{ steps.main.outputs.summary }}
```
<details>
<summary>expand for details...</summary>
```
${{ steps.main.outputs.details }}
```
</details>
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

98
Makefile
Wyświetl plik

@ -24,7 +24,7 @@ else
endif endif
.PHONY: all .PHONY: all
all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
# Set OpenMapTiles host # Set OpenMapTiles host
OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost) OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
@ -38,8 +38,9 @@ help:
@echo " ./quickstart.sh <<your-area>> # example: ./quickstart.sh madagascar " @echo " ./quickstart.sh <<your-area>> # example: ./quickstart.sh madagascar "
@echo " " @echo " "
@echo "Hints for designers:" @echo "Hints for designers:"
@echo " make start-postserve # start Postserver + Maputnik Editor [ see $(OMT_HOST):8088 ] " @echo " make maputnik-start # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]"
@echo " make start-tileserver # start maptiler/tileserver-gl [ see $(OMT_HOST):8080 ] " @echo " make postserve-start # start dynamic tile server [ see $(OMT_HOST):8090 ]"
@echo " make tileserver-start # start maptiler/tileserver-gl [ see $(OMT_HOST):8080 ]"
@echo " " @echo " "
@echo "Hints for developers:" @echo "Hints for developers:"
@echo " make # build source code" @echo " make # build source code"
@ -53,10 +54,12 @@ help:
@echo " make psql-analyze # PostgreSQL: ANALYZE" @echo " make psql-analyze # PostgreSQL: ANALYZE"
@echo " make generate-qareports # generate reports [./build/qareports]" @echo " make generate-qareports # generate reports [./build/qareports]"
@echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]" @echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]"
@echo " make tools-dev # start openmaptiles-tools /bin/bash terminal"
@echo " make db-destroy # remove docker containers and PostgreSQL data volume"
@echo " make db-start # start PostgreSQL, creating it if it doesn't exist"
@echo " make db-stop # stop PostgreSQL database without destroying the data"
@echo " make import-sql-dev # start import-sql /bin/bash terminal" @echo " make import-sql-dev # start import-sql /bin/bash terminal"
@echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)"
@echo " make clean-docker # remove docker containers, PG data volume"
@echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment"
@echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)" @echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)"
@echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" @echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB"
@echo " make remove-docker-images # remove openmaptiles docker images" @echo " make remove-docker-images # remove openmaptiles docker images"
@ -69,7 +72,9 @@ help:
.PHONY: init-dirs .PHONY: init-dirs
init-dirs: init-dirs:
mkdir -p build && mkdir -p data && mkdir -p cache @mkdir -p build
@mkdir -p data
@mkdir -p cache
build/openmaptiles.tm2source/data.yml: init-dirs build/openmaptiles.tm2source/data.yml: init-dirs
mkdir -p build/openmaptiles.tm2source mkdir -p build/openmaptiles.tm2source
@ -78,15 +83,16 @@ build/openmaptiles.tm2source/data.yml: init-dirs
build/mapping.yaml: init-dirs build/mapping.yaml: init-dirs
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@
build/tileset.sql: init-dirs .PHONY: build-sql
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@ build-sql: init-dirs
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql
.PHONY: clean .PHONY: clean
clean: clean:
rm -rf build rm -rf build
.PHONY: clean-docker .PHONY: db-destroy
clean-docker: db-destroy:
$(DOCKER_COMPOSE) down -v --remove-orphans $(DOCKER_COMPOSE) down -v --remove-orphans
$(DOCKER_COMPOSE) rm -fv $(DOCKER_COMPOSE) rm -fv
docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm
@ -94,14 +100,19 @@ clean-docker:
.PHONY: db-start .PHONY: db-start
db-start: db-start:
$(DOCKER_COMPOSE) up -d postgres $(DOCKER_COMPOSE) up --no-recreate -d postgres
@echo "Wait for PostgreSQL to start..." @echo "Wait for PostgreSQL to start..."
$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh
.PHONY: db-stop .PHONY: db-stop
db-stop: db-stop:
@echo "Stopping PostgreSQL..."
$(DOCKER_COMPOSE) stop postgres $(DOCKER_COMPOSE) stop postgres
.PHONY: list-geofabrik
list-geofabrik:
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik
OSM_SERVERS:=geofabrik osmfr bbbike OSM_SERVERS:=geofabrik osmfr bbbike
ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS)) ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS))
OSM_SERVER=$(patsubst download-%,%,$@) OSM_SERVER=$(patsubst download-%,%,$@)
@ -135,28 +146,20 @@ psql: db-start
import-osm: db-start all import-osm: db-start all
$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm
.PHONY: import-sql
import-sql: db-start all
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
.PHONY: import-osmsql .PHONY: import-osmsql
import-osmsql: db-start all import-osm import-sql import-osmsql: db-start all import-osm import-sql
.PHONY: import-data
import-data: db-start
$(DOCKER_COMPOSE) run $(DC_OPTS) import-data
.PHONY: import-borders .PHONY: import-borders
import-borders: db-start import-borders: db-start
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
.PHONY: import-water .PHONY: import-sql
import-water: db-start import-sql: db-start all
$(DOCKER_COMPOSE) run $(DC_OPTS) import-water $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
.PHONY: import-natural-earth
import-natural-earth: db-start
$(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth
.PHONY: import-lakelines
import-lakelines: db-start
$(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines
.PHONY: generate-tiles .PHONY: generate-tiles
ifneq ($(wildcard data/docker-compose-config.yml),) ifneq ($(wildcard data/docker-compose-config.yml),)
@ -164,11 +167,13 @@ ifneq ($(wildcard data/docker-compose-config.yml),)
endif endif
generate-tiles: init-dirs db-start all generate-tiles: init-dirs db-start all
rm -rf data/tiles.mbtiles rm -rf data/tiles.mbtiles
echo "Generating tiles ..."; \
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles $(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
@echo "Updating generated tile metadata ..."
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles
.PHONY: start-tileserver .PHONY: tileserver-start
start-tileserver: init-dirs tileserver-start: init-dirs
@echo " " @echo " "
@echo "***********************************************************" @echo "***********************************************************"
@echo "* " @echo "* "
@ -188,29 +193,42 @@ start-tileserver: init-dirs
@echo " " @echo " "
docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:8080 maptiler/tileserver-gl --port 8080 docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:8080 maptiler/tileserver-gl --port 8080
.PHONY: start-postserve .PHONY: postserve-start
start-postserve: db-start postserve-start: db-start
@echo " " @echo " "
@echo "***********************************************************" @echo "***********************************************************"
@echo "* " @echo "* "
@echo "* Bring up postserve at $(OMT_HOST):8090" @echo "* Bring up postserve at $(OMT_HOST):8090"
@echo "* --> can view it locally (use make maputnik-start)"
@echo "* --> or can use https://maputnik.github.io/editor"
@echo "* "
@echo "* set data source / TileJSON URL to http://$(OMT_HOST):8090"
@echo "* " @echo "* "
@echo "***********************************************************" @echo "***********************************************************"
@echo " " @echo " "
$(DOCKER_COMPOSE) up -d postserve $(DOCKER_COMPOSE) up -d postserve
docker pull maputnik/editor
.PHONY: postserve-stop
postserve-stop:
$(DOCKER_COMPOSE) stop postserve
.PHONY: maputnik-start
maputnik-start: maputnik-stop postserve-start
@echo " " @echo " "
@echo "***********************************************************" @echo "***********************************************************"
@echo "* " @echo "* "
@echo "* Start maputnik/editor " @echo "* Start maputnik/editor "
@echo "* ---> go to http://$(OMT_HOST):8088 " @echo "* ---> go to http://$(OMT_HOST):8088 "
@echo "* ---> set 'data source' to http://$(OMT_HOST):8090" @echo "* ---> set data source / TileJSON URL to http://$(OMT_HOST):8090"
@echo "* " @echo "* "
@echo "***********************************************************" @echo "***********************************************************"
@echo " " @echo " "
-docker rm -f maputnik_editor
docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor
.PHONY: maputnik-stop
maputnik-stop:
-docker rm -f maputnik_editor
.PHONY: generate-qareports .PHONY: generate-qareports
generate-qareports: generate-qareports:
./qa/run.sh ./qa/run.sh
@ -219,12 +237,12 @@ generate-qareports:
.PHONY: generate-devdoc .PHONY: generate-devdoc
generate-devdoc: init-dirs generate-devdoc: init-dirs
mkdir -p ./build/devdoc && \ mkdir -p ./build/devdoc && \
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
.PHONY: import-sql-dev .PHONY: tools-dev
import-sql-dev: tools-dev:
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash
.PHONY: import-osm-dev .PHONY: import-osm-dev
@ -233,7 +251,7 @@ import-osm-dev:
.PHONY: import-wikidata .PHONY: import-wikidata
import-wikidata: import-wikidata:
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml
.PHONY: psql-pg-stat-reset .PHONY: psql-pg-stat-reset
psql-pg-stat-reset: psql-pg-stat-reset:
@ -277,7 +295,13 @@ list-docker-images:
.PHONY: refresh-docker-images .PHONY: refresh-docker-images
refresh-docker-images: refresh-docker-images:
ifneq ($(strip $(NO_REFRESH)),)
@echo "Skipping docker image refresh"
else
@echo ""
@echo "Refreshing docker images... Use NO_REFRESH=1 to skip."
$(DOCKER_COMPOSE) pull --ignore-pull-failures $(DOCKER_COMPOSE) pull --ignore-pull-failures
endif
.PHONY: remove-docker-images .PHONY: remove-docker-images
remove-docker-images: remove-docker-images:

6
README.md
Wyświetl plik

@ -100,9 +100,7 @@ make db-start
Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines). Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).
```bash ```bash
make import-water make import-data
make import-natural-earth
make import-lakelines
``` ```
[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory. [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
@ -119,7 +117,7 @@ make download-geofabrik area=albania
``` ```
[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from [Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. `build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
```bash ```bash
make import-osm make import-osm

57
docker-compose.yml
Wyświetl plik

@ -1,9 +1,17 @@
version: "2" # This version must match the MAKE_DC_VERSION value below
version: "2.3"
volumes: volumes:
pgdata: pgdata:
networks:
postgres_conn:
driver: bridge
services: services:
postgres: postgres:
image: "openmaptiles/postgis:${TOOLS_VERSION}" image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
volumes: volumes:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
networks: networks:
@ -11,21 +19,13 @@ services:
ports: ports:
- "5432" - "5432"
env_file: .env env_file: .env
import-natural-earth:
image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}" import-data:
env_file: .env image: "openmaptiles/import-data:${TOOLS_VERSION}"
networks:
- postgres_conn
import-water:
image: "openmaptiles/import-water:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
import-lakelines:
image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
env_file: .env env_file: .env
networks: networks:
- postgres_conn - postgres_conn
import-osm: import-osm:
image: "openmaptiles/import-osm:${TOOLS_VERSION}" image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env env_file: .env
@ -37,6 +37,7 @@ services:
- ./data:/import - ./data:/import
- ./build:/mapping - ./build:/mapping
- ./cache:/cache - ./cache:/cache
import-osm-diff: import-osm-diff:
image: "openmaptiles/import-osm:${TOOLS_VERSION}" image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env env_file: .env
@ -49,6 +50,7 @@ services:
- ./data:/import - ./data:/import
- ./build:/mapping - ./build:/mapping
- ./cache:/cache - ./cache:/cache
update-osm: update-osm:
image: "openmaptiles/import-osm:${TOOLS_VERSION}" image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env env_file: .env
@ -61,26 +63,25 @@ services:
- ./data:/import - ./data:/import
- ./build:/mapping - ./build:/mapping
- ./cache:/cache - ./cache:/cache
openmaptiles-tools: openmaptiles-tools:
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
env_file: .env env_file: .env
environment:
# Must match the version of this file (first line)
# download-osm will use it when generating a composer file
MAKE_DC_VERSION: "2.3"
# Allow DIFF_MODE to be overwritten from shell
DIFF_MODE: ${DIFF_MODE}
networks: networks:
- postgres_conn - postgres_conn
volumes: volumes:
- .:/tileset - .:/tileset
- ./data:/import - ./data:/import
- ./build:/sql - ./build:/sql
openmaptiles-tools-latest: - ./build:/mapping
# This target exists for experimental tools that have not yet been published. - ./cache:/cache
# Do not use this for production.
image: "openmaptiles/openmaptiles-tools:latest"
env_file: .env
networks:
- postgres_conn
volumes:
- .:/tileset
- ./data:/import
- ./build:/sql
generate-changed-vectortiles: generate-changed-vectortiles:
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
command: ./export-list.sh command: ./export-list.sh
@ -90,6 +91,7 @@ services:
networks: networks:
- postgres_conn - postgres_conn
env_file: .env env_file: .env
generate-vectortiles: generate-vectortiles:
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
volumes: volumes:
@ -102,6 +104,7 @@ services:
BBOX: ${BBOX} BBOX: ${BBOX}
MIN_ZOOM: ${MIN_ZOOM} MIN_ZOOM: ${MIN_ZOOM}
MAX_ZOOM: ${MAX_ZOOM} MAX_ZOOM: ${MAX_ZOOM}
postserve: postserve:
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
command: postserve openmaptiles.yaml --verbose command: postserve openmaptiles.yaml --verbose
@ -112,7 +115,3 @@ services:
- "8090:8090" - "8090:8090"
volumes: volumes:
- .:/tileset - .:/tileset
networks:
postgres_conn:
driver: bridge

6
layers/building/building.sql
Wyświetl plik

@ -72,7 +72,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS (
FALSE as hide_3d FALSE as hide_3d
FROM FROM
osm_building_polygon obp osm_building_polygon obp
WHERE osm_id < 0 -- OSM mulipolygons once imported can give unique postgis polygons with holes, or multi parts polygons
WHERE osm_id < 0 AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon')
UNION ALL UNION ALL
-- etldoc: osm_building_polygon -> layer_building:z14_ -- etldoc: osm_building_polygon -> layer_building:z14_
@ -88,7 +89,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS (
FROM FROM
osm_building_polygon obp osm_building_polygon obp
LEFT JOIN osm_building_relation obr ON (obr.member = obp.osm_id) LEFT JOIN osm_building_relation obr ON (obr.member = obp.osm_id)
WHERE obp.osm_id >= 0 -- Only check for ST_Polygon as we exclude buildings from relations keeping only positive ids
WHERE obp.osm_id >= 0 AND ST_GeometryType(obp.geometry) = 'ST_Polygon'
); );
CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int) CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int)

4
layers/poi/poi.yaml
Wyświetl plik

@ -146,10 +146,10 @@ layer:
query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, subclass, agg_stop, layer, level, indoor, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, subclass, agg_stop, layer, level, indoor, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
schema: schema:
- ./public_transport_stop_type.sql - ./public_transport_stop_type.sql
- ./update_poi_polygon.sql
- ./update_poi_point.sql
- ./class.sql - ./class.sql
- ./poi_stop_agg.sql - ./poi_stop_agg.sql
- ./update_poi_polygon.sql
- ./update_poi_point.sql
- ./layer.sql - ./layer.sql
datasources: datasources:
- type: imposm3 - type: imposm3

3
layers/poi/poi_stop_agg.sql
Wyświetl plik

@ -33,6 +33,3 @@ CREATE MATERIALIZED VIEW osm_poi_stop_rank AS (
subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway') subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway')
ORDER BY p.uic_ref, rk ORDER BY p.uic_ref, rk
) /* DELAY_MATERIALIZED_VIEW_CREATION */; ) /* DELAY_MATERIALIZED_VIEW_CREATION */;
ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL;
SELECT update_osm_poi_point_agg();

3
layers/poi/update_poi_point.sql
Wyświetl plik

@ -43,6 +43,9 @@ BEGIN
END; END;
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL;
SELECT update_osm_poi_point_agg();
-- Handle updates -- Handle updates
CREATE SCHEMA IF NOT EXISTS poi_point; CREATE SCHEMA IF NOT EXISTS poi_point;

60
quickstart.sh
Wyświetl plik

@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
echo "system memory (GB): ${mem}" echo "system memory (GB): ${mem}"
grep SwapTotal /proc/meminfo grep SwapTotal /proc/meminfo
echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)" echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
cat /proc/meminfo | grep Free grep Free /proc/meminfo
else else
echo " " echo " "
echo "Warning : Platforms other than Linux are less tested" echo "Warning : Platforms other than Linux are less tested"
@ -116,11 +116,11 @@ fi
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : Stopping running services & removing old containers" echo "====> : Stopping running services & removing old containers"
make clean-docker make db-destroy
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : Checking OpenMapTiles docker images " echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")"
docker images | grep openmaptiles docker images | grep openmaptiles
echo " " echo " "
@ -162,7 +162,7 @@ echo "--------------------------------------------------------------------------
echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
make make all
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL public schema "
# This adds an extra safety belt if the user modifies the docker volume settings # This adds an extra safety belt if the user modifies the docker volume settings
make forced-clean-sql make forced-clean-sql
echo " " echo "====> : Importing all the data:"
echo "-------------------------------------------------------------------------------------" echo " : * Water data from http://osmdata.openstreetmap.de"
echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL " echo " : Data license: https://osmdata.openstreetmap.de/info/license.html"
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water " echo " : * Natural Earth from http://www.naturalearthdata.com"
echo " : Data license: https://osmdata.openstreetmap.de/info/license.html " echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
echo " : Thank you: https://osmdata.openstreetmap.de/info/ " echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
make import-water echo " :"
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
echo " " echo " : includes all data from the import-data image"
echo "-------------------------------------------------------------------------------------" echo " :"
echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !"
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth " make import-data
echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use "
echo " : Thank you: Natural Earth Contributors! "
make import-natural-earth
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing OpenStreetMap Lakelines data "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines "
echo " : https://github.com/lukasmartinelli/osm-lakelines "
echo " : Data license: .. "
make import-lakelines
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
@ -269,16 +258,13 @@ echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : Inputs - Outputs md5sum for debugging " echo "====> : Inputs - Outputs md5sum for debugging "
rm -f ./data/quickstart_checklist.chk rm -f ./data/quickstart_checklist.chk
md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk {
md5sum build/tileset.sql >> ./data/quickstart_checklist.chk find build -type f | sort | xargs md5sum ;
md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk find data -type f | sort | xargs md5sum ;
md5sum "./data/${testdata}" >> ./data/quickstart_checklist.chk } >> ./data/quickstart_checklist.chk
md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk
md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk cat ./data/quickstart_checklist.chk
ENDTIME=$(date +%s) ENDTIME=$(date +%s)
ENDDATE=$(date +"%Y-%m-%dT%H:%M%z")
if stat --help >/dev/null 2>&1; then if stat --help >/dev/null 2>&1; then
MODDATE=$(stat -c %y "./data/${testdata}" ) MODDATE=$(stat -c %y "./data/${testdata}" )
else else
@ -312,8 +298,8 @@ echo "We saved the log file to $log_file ( for debugging ) You can compare with
echo " " echo " "
echo "Start experimenting! And check the QUICKSTART.MD file!" echo "Start experimenting! And check the QUICKSTART.MD file!"
echo " " echo " "
echo "* Use make start-postserve to explore tile generation on request" echo "* Use make maputnik-start to explore tile generation on request"
echo "* Use make start-tileserver to view pre-generated tiles" echo "* Use make tileserver-start to view pre-generated tiles"
echo " " echo " "
echo "Available help commands (make help) " echo "Available help commands (make help) "
make help make help