Merge pull request #32 from openmaptiles/quickstart_v2

[quickstart.sh]  -  improvements, checks
pull/111/head
Lukas Martinelli 2016-12-30 21:29:46 +01:00 zatwierdzone przez GitHub
commit 5d4d74b3ff
12 zmienionych plików z 928 dodań i 38 usunięć

3
.env
Wyświetl plik

@ -3,3 +3,6 @@ POSTGRES_USER=openmaptiles
POSTGRES_PASSWORD=openmaptiles
POSTGRES_HOST=postgres
POSTGRES_PORT=5432
QUICKSTART_MIN_ZOOM=0
QUICKSTART_MAX_ZOOM=7
DIFF_MODE=false

10
.gitignore vendored
Wyświetl plik

@ -3,3 +3,13 @@ data.xml
*.thumb.png
*.swp
*.pbf
# quickstart
quickstart.log
# imput / output data
data/*
# generated source files
build/*

Wyświetl plik

@ -13,9 +13,9 @@ before_install:
- sudo apt-get clean
- sudo apt-get update
- sudo apt-key update
- sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.12.0-0~precise
- sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.10.0-0~precise
- sudo rm -f /usr/local/bin/docker-compose
- curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > docker-compose
- curl -L https://github.com/docker/compose/releases/download/1.7.1/docker-compose-`uname -s`-`uname -m` > docker-compose
- chmod +x docker-compose
- sudo mv docker-compose /usr/local/bin
- sudo service docker restart

129
Makefile
Wyświetl plik

@ -1,5 +1,40 @@
all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
help:
@echo "=============================================================================="
@echo " OpenMapTiles https://github.com/openmaptiles/openmaptiles "
@echo "Hints for testing areas "
@echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <<your-area>> "
@echo " make list # list actual geofabrik OSM extracts for download -> <<your-area>> "
@echo " ./quickstart.sh <<your-area>> # example: ./quickstart.sh madagascar "
@echo " "
@echo "Hints for designers:"
@echo " ....TODO.... # start Maputnik "
@echo " ....TODO.... # start Tileserver-gl-light"
@echo " make start-mapbox-studio # start Mapbox Studio"
@echo " "
@echo "Hints for developers:"
@echo " make # build source code "
@echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file"
@echo " make psql # start PostgreSQL console "
@echo " make psql-list-tables # list all PostgreSQL tables "
@echo " make generate-qareports # generate reports [./build/qareports]"
@echo " make generate-devdoc # generate devdoc [./build/devdoc]"
@echo " make import-sql-dev # start import-sql /bin/bash terminal "
@echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)"
@echo " make clean-docker # remove docker containers, PG data volume "
@echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment "
@echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)"
@echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB"
@echo " make remove-docker-images # remove openmaptiles docker images"
@echo " make pgclimb-list-views # list PostgreSQL public schema views"
@echo " make pgclimb-list-tables # list PostgreSQL public schema tabless"
@echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations"
@echo " cat ./quickstart.log # backup of the last ./quickstart.sh "
@echo " ....TODO.... # start lukasmartinelli/postgis-editor"
@echo " make help # help about avaialable commands"
@echo "=============================================================================="
build/openmaptiles.tm2source/data.yml:
mkdir -p build/openmaptiles.tm2source && generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml
@ -11,3 +46,97 @@ build/tileset.sql:
clean:
rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql
clean-docker:
docker-compose down -v --remove-orphans
docker-compose rm -fv
docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true
list-docker-images:
docker images | grep openmaptiles
refresh-docker-images:
docker-compose pull --ignore-pull-failures
remove-docker-images:
@echo "Deleting all openmaptiles related docker image(s)..."
@docker-compose down
@docker images | grep "<none>" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi
@docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi
@docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi
docker-unnecessary-clean:
@echo "Deleting unnecessary container(s)..."
@docker ps -a | grep Exited | awk -F" " '{print $$1}' | xargs --no-run-if-empty docker rm
@echo "Deleting unnecessary image(s)..."
@docker images | grep \<none\> | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi
psql:
docker-compose run --rm import-osm /usr/src/app/psql.sh
psql-list-tables:
docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c "\d+"
psql-pg-stat-reset:
docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();'
forced-clean-sql:
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; "
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION pg_stat_statements;"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';"
pgclimb-list-views:
docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv
pgclimb-list-tables:
docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv
import-sql-dev:
docker-compose run --rm import-sql /bin/bash
import-osm-dev:
docker-compose run --rm import-osm /bin/bash
download-geofabrik:
@echo =============== download-geofabrik =======================
@echo Download area : $(area)
@echo [[ example: make download-geofabrik area=albania ]]
@echo [[ list areas: make download-geofabrik-list ]]
docker-compose run --rm import-osm ./download-geofabrik.sh $(area)
ls -la ./data/$(area).*
@echo "Generated config file: ./data/docker-compose-config.yml"
@echo " "
cat ./data/docker-compose-config.yml
@echo " "
# the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command
list:
docker-compose run --rm import-osm ./download-geofabrik-list.sh
# same as a `make list`
download-geofabrik-list:
docker-compose run --rm import-osm ./download-geofabrik-list.sh
start-mapbox-studio:
docker-compose up mapbox-studio
generate-qareports:
./qa/run.sh
# work in progress - please don't remove
generate-devdoc:
mkdir -p ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/building/building.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/park/park.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/place/place.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/poi/poi.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water/water.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc
docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc

294
QUICKSTART.md 100644
Wyświetl plik

@ -0,0 +1,294 @@
## Quickstart - for small extracts
### Req:
* CPU: AMD64 ( = Intel 64 bit)
* The base docker debian images are x86_64 based, so the ARM,MIPS currently not supported!
* Operating system
* Linux is suggested
* The development and the testing platform is Linux.
* If you are using FreeBSD, Solaris, Windows, ...
* Please give a feedback, share your experience, write a tutorial
* bash
* git
* make
* docker >=1.10
* https://www.docker.com/products/overview
* docker-compose >=1.7.1
* https://docs.docker.com/compose/install/
* disk space ( >= ~15Gb )
* for small extracts >= ~15Gb
* for big extracts ( continents, planet) > 20 ... 1000 Gb
* And depends on
* OpenStreetMap data size
* Zoom level
* memory ( >= 3Gb )
* for small extracts 3Gb-8Gb RAM
* for big extracts ( Europe, Planet) > 8-32 Gb
* internet connections
* for downloading docker images
* for downloading OpenStreetMap data from Geofabrik
Important: The ./quickstart.sh is for small extracts - not optimal for a Planet rendering !!
### First experiment - with `albania` ( small extracts! )
```bash
git clone https://github.com/openmaptiles/openmaptiles.git
cd openmaptiles
./quickstart.sh
```
If you have problems with the quickstart
* check the ./quickstart.log!
* check again the system requirements!
* check the current issues : https://github.com/openmaptiles/openmaptiles/issues
* create new issues:
* create a new gist https://gist.github.com/ from your ./quickstart.log
* double check: if this is not contain any sensitive informations about your system
* create a new issues: https://github.com/openmaptiles/openmaptiles/issues
* describe the problems
* add any important informations your environment
* and link your (quickstart.log) gist !
### Check other extracts
IF the previous step is working,
THAN you can test other available quickstart extracts ( based on geofabrik extracts) !
* We are using https://github.com/julien-noblet/download-geofabrik tool
* The current extract list, and more informations -> `make list`
This is generating `.mbtiles` for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ]
```bash
./quickstart.sh africa
./quickstart.sh alabama
./quickstart.sh alaska
./quickstart.sh albania
./quickstart.sh alberta
./quickstart.sh alps
./quickstart.sh alsace
./quickstart.sh andorra
./quickstart.sh antarctica
./quickstart.sh aquitaine
./quickstart.sh argentina
./quickstart.sh arizona
./quickstart.sh arkansas
./quickstart.sh asia
./quickstart.sh australia
./quickstart.sh austria
./quickstart.sh auvergne
./quickstart.sh azerbaijan
./quickstart.sh azores
./quickstart.sh bangladesh
./quickstart.sh bayern
./quickstart.sh belarus
./quickstart.sh belgium
./quickstart.sh belize
./quickstart.sh berlin
./quickstart.sh bolivia
./quickstart.sh botswana
./quickstart.sh bourgogne
./quickstart.sh brandenburg
./quickstart.sh brazil
./quickstart.sh bremen
./quickstart.sh bretagne
./quickstart.sh buckinghamshire
./quickstart.sh bulgaria
./quickstart.sh california
./quickstart.sh cambridgeshire
./quickstart.sh cameroon
./quickstart.sh canada
./quickstart.sh centre
./quickstart.sh cheshire
./quickstart.sh chile
./quickstart.sh china
./quickstart.sh colombia
./quickstart.sh colorado
./quickstart.sh connecticut
./quickstart.sh cornwall
./quickstart.sh corse
./quickstart.sh croatia
./quickstart.sh cuba
./quickstart.sh cumbria
./quickstart.sh cyprus
./quickstart.sh dach
./quickstart.sh delaware
./quickstart.sh denmark
./quickstart.sh derbyshire
./quickstart.sh devon
./quickstart.sh dorset
./quickstart.sh ecuador
./quickstart.sh egypt
./quickstart.sh england
./quickstart.sh essex
./quickstart.sh estonia
./quickstart.sh ethiopia
./quickstart.sh europe
./quickstart.sh fiji
./quickstart.sh finland
./quickstart.sh florida
./quickstart.sh france
./quickstart.sh germany
./quickstart.sh gloucestershire
./quickstart.sh greece
./quickstart.sh greenland
./quickstart.sh guadeloupe
./quickstart.sh guatemala
./quickstart.sh guinea
./quickstart.sh guyane
./quickstart.sh hamburg
./quickstart.sh hampshire
./quickstart.sh hawaii
./quickstart.sh herefordshire
./quickstart.sh hertfordshire
./quickstart.sh hessen
./quickstart.sh hungary
./quickstart.sh iceland
./quickstart.sh idaho
./quickstart.sh illinois
./quickstart.sh india
./quickstart.sh indiana
./quickstart.sh indonesia
./quickstart.sh iowa
./quickstart.sh irak
./quickstart.sh iran
./quickstart.sh italy
./quickstart.sh japan
./quickstart.sh jordan
./quickstart.sh kansas
./quickstart.sh kazakhstan
./quickstart.sh kent
./quickstart.sh kentucky
./quickstart.sh kenya
./quickstart.sh kosovo
./quickstart.sh kyrgyzstan
./quickstart.sh lancashire
./quickstart.sh latvia
./quickstart.sh lebanon
./quickstart.sh leicestershire
./quickstart.sh lesotho
./quickstart.sh liberia
./quickstart.sh libya
./quickstart.sh liechtenstein
./quickstart.sh limousin
./quickstart.sh lithuania
./quickstart.sh lorraine
./quickstart.sh louisiana
./quickstart.sh luxembourg
./quickstart.sh macedonia
./quickstart.sh madagascar
./quickstart.sh maine
./quickstart.sh malta
./quickstart.sh manitoba
./quickstart.sh martinique
./quickstart.sh maryland
./quickstart.sh massachusetts
./quickstart.sh mayotte
./quickstart.sh mexico
./quickstart.sh michigan
./quickstart.sh minnesota
./quickstart.sh mississippi
./quickstart.sh missouri
./quickstart.sh mittelfranken
./quickstart.sh moldova
./quickstart.sh monaco
./quickstart.sh mongolia
./quickstart.sh montana
./quickstart.sh montenegro
./quickstart.sh morocco
./quickstart.sh nebraska
./quickstart.sh nepal
./quickstart.sh netherlands
./quickstart.sh nevada
./quickstart.sh niederbayern
./quickstart.sh niedersachsen
./quickstart.sh nigeria
./quickstart.sh norfolk
./quickstart.sh norway
./quickstart.sh nottinghamshire
./quickstart.sh nunavut
./quickstart.sh oberbayern
./quickstart.sh oberfranken
./quickstart.sh oberpfalz
./quickstart.sh ohio
./quickstart.sh oklahoma
./quickstart.sh ontario
./quickstart.sh oregon
./quickstart.sh oxfordshire
./quickstart.sh pakistan
./quickstart.sh paraguay
./quickstart.sh pennsylvania
./quickstart.sh peru
./quickstart.sh philippines
./quickstart.sh picardie
./quickstart.sh poland
./quickstart.sh portugal
./quickstart.sh quebec
./quickstart.sh reunion
./quickstart.sh romania
./quickstart.sh saarland
./quickstart.sh sachsen
./quickstart.sh saskatchewan
./quickstart.sh schwaben
./quickstart.sh scotland
./quickstart.sh serbia
./quickstart.sh shropshire
./quickstart.sh slovakia
./quickstart.sh slovenia
./quickstart.sh somalia
./quickstart.sh somerset
./quickstart.sh spain
./quickstart.sh staffordshire
./quickstart.sh suffolk
./quickstart.sh surrey
./quickstart.sh sweden
./quickstart.sh switzerland
./quickstart.sh syria
./quickstart.sh taiwan
./quickstart.sh tajikistan
./quickstart.sh tanzania
./quickstart.sh tennessee
./quickstart.sh texas
./quickstart.sh thailand
./quickstart.sh thueringen
./quickstart.sh turkey
./quickstart.sh turkmenistan
./quickstart.sh ukraine
./quickstart.sh unterfranken
./quickstart.sh uruguay
./quickstart.sh us
./quickstart.sh utah
./quickstart.sh uzbekistan
./quickstart.sh vermont
./quickstart.sh vietnam
./quickstart.sh virginia
./quickstart.sh wales
./quickstart.sh washington
./quickstart.sh wiltshire
./quickstart.sh wisconsin
./quickstart.sh wyoming
./quickstart.sh yukon
```
### Check other commands
`make help`
### Change MIN_ZOOM and MAX_ZOOM
modify the settings in the `.env` file
* QUICKSTART_MIN_ZOOM=0
* QUICKSTART_MAX_ZOOM=7
and re-start `./quickstart.sh `
* the new config file re-generating to here ./data/docker-compose-config.yml
* Known problems:
* If you use same area - then the ./data/docker-compose-config.yml not re-generating, so you have to modify by hand!
Hints:
* Small increments! Never starts with the MAX_ZOOM = 14
* The suggested MAX_ZOOM = 14 - use only with small extracts

Wyświetl plik

@ -1,8 +0,0 @@
version: "2"
services:
generate-vectortiles:
environment:
BBOX: "8.25,46.97,9.58,47.52"
MIN_ZOOM: "0"
MAX_ZOOM: "7"

Wyświetl plik

@ -26,7 +26,7 @@ services:
links:
- postgres
import-osm:
image: "openmaptiles/import-osm:0.1"
image: "openmaptiles/import-osm:latest"
env_file: .env
environment:
DIFF_MODE: ${DIFF_MODE}
@ -43,6 +43,14 @@ services:
- postgres
volumes:
- ./build:/sql
openmaptiles-tools:
image: "openmaptiles/openmaptiles-tools:latest"
env_file: .env
links:
- postgres
volumes:
- .:/tileset
- ./build:/sql
mapbox-studio:
image: "osm2vectortiles/mapbox-studio"
volumes:

35
qa/layer_freq.sh 100755
Wyświetl plik

@ -0,0 +1,35 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
layerid=$1
classvars=$2
echo "# FREQ - $layerid group by : $classvars "
for z in {0..15}
do
echo " "
echo "## $layerid z$z - freq"
SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
select $classvars , count(*) as _count_ from
( $SQL ) as t
GROUP BY $classvars
ORDER BY $classvars
;
END
)
#echo "\`\`\`sql"
#echo "$SQLCODE"
#echo "\`\`\`"
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
done

Wyświetl plik

@ -0,0 +1,40 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
layerid=$1
var=$2
echo "# NUMVAR-ANALYZE - $layerid - $var "
for z in {0..15}
do
echo " "
echo "## $layerid z$z - $var "
SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
SELECT
count($var) as count
,min($var) as min
,max($var) as max
,avg($var) as avg
,stddev($var) as stddev
,variance($var) as variance
FROM
( $SQL ) as t
;
END
)
#echo "\`\`\`sql"
#echo "$SQLCODE"
#echo "\`\`\`"
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
done

Wyświetl plik

@ -0,0 +1,34 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
layerid=$1
classvar=$2
for z in {0..15}
do
echo " "
echo "## $layerid z$z max length ($classvar)"
SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from
( $SQL ) as t
WHERE length( $classvar ) > 0
ORDER BY length( $classvar ) DESC NULLS LAST
LIMIT 30
;
END
)
#echo "\`\`\`sql"
#echo "$SQLCODE"
#echo "\`\`\`"
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
done

83
qa/run.sh 100755
Wyświetl plik

@ -0,0 +1,83 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
# example call from the parent folder : ./qa/run.sh
# or make generate-qareports
# ---- freq ---------------------------------------------------------------
mkdir -p ./build/qareports
rm -f ./build/qareports/*.md
# -----
./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md
###Todo:./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md
./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md
###Todo: ./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md
./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md
#./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md
./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md
./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md
./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md
./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md
./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md
./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md
./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md
./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md
./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md
./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md
./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md
./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md
./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md
./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md
./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md
./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md
./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md
./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md
./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md
./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md
./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md
./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md
# ---- toplength -------------------------------------------
./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md
./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md
./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md
./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md
./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md
./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md
./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md
###Todo: ./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md
./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md
./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md
./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md
# ---- numvar analyze -------------------------------------
./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md
./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md
./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md

Wyświetl plik

@ -3,44 +3,306 @@ set -o errexit
set -o pipefail
set -o nounset
DOCKER_EXEC=docker
DC_EXEC=docker-compose
#Check installed versions
echo "This requires a docker engine version 1.10.0+ and docker-compose 1.6.0+. If not, it is expected to fail. See https://docs.docker.com/engine/installation/ and https://docs.docker.com/compose/install/"
$DOCKER_EXEC --version
$DC_EXEC --version
###########################################
# OpenMapTiles quickstart.sh for x86_64 linux
#
# Example calls ...
# ./quickstart.sh
# ./quickstart.sh africa
# ./quickstart.sh alabama
# ./quickstart.sh alaska
# ./quickstart.sh albania
# ./quickstart.sh alberta
# ./quickstart.sh alps
# ....
#
# to list areas : make download-geofabrik-list
# see more QUICKSTART.md
#
#Remove
$DC_EXEC down
$DC_EXEC rm -fv
echo "Remove old volume"
$DOCKER_EXEC volume ls -q | grep openmaptiles | xargs -r docker volume rm || true
if [ $# -eq 0 ]; then
osm_area=albania # default test country
echo "No parameter - set area=$osm_area "
else
osm_area=$1
fi
testdata=${osm_area}.osm.pbf
echo "Make directories "
## Min versions ...
MIN_COMPOSE_VER=1.7.1
MIN_DOCKER_VER=1.10.0
STARTTIME=$(date +%s)
STARTDATE=$(date -Iminutes)
githash=$( git rev-parse HEAD )
log_file=./quickstart.log
rm -f $log_file
echo " "
echo "====================================================================================="
echo " Docker check & Download images "
echo "-------------------------------------------------------------------------------------"
echo "====> : Please check the docker and docker-compose version!"
echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/"
echo " : (theoretically;not tested) minumum Docker version is $MIN_DOCKER_VER+."
echo " : (theoretically;not tested) minimum Docker-compose version is $MIN_COMPOSE_VER+."
echo " : See the .travis testfile for the current supported versions "
echo " : Your docker systems is:"
docker --version
docker-compose --version
COMPOSE_VER=$(docker-compose version --short)
if [ $COMPOSE_VER "<" $MIN_COMPOSE_VER ]; then
echo "ERR: Your Docker-compose version is Known to have bugs , Please Update docker-compose!"
exit 1
fi
DOCKER_VER="$(docker -v | awk -F '[ ,]+' '{ print $3 }')"
if [ $DOCKER_VER "<" $MIN_DOCKER_VER ]; then
echo "ERR: Your Docker version is not compatible , Please Update docker!"
exit 1
fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Pulling or Refreshing OpenMapTiles docker images "
make refresh-docker-images
##### backup log from here ...
exec &> >(tee -a "$log_file")
echo " "
echo "====================================================================================="
echo " Start processing "
echo "-------------------------------------------------------------------------------------"
echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] "
echo " : This will be logged to the $log_file file ( for debugging ) and to the screen"
echo " : Area : $osm_area "
echo " : Git version : $githash "
echo " : Started : $STARTDATE "
echo " : Your bash version: $BASH_VERSION"
echo " : Your OS : $OSTYPE"
docker --version
docker-compose --version
if [[ "$OSTYPE" == "linux-gnu" ]]; then
echo " : Your system is:"
lsb_release -a
echo " "
echo "-------------------------------------------------------------------------------------"
echo " : This is working on x86_64 ; Your kernel is:"
uname -r
uname -m
KERNEL_CPU_VER=$(uname -m)
if [ "$KERNEL_CPU_VER" != "x86_64" ]; then
echo "ERR: Sorry this is working only on x86_64!"
exit 1
fi
echo " : --- Memory, CPU info ---- "
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
echo "system memory (GB): ${mem} "
grep SwapTotal /proc/meminfo
echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)
cat /proc/meminfo | grep Free
else
echo " "
echo "Warning : This is not a Linux ... ( Less tested ... ) "
echo " "
fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Stopping running services & removing old containers "
make clean-docker
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Checking OpenMapTiles docker images "
docker images | grep openmaptiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) "
mkdir -p pgdata
mkdir -p build
mkdir -p data
testdata=zurich_switzerland.osm.pbf
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) "
rm -f ./data/*.mbtiles
if [ ! -f ./data/${testdata} ]; then
echo "Download $testdata "
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Downloading testdata $testdata "
rm -f ./data/*
wget https://s3.amazonaws.com/metro-extracts.mapzen.com/zurich_switzerland.osm.pbf -P ./data
#wget $testdataurl -P ./data
make download-geofabrik area=${osm_area}
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Osm metadata : $testdata "
cat ./data/osmstat.txt
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Generated docker-compose config "
cat ./data/docker-compose-config.yml
else
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : The testdata ./data/$testdata exists, we don't need to download! "
fi
$DOCKER_EXEC run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make
$DC_EXEC up -d postgres
sleep 30
$DC_EXEC run --rm import-water
$DC_EXEC run --rm import-natural-earth
$DC_EXEC run --rm import-lakelines
$DC_EXEC run --rm import-osm
$DC_EXEC run --rm import-sql
if [ ! -f ./data/${testdata} ]; then
echo " "
echo "Missing ./data/$testdata , Download or Parameter error? "
exit 404
fi
$DC_EXEC -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) "
docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make clean
$DC_EXEC stop postgres
echo "The vectortiles created from $testdata "
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start PostgreSQL service ; create PostgreSQL data volume "
echo " : Source code: https://github.com/openmaptiles/postgis "
echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !"
docker-compose up -d postgres
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Drop and Recreate PostgreSQL public schema "
# Drop all PostgreSQL tables
# This is add an extra safe belt , if the user modify the docker volume seetings
make forced-clean-sql
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing water data from http://openstreetmapdata.com into PostgreSQL "
echo " : Source code: https://github.com/openmaptiles/import-water "
echo " : Data license: http://openstreetmapdata.com/info/license "
echo " : Thank you: http://openstreetmapdata.com/info/supporting "
docker-compose run --rm import-water
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL "
echo " : Source code: https://github.com/openmaptiles/import-natural-earth "
echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use "
echo " : Thank you: Natural Earth Contributors! "
docker-compose run --rm import-natural-earth
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing OpenStreetMap Lakelines data "
echo " : Source code: https://github.com/openmaptiles/import-lakelines "
echo " : https://github.com/lukasmartinelli/osm-lakelines "
echo " : Data license: .. "
docker-compose run --rm import-lakelines
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing OpenStreetMap data: ./data/${testdata} -> imposm3[./build/mapping.yaml] -> PostgreSQL"
echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html "
echo " : Thank you Omniscale! "
echo " : Source code: https://github.com/openmaptiles/import-osm "
echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) "
echo " : Thank you OpenStreetMap Contributors ! "
docker-compose run --rm import-osm
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL "
echo " : Source code: https://github.com/openmaptiles/import-sql "
docker-compose run --rm import-sql
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "
echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml "
echo " : Output MBTiles: ./data/tiles.mbtiles "
echo " : Source code: https://github.com/openmaptiles/generate-vectortiles "
echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox "
echo " : Thank you https://www.mapbox.com !"
echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles "
echo " : "
echo " : You will see a lot of deprecated warning in the log! This is normal! "
echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... "
docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run --rm generate-vectortiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )"
docker-compose stop postgres
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Inputs - Outputs md5sum for debugging "
rm -f ./data/quickstart_checklist.chk
md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk
md5sum build/tileset.sql >> ./data/quickstart_checklist.chk
md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk
md5sum ./data/${testdata} >> ./data/quickstart_checklist.chk
md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk
md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk
md5sum ./data/osmstat.txt >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk
ENDTIME=$(date +%s)
ENDDATE=$(date -Iminutes)
MODDATE=$(stat -c %y ./data/${testdata} )
echo " "
echo " "
echo "-------------------------------------------------------------------------------------"
echo "-- S u m m a r y --"
echo "-------------------------------------------------------------------------------------"
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : (disk space) We have created a lot of docker images: "
echo " : Hint: you can remove with: docker rmi IMAGE "
docker images | grep openmaptiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:"
echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata "
docker volume ls -q | grep openmaptiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) "
echo " : The OpenMapTiles MBTILES license: ..... "
echo " : We created from $testdata ( file moddate: $MODDATE ) "
echo " : Size: "
ls ./data/*.mbtiles -la
echo "Hello ... start experimenting - see docs ! "
echo " "
echo "-------------------------------------------------------------------------------------"
echo "The ./quickstart.sh $osm_area is finished! "
echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete"
echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !"
echo " "
echo "Start experimenting ! And check the QUICKSTART.MD file !"
echo "Avaialable help commands ( make help ) "
make help
echo "-------------------------------------------------------------------------------------"
echo " Acknowledgments "
echo " Thanks to all free, open source software developers and Open Data Contributors ! "
echo "-------------------------------------------------------------------------------------"