diff --git a/.env b/.env index 54cf241d..6a655725 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ -POSTGRES_DB=osm -POSTGRES_USER=osm -POSTGRES_PASSWORD=osm +POSTGRES_DB=openmaptiles +POSTGRES_USER=openmaptiles +POSTGRES_PASSWORD=openmaptiles POSTGRES_HOST=postgres POSTGRES_PORT=5432 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..3e63fa3b --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql + +build/openmaptiles.tm2source/data.yml: + mkdir -p build/openmaptiles.tm2source && generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml + +build/mapping.yaml: + mkdir -p build && generate-imposm3 openmaptiles.yaml > build/mapping.yaml + +build/tileset.sql: + mkdir -p build && generate-sql openmaptiles.yaml > build/tileset.sql + +clean: + rm -rf build diff --git a/README.md b/README.md index 6031cc72..4619c1ec 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,19 @@ To work on the database and vector tile schema read the *Develop* section. ## Develop -To work on *osm2vectortiles.tm2source* we recommend using Docker. +To work on *osm2vectortiles.tm2source* you need Docker and Python. - Install [Docker](https://docs.docker.com/engine/installation/) - Install [Docker Compose](https://docs.docker.com/compose/install/) +- Install [OpenMapTiles tools](https://github.com/openmaptiles/openmaptiles-tools) with `pip install openmaptiles-tools` + +### Build + +Build the tileset. + +``` +make +``` ### Prepare the Database @@ -69,23 +78,20 @@ docker-compose run import-natural-earth Import [OpenStreetMap](http://wiki.openstreetmap.org/wiki/Osm2pgsql) data based on the [ClearTables osm2pgsql style](https://github.com/ClearTables/ClearTables). In order to do this you first need to clone the latest ClearTables. -- [ ] *Should we use a submodule for ClearTables? On the other hand it is quite convenient to able to work directly on ClearTables while working on the style.* - ```bash -git clone https://github.com/ClearTables/ClearTables.git docker-compose run import-osm ``` -### Work on Vector Tile Schema +### Work on Layers -Build and run the `db-schema` container each time you modify SQL code inside `./schema`. +Each time you modify layer SQL code run `make` and `docker-compose run import-sql`. -```bash -docker-compose build db-schema -docker-compose run db-schema +``` +make +docker-compose run import-sql ``` -To work on the *data.yml* you can start up Mapbox Studio Classic in a container +To look at the vector tiles you can start up Mapbox Studio Classic in a container and visit `localhost:3000` and open the vector source project under `projects/osm2vectortiles.tm2source`. ```bash diff --git a/docker-compose.yml b/docker-compose.yml index f2496987..85e3e85b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,50 +1,52 @@ -version: '2' +version: "2" volumes: pgdata: services: postgres: - image: "osm2vectortiles/postgis:latest" + image: "openmaptiles/postgis:latest" volumes: - pgdata:/var/lib/postgresql/data ports: - "5432" env_file: .env - db-schema: - build: "./schema" + import-natural-earth: + image: "openmaptiles/import-natural-earth" env_file: .env links: - - postgres:postgres + - postgres + import-water: + image: "openmaptiles/import-water" + env_file: .env + links: + - postgres import-osm: - image: "osm2vectortiles/import-osm" + image: "openmaptiles/import-osm" env_file: .env links: - postgres volumes: - ./data:/import - - ./:/mapping - import-natural-earth: - image: "osm2vectortiles/import-natural-earth" - env_file: .env - links: - - postgres - import-water: - image: "osm2vectortiles/import-water" + - ./build:/mapping + import-sql: + image: "openmaptiles/import-sql" env_file: .env links: - postgres + volumes: + - ./build:/sql mapbox-studio: image: "osm2vectortiles/mapbox-studio" volumes: - - ./:/projects/osm2vectortiles.tm2source + - ./build/openmaptiles.tm2source:/projects/openmaptiles.tm2source links: - postgres:db ports: - "3000:3000" export-vectortiles: - image: "osm2vectortiles/export-vectortiles" + image: "openmaptiles/export-vectortiles" volumes: - ./data:/export - - ./:/tm2source + - ./build/openmaptiles.tm2source:/tm2source links: - postgres:postgres env_file: .env diff --git a/layers/place/place.yaml b/layers/place/place.yaml index 0c74cc87..539d3ea7 100644 --- a/layers/place/place.yaml +++ b/layers/place/place.yaml @@ -1,6 +1,7 @@ layer: id: "place" - description: [OSM Places](http://wiki.openstreetmap.org/wiki/Key:place) + description: | + [OSM Places](http://wiki.openstreetmap.org/wiki/Key:place) buffer_size: 128 datasource: query: (SELECT * FROM layer_place(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t diff --git a/omtgen.py b/omtgen.py deleted file mode 100755 index 8cb2dc62..00000000 --- a/omtgen.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python -""" -Usage: - omtgen tm2source [--host=] [--port=] [--database=] [--user=] [--password=] - omtgen imposm3 - omtgen dump-sql - omtgen --help - omtgen --version -Options: - --help Show this screen. - --version Show version. - --host= PostGIS host. - --port= PostGIS port. - --database= PostGIS database name. - --user= PostGIS user. - --password= PostGIS password. -""" -import os -import sys -import yaml -import collections -from docopt import docopt - -DbParams = collections.namedtuple('DbParams', ['dbname', 'host', 'port', - 'password', 'user']) - - -def generate_layer(layer_def, layer_defaults, db_params): - layer = layer_def['layer'] - datasource = layer['datasource'] - tm2layer = { - 'id': layer['id'], - 'srs': layer.get('srs', layer_defaults['srs']), - 'properties': { - 'buffer-size': layer['buffer_size'] - }, - 'Datasource': { - 'extent': [-20037508.34, -20037508.34, 20037508.34, 20037508.34], - 'geometry_field': datasource.get('geometry_field', 'geom'), - 'key_field': datasource.get('key_field', ''), - 'key_field_as_attribute': datasource.get('key_field_as_attribute', ''), - 'max_size': datasource.get('max_size', 512), - 'port': db_params.port, - 'srid': datasource.get('srid', layer_defaults['datasource']['srid']), - 'table': datasource['query'], - 'type': 'postgis', - 'host': db_params.host, - 'dbname': db_params.dbname, - 'user': db_params.user, - 'password': db_params.password, - } - } - return tm2layer - - -def parse_tileset(filename): - return parse_file(filename)['tileset'] - - -def collect_sql(tileset_filename): - tileset = parse_tileset(tileset_filename) - sql = '' - for layer_filename in tileset['layers']: - with open(layer_filename, 'r') as stream: - try: - layer_def = yaml.load(stream) - for sql_filename in layer_def['schema']: - with open(os.path.join(os.path.dirname(layer_filename), sql_filename), 'r') as stream: - sql += stream.read() - except yaml.YAMLError as e: - print('Could not parse ' + layer_filename) - print(e) - sys.exit(403) - return sql - - -def parse_file(filename): - with open(filename, 'r') as stream: - try: - return yaml.load(stream) - except yaml.YAMLError as e: - print('Could not parse ' + filename) - print(e) - sys.exit(403) - - -def create_imposm3_mapping(tileset_filename): - tileset = parse_tileset(tileset_filename) - generalized_tables = {} - tables = {} - for layer_filename in tileset['layers']: - layer_def = parse_file(layer_filename) - for data_source in layer_def.get('datasources', []): - if data_source['type'] != 'imposm3': - continue - mapping_path = os.path.join(os.path.dirname(layer_filename), - data_source['mapping_file']) - mapping = parse_file(mapping_path) - for table_name, definition in mapping.get('generalized_tables', {}).items(): - generalized_tables[table_name] = definition - for table_name, definition in mapping.get('tables', {}).items(): - tables[table_name] = definition - return { - 'generalized_tables': generalized_tables, - 'tables': tables, - } - - -def generate_tm2source(tileset_filename, db_params): - tileset = parse_tileset(tileset_filename) - layer_defaults = tileset['defaults'] - tm2 = { - 'attribution': tileset['attribution'], - 'center': tileset['center'], - 'description': tileset['description'], - 'maxzoom': tileset['maxzoom'], - 'minzoom': tileset['minzoom'], - 'name': tileset['name'], - 'Layer': [], - } - - for layer_filename in tileset['layers']: - with open(layer_filename, 'r') as stream: - try: - layer_def = yaml.load(stream) - tm2layer = generate_layer(layer_def, layer_defaults, db_params) - tm2['Layer'].append(tm2layer) - except yaml.YAMLError as e: - print('Could not parse ' + layer_filename) - print(e) - sys.exit(403) - - return tm2 - -if __name__ == '__main__': - args = docopt(__doc__, version=0.1) - if args.get('tm2source'): - db_params = DbParams( - dbname=args['--database'], - port=int(args['--port']), - user=args['--user'], - password=args['--password'], - host=args['--host'] - ) - tm2 = generate_tm2source(args[''], db_params) - print(yaml.dump(tm2)) - if args.get('dump-sql'): - sql = collect_sql(args['']) - print(sql) - if args.get('imposm3'): - mapping = create_imposm3_mapping(args['']) - print(yaml.dump(mapping)) diff --git a/osm2vectortiles.yaml b/openmaptiles.yaml similarity index 86% rename from osm2vectortiles.yaml rename to openmaptiles.yaml index 51c96685..206181ae 100644 --- a/osm2vectortiles.yaml +++ b/openmaptiles.yaml @@ -11,8 +11,8 @@ tileset: - layers/urban/urban.yaml - layers/water/water.yaml - layers/landcover/landcover.yaml - name: OSM2VectorTiles v3.0 - description: "Free global vector tiles from OpenStreetMap. http://osm2vectortiles.org" + name: OpenMapTiles v1.0 + description: "A tileset showcasing all layers in OpenMapTiles. http://openmaptiles.org" attribution: "© OpenStreetMap contributors" center: [-12.2168, 28.6135, 4] maxzoom: 14 diff --git a/schema/Dockerfile b/schema/Dockerfile deleted file mode 100644 index 83d95209..00000000 --- a/schema/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM osm2vectortiles/postgis:latest -MAINTAINER Lukas Martinelli - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - wget \ - && rm -rf /var/lib/apt/lists/* - -ENV VT_UTIL_DIR=/opt/postgis-vt-util \ - VT_UTIL_URL="https://raw.githubusercontent.com/mapbox/postgis-vt-util/v1.0.0/postgis-vt-util.sql" - -RUN mkdir -p "$VT_UTIL_DIR" \ - && wget -P "$VT_UTIL_DIR" --quiet "$VT_UTIL_URL" -ENV GENERATED_SQL_FILE=/usr/src/app/gen.sql -COPY . /usr/src/app/ -WORKDIR /usr/src/app -CMD ["./create_schema.sh"] diff --git a/schema/create_schema.sh b/schema/create_schema.sh deleted file mode 100755 index 76a66dc8..00000000 --- a/schema/create_schema.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -set -o nounset - -function exec_psql_file() { - local file_name="$1" - PGPASSWORD="$POSTGRES_PASSWORD" psql \ - -v ON_ERROR_STOP="1" \ - --host="$POSTGRES_HOST" \ - --port="$POSTGRES_PORT" \ - --dbname="$POSTGRES_DB" \ - --username="$POSTGRES_USER" \ - -f "$file_name" -} - -function main() { - exec_psql_file "$VT_UTIL_DIR/postgis-vt-util.sql" - exec_psql_file "$GENERATED_SQL_FILE" -} - -main