Switch to Makefile and openmaptiles-tools

2016-10-23 17:36:48 +02:00 · 2016-10-23 17:36:48 +02:00 · 250bcd01a4
commit 250bcd01a4
--- a/.env
+++ b/.env
@ -1,5 +1,5 @@
-POSTGRES_DB=osm
-POSTGRES_USER=osm
-POSTGRES_PASSWORD=osm
+POSTGRES_DB=openmaptiles
+POSTGRES_USER=openmaptiles
+POSTGRES_PASSWORD=openmaptiles
 POSTGRES_HOST=postgres
 POSTGRES_PORT=5432
--- a/13
+++ b/13
@ -0,0 +1,13 @@
+all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
+
+build/openmaptiles.tm2source/data.yml:
+	mkdir -p build/openmaptiles.tm2source && generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml
+
+build/mapping.yaml:
+	mkdir -p build && generate-imposm3 openmaptiles.yaml > build/mapping.yaml
+
+build/tileset.sql:
+	mkdir -p build && generate-sql openmaptiles.yaml > build/tileset.sql
+
+clean:
+	rm -rf build
--- a/README.md
+++ b/README.md
@ -41,10 +41,19 @@ To work on the database and vector tile schema read the *Develop* section.

 ## Develop

-To work on *osm2vectortiles.tm2source* we recommend using Docker.
+To work on *osm2vectortiles.tm2source* you need Docker and Python.

 - Install [Docker](https://docs.docker.com/engine/installation/)
 - Install [Docker Compose](https://docs.docker.com/compose/install/)
+- Install [OpenMapTiles tools](https://github.com/openmaptiles/openmaptiles-tools) with `pip install openmaptiles-tools`
+
+### Build
+
+Build the tileset.
+
+```
+make
+```

 ### Prepare the Database

@ -69,23 +78,20 @@ docker-compose run import-natural-earth
 Import [OpenStreetMap](http://wiki.openstreetmap.org/wiki/Osm2pgsql) data based on the [ClearTables osm2pgsql style](https://github.com/ClearTables/ClearTables).
 In order to do this you first need to clone the latest ClearTables.

- [ ] *Should we use a submodule for ClearTables? On the other hand it is quite convenient to able to work directly on ClearTables while working on the style.*
-
 ```bash
-git clone https://github.com/ClearTables/ClearTables.git
 docker-compose run import-osm
 ```

-### Work on Vector Tile Schema
+### Work on Layers

-Build and run the `db-schema` container each time you modify SQL code inside `./schema`.
+Each time you modify layer SQL code run `make` and `docker-compose run import-sql`.

-```bash
-docker-compose build db-schema
-docker-compose run db-schema
+```
+make
+docker-compose run import-sql
 ```

-To work on the *data.yml* you can start up Mapbox Studio Classic in a container
+To look at the vector tiles you can start up Mapbox Studio Classic in a container
 and visit `localhost:3000` and open the vector source project under `projects/osm2vectortiles.tm2source`.

 ```bash
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,50 +1,52 @@
-version: '2'
+version: "2"
 volumes:
  pgdata:
 services:
  postgres:
-    image: "osm2vectortiles/postgis:latest"
+    image: "openmaptiles/postgis:latest"
    volumes:
     - pgdata:/var/lib/postgresql/data
    ports:
     - "5432"
    env_file: .env
-  db-schema:
-    build: "./schema"
+  import-natural-earth:
+    image: "openmaptiles/import-natural-earth"
    env_file: .env
    links:
-    - postgres:postgres
+    - postgres
+  import-water:
+    image: "openmaptiles/import-water"
+    env_file: .env
+    links:
+    - postgres
  import-osm:
-    image: "osm2vectortiles/import-osm"
+    image: "openmaptiles/import-osm"
    env_file: .env
    links:
    - postgres
    volumes:
     - ./data:/import
-     - ./:/mapping
-  import-natural-earth:
-    image: "osm2vectortiles/import-natural-earth"
-    env_file: .env
-    links:
-    - postgres
-  import-water:
-    image: "osm2vectortiles/import-water"
+     - ./build:/mapping
+  import-sql:
+    image: "openmaptiles/import-sql"
    env_file: .env
    links:
    - postgres
+    volumes:
+     - ./build:/sql
  mapbox-studio:
    image: "osm2vectortiles/mapbox-studio"
    volumes:
-     - ./:/projects/osm2vectortiles.tm2source
+     - ./build/openmaptiles.tm2source:/projects/openmaptiles.tm2source
    links:
     - postgres:db
    ports:
     - "3000:3000"
  export-vectortiles:
-    image: "osm2vectortiles/export-vectortiles"
+    image: "openmaptiles/export-vectortiles"
    volumes:
    - ./data:/export
-    - ./:/tm2source
+    - ./build/openmaptiles.tm2source:/tm2source
    links:
    - postgres:postgres
    env_file: .env
--- a/layers/place/place.yaml
+++ b/layers/place/place.yaml
@ -1,6 +1,7 @@
 layer:
  id: "place"
-  description: [OSM Places](http://wiki.openstreetmap.org/wiki/Key:place)
+  description: |
+      [OSM Places](http://wiki.openstreetmap.org/wiki/Key:place)
  buffer_size: 128
  datasource:
    query: (SELECT * FROM layer_place(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
--- a/omtgen.py
+++ b/omtgen.py
@ -1,152 +0,0 @@
-#!/usr/bin/env python
-"""
-Usage:
-  omtgen tm2source <tileset> [--host=<host>] [--port=<port>] [--database=<dbname>] [--user=<user>] [--password=<pw>]
-  omtgen imposm3 <tileset>
-  omtgen dump-sql <tileset>
-  omtgen --help
-  omtgen --version
-Options:
-  --help              Show this screen.
-  --version           Show version.
-  --host=<host>       PostGIS host.
-  --port=<port>       PostGIS port.
-  --database=<dbname> PostGIS database name.
-  --user=<user>       PostGIS user.
-  --password=<pw>     PostGIS password.
-"""
-import os
-import sys
-import yaml
-import collections
-from docopt import docopt
-
-DbParams = collections.namedtuple('DbParams', ['dbname', 'host', 'port',
-                                               'password', 'user'])
-
-
-def generate_layer(layer_def, layer_defaults, db_params):
-    layer = layer_def['layer']
-    datasource = layer['datasource']
-    tm2layer = {
-        'id': layer['id'],
-        'srs': layer.get('srs', layer_defaults['srs']),
-        'properties': {
-            'buffer-size': layer['buffer_size']
-        },
-        'Datasource': {
-          'extent': [-20037508.34, -20037508.34, 20037508.34, 20037508.34],
-          'geometry_field': datasource.get('geometry_field', 'geom'),
-          'key_field': datasource.get('key_field', ''),
-          'key_field_as_attribute': datasource.get('key_field_as_attribute', ''),
-          'max_size': datasource.get('max_size', 512),
-          'port': db_params.port,
-          'srid': datasource.get('srid', layer_defaults['datasource']['srid']),
-          'table': datasource['query'],
-          'type': 'postgis',
-          'host': db_params.host,
-          'dbname': db_params.dbname,
-          'user': db_params.user,
-          'password': db_params.password,
-        }
-    }
-    return tm2layer
-
-
-def parse_tileset(filename):
-    return parse_file(filename)['tileset']
-
-
-def collect_sql(tileset_filename):
-    tileset = parse_tileset(tileset_filename)
-    sql = ''
-    for layer_filename in tileset['layers']:
-        with open(layer_filename, 'r') as stream:
-            try:
-                layer_def = yaml.load(stream)
-                for sql_filename in layer_def['schema']:
-                    with open(os.path.join(os.path.dirname(layer_filename), sql_filename), 'r') as stream:
-                        sql += stream.read()
-            except yaml.YAMLError as e:
-                print('Could not parse ' + layer_filename)
-                print(e)
-                sys.exit(403)
-    return sql
-
-
-def parse_file(filename):
-    with open(filename, 'r') as stream:
-        try:
-            return yaml.load(stream)
-        except yaml.YAMLError as e:
-            print('Could not parse ' + filename)
-            print(e)
-            sys.exit(403)
-
-
-def create_imposm3_mapping(tileset_filename):
-    tileset = parse_tileset(tileset_filename)
-    generalized_tables = {}
-    tables = {}
-    for layer_filename in tileset['layers']:
-        layer_def = parse_file(layer_filename)
-        for data_source in layer_def.get('datasources', []):
-            if data_source['type'] != 'imposm3':
-                continue
-            mapping_path = os.path.join(os.path.dirname(layer_filename),
-                                        data_source['mapping_file'])
-            mapping = parse_file(mapping_path)
-            for table_name, definition in mapping.get('generalized_tables', {}).items():
-                generalized_tables[table_name] = definition
-            for table_name, definition in mapping.get('tables', {}).items():
-                tables[table_name] = definition
-    return {
-        'generalized_tables': generalized_tables,
-        'tables': tables,
-    }
-
-
-def generate_tm2source(tileset_filename, db_params):
-    tileset = parse_tileset(tileset_filename)
-    layer_defaults = tileset['defaults']
-    tm2 = {
-        'attribution': tileset['attribution'],
-        'center': tileset['center'],
-        'description': tileset['description'],
-        'maxzoom': tileset['maxzoom'],
-        'minzoom': tileset['minzoom'],
-        'name': tileset['name'],
-        'Layer': [],
-    }
-
-    for layer_filename in tileset['layers']:
-        with open(layer_filename, 'r') as stream:
-            try:
-                layer_def = yaml.load(stream)
-                tm2layer = generate_layer(layer_def, layer_defaults, db_params)
-                tm2['Layer'].append(tm2layer)
-            except yaml.YAMLError as e:
-                print('Could not parse ' + layer_filename)
-                print(e)
-                sys.exit(403)
-
-    return tm2
-
-if __name__ == '__main__':
-    args = docopt(__doc__, version=0.1)
-    if args.get('tm2source'):
-        db_params = DbParams(
-            dbname=args['--database'],
-            port=int(args['--port']),
-            user=args['--user'],
-            password=args['--password'],
-            host=args['--host']
-        )
-        tm2 = generate_tm2source(args['<tileset>'], db_params)
-        print(yaml.dump(tm2))
-    if args.get('dump-sql'):
-        sql = collect_sql(args['<tileset>'])
-        print(sql)
-    if args.get('imposm3'):
-        mapping = create_imposm3_mapping(args['<tileset>'])
-        print(yaml.dump(mapping))
--- a/osm2vectortiles.yaml
+++ b/osm2vectortiles.yaml
@ -11,8 +11,8 @@ tileset:
    - layers/urban/urban.yaml
    - layers/water/water.yaml
    - layers/landcover/landcover.yaml
-  name: OSM2VectorTiles v3.0
-  description: "Free global vector tiles from OpenStreetMap. http://osm2vectortiles.org"
+  name: OpenMapTiles v1.0
+  description: "A tileset showcasing all layers in OpenMapTiles. http://openmaptiles.org"
  attribution: "<a href=\"http://www.openstreetmap.org/about/\" target=\"_blank\">&copy; OpenStreetMap contributors</a>"
  center: [-12.2168, 28.6135, 4]
  maxzoom: 14
--- a/schema/Dockerfile
+++ b/schema/Dockerfile
@ -1,17 +0,0 @@
-FROM osm2vectortiles/postgis:latest
-MAINTAINER Lukas Martinelli <me@lukasmartinelli.ch>
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-      ca-certificates \
-      wget \
- && rm -rf /var/lib/apt/lists/*
-
-ENV VT_UTIL_DIR=/opt/postgis-vt-util \
-    VT_UTIL_URL="https://raw.githubusercontent.com/mapbox/postgis-vt-util/v1.0.0/postgis-vt-util.sql"
-
-RUN mkdir -p "$VT_UTIL_DIR" \
- && wget -P "$VT_UTIL_DIR" --quiet "$VT_UTIL_URL"
-ENV GENERATED_SQL_FILE=/usr/src/app/gen.sql
-COPY . /usr/src/app/
-WORKDIR /usr/src/app
-CMD ["./create_schema.sh"]
--- a/schema/create_schema.sh
+++ b/schema/create_schema.sh
@ -1,22 +0,0 @@
-#!/bin/bash
-set -o errexit
-set -o pipefail
-set -o nounset
-
-function exec_psql_file() {
-    local file_name="$1"
-    PGPASSWORD="$POSTGRES_PASSWORD" psql \
-        -v ON_ERROR_STOP="1" \
-        --host="$POSTGRES_HOST" \
-        --port="$POSTGRES_PORT" \
-        --dbname="$POSTGRES_DB" \
-        --username="$POSTGRES_USER" \
-        -f "$file_name"
-}
-
-function main() {
-    exec_psql_file "$VT_UTIL_DIR/postgis-vt-util.sql"
-    exec_psql_file "$GENERATED_SQL_FILE"
-}
-
-main