Merge pull request #84 from osm2vectortiles/feature/better-serve

Serve supports multiple style projects
2015-12-11 10:26:23 +01:00 · 2015-12-11 10:26:23 +01:00 · 389b677500
commit 389b677500
--- a/.travis.yml
+++ b/.travis.yml
@ -9,6 +9,13 @@ before_install:
  - wget -nc -P "$HOME/cache" "https://s3.amazonaws.com/metro-extracts.mapzen.com/zurich_switzerland.osm.pbf"
  - mkdir -p "$TRAVIS_BUILD_DIR/import" && cp "$HOME/cache/zurich_switzerland.osm.pbf" "$TRAVIS_BUILD_DIR/import"
  - docker-compose build
+  - docker build -t osm2vectortiles/postgis src/postgis
+  - docker build -t osm2vectortiles/export src/export
+  - docker build -t osm2vectortiles/import-osm src/import-osm
+  - docker build -t osm2vectortiles/import-external src/import-external
+  - docker build -t osm2vectortiles/import-sql src/import-sql
+  - docker build -t osm2vectortiles/update-scaleranks src/update-scaleranks
+  - docker build -t osm2vectortiles/serve src/serve
 script:
  - docker-compose up -d postgis
  - sleep 5
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -7,19 +7,19 @@ cache:
  volumes:
  - /data/cache
 postgis:
-  build: ./src/postgis
+  image: "osm2vectortiles/postgis"
  volumes_from:
   - pgdata
  ports:
   - "5432"
 pgbouncer:
-  build: ./src/pgbouncer
+  image: "osm2vectortiles/pgbouncer"
  links:
   - postgis:pg
  ports:
   - "6432"
 import-osm:
-  build: ./src/import-osm
+  image: "osm2vectortiles/import-osm"
  volumes:
   - ./import:/data/import
  volumes_from:
@ -27,26 +27,22 @@ import-osm:
  links:
   - postgis:db
 import-labels:
-  build: ./src/import-external
+  image: "osm2vectortiles/import-external"
  command: ./import-labels.sh
  links:
  - postgis:db
 import-natural-earth:
-  build: ./src/import-external
+  image: "osm2vectortiles/import-external"
  command: ./import-natural-earth.sh
  links:
  - postgis:db
 import-water:
-  build: ./src/import-external
+  image: "osm2vectortiles/import-external"
  command: ./import-water.sh
  links:
  - postgis:db
-import-external:
-  build: ./src/import-external
-  links:
-  - postgis:db
 update-scaleranks:
-  build: ./src/update-scaleranks
+  image: "osm2vectortiles/update-scaleranks"
  links:
  - postgis:db
 enqueue-jobs:
@ -56,7 +52,7 @@ enqueue-jobs:
    TASK_ZOOM_LEVEL: 8
    QUEUE_NAME: "osm2vectortiles_jobs"
 export-remote:
-  build: ./src/export
+  image: "osm2vectortiles/export-remote"
  command: ./export-remote.sh
  volumes:
   - ./open-streets.tm2source:/data/tm2source
@ -67,7 +63,7 @@ export-remote:
    AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
    AWS_REGION: "eu-central-1"
 export-local:
-  build: ./src/export
+  image: "osm2vectortiles/export"
  command: ./export-local.sh
  volumes:
   - ./export:/data/export
@ -79,17 +75,17 @@ export-local:
    MIN_ZOOM: "8"
    MAX_ZOOM: "14"
 serve:
-  build: ./src/serve
+  image: "osm2vectortiles/serve"
  volumes:
   - ./export:/data
  ports:
   - "8080:80"
 import-sql:
-  build: ./src/import-sql
+  image: "osm2vectortiles/import-sql"
  links:
   - postgis:db
 mapbox-studio:
-  build: ./tools/mapbox-studio
+  image: "osm2vectortiles/mapbox-studio"
  volumes:
   - ./open-streets.tm2source:/projects/open-streets.tm2source
  links:
@ -97,7 +93,7 @@ mapbox-studio:
  ports:
   - "3000:3000"
 compare-visual:
-  build: ./tools/compare-visual
+  image: "osm2vectortiles/compare-visual"
  ports:
   - "4000:3000"
   - "4001:3001"
--- a/src/serve/Dockerfile
+++ b/src/serve/Dockerfile
@ -17,29 +17,26 @@ RUN wget --quiet -P /fonts https://github.com/aaronlidman/Toner-for-Tilemill/raw
    wget --quiet -P /fonts https://github.com/aaronlidman/Toner-for-Tilemill/raw/master/toner4tilemill/fonts/Arial-Unicode-Italic.ttf && \
    wget --quiet -P /fonts https://github.com/aaronlidman/Toner-for-Tilemill/raw/master/toner4tilemill/fonts/Arial-Unicode-Regular.ttf

-ENV MAPNIK_FONT_PATH=/fonts
-
 # only install minimal amount of tessera packages
 # be careful as some tessera packages collide with itself
-RUN npm install -g tessera
-RUN npm install -g mbtiles \
-           tilelive-tmstyle \
-           tilelive-xray \
-           tilelive-http \
-           git+https://git@github.com/mojodna/node-tilejson.git\#always-xyz
+RUN mkdir -p /usr/src/app && mkdir -p /project
+WORKDIR /usr/src/app

-RUN mkdir -p /usr/src/app
-COPY run.sh /usr/src/app/
+RUN npm install -g \
+   git+https://github.com/klokantech/tileserver-tessera.git \
+   tessera@0.5.3  \
+   mbtiles@0.8.2  \
+   tilelive-tmstyle@0.4.2 \
+   tilelive-xray@0.2.0  \
+   tilelive-http@0.8.0

 # mountpoint of custom mbtiles files and tm2 projects
 VOLUME /data
-ENV SOURCE_DATA_DIR=/data
-
-# destination of modified tm2 projects
-RUN mkdir -p /project
-ENV DEST_DATA_DIR=/project
+ENV SOURCE_DATA_DIR=/data \
+    DEST_DATA_DIR=/project \
+    PORT=80 \
+    MAPNIK_FONT_PATH=/fonts

+COPY run.sh /usr/src/app/
 EXPOSE 80
-ENV PORT=80
-
 CMD ["/usr/src/app/run.sh"]
--- a/src/serve/run.sh
+++ b/src/serve/run.sh
@ -5,6 +5,7 @@ set -o nounset

 readonly SOURCE_DATA_DIR=${SOURCE_DATA_DIR:-/data}
 readonly DEST_DATA_DIR=${DEST_DATA_DIR:-/project}
+readonly TESSERA_CONFIG="$DEST_DATA_DIR/config.json"
 readonly PORT=${PORT:-80}
 readonly CACHE_SIZE=${CACHE_SIZE:-10}
 readonly SOURCE_CACHE_SIZE=${SOURCE_CACHE_SIZE:-10}
@ -17,70 +18,98 @@ function serve_xray() {
        --source-cache-size $SOURCE_CACHE_SIZE
 }

-# find first tm2 project
-if [ "$(ls -A $SOURCE_DATA_DIR)" ]; then
-    if [ -d "$SOURCE_DATA_DIR"/*.tm2 ]; then
-        for _PROJECT_DIR in "$SOURCE_DATA_DIR"/*.tm2; do
-            [ -d "${_PROJECT_DIR}" ] && PROJECT_DIR="${_PROJECT_DIR}" && break
-        done
-    else
-        MBTILES_FILE=""
-        for _MBTILES_FILE in "$SOURCE_DATA_DIR"/*.mbtiles; do
-            MBTILES_FILE="${_MBTILES_FILE}"
-            break
-        done
-
-        if [ -f "$MBTILES_FILE" ]; then
-            echo "The mbtiles file is now served with X-Ray styles"
-            serve_xray "$MBTILES_FILE"
-        else
-            echo "No tm2 projects found. Please add a tm2 project to your mounted folder."
-            exit 404
-        fi
-    fi
-else
-    echo "No tm2 projects found. Please mount the $SOURCE_DATA_DIR volume to a folder containing tm2 projects."
-    exit 404
-fi
-
-# find all tm2 projects but only the first project will be hosted for now
-PROJECT_NAME="${PROJECT_DIR##*/}"
-PROJECT_CONFIG_FILE="${PROJECT_DIR%%/}/project.yml"
-VECTORTILES_NAME="${PROJECT_NAME%.tm2}.mbtiles"
-MBTILES_FILE="${SOURCE_DATA_DIR%%/}/$VECTORTILES_NAME"
-
-echo "Found project ${PROJECT_NAME}"
-
-# project config will be copied to new folder because we
-# modify the source configuration of the style and don't want
-# that to effect the original file
-DEST_PROJECT_DIR="${DEST_DATA_DIR%%/}/$PROJECT_NAME"
-DEST_PROJECT_CONFIG_FILE="${DEST_PROJECT_DIR%%/}/project.yml"
-cp -rf "$PROJECT_DIR" "$DEST_PROJECT_DIR"
-
-# project.yml is single source of truth, therefore the mapnik
-# stylesheet is not necessary
-rm -f "${DEST_PROJECT_DIR%%/}/project.xml"
-
-# replace external vector tile sources with mbtiles source
-# this allows developing rapidyl with an external source and then use the
-# mbtiles for dependency free deployment
-if [ -f "$MBTILES_FILE" ]; then
-    echo "Associating $VECTORTILES_NAME with $PROJECT_NAME"
-    replace_expr="s|source: \".*\"|source: \"mbtiles://$MBTILES_FILE\"|g"
-    sed -e "$replace_expr" $PROJECT_CONFIG_FILE > $DEST_PROJECT_CONFIG_FILE
-else
-    echo "No mbtiles matching project $PROJECT_NAME found."
-    echo "Please name the mbtiles file the same as your style project."
-    exit 500
-fi
-
-function serve_style() {
-    local style_dir=$1
-    exec tessera "tmstyle://$style_dir" \
-        --PORT $PORT \
-        --cache-size $CACHE_SIZE \
-        --source-cache-size $SOURCE_CACHE_SIZE
+function find_first_mbtiles() {
+    for mbtiles_file in "$SOURCE_DATA_DIR"/*.mbtiles; do
+        echo "${mbtiles_file}"
+        break
+    done
 }

-serve_style "$DEST_PROJECT_DIR"
+function find_first_tm2() {
+    for tm2project in "$SOURCE_DATA_DIR"/*.tm2/; do
+        echo "${tm2project}"
+        break
+    done
+}
+
+function tessera_config_entry() {
+    local tm2project=$1
+    local serve_dir=${tm2project%.tm2}
+    local serve_path=${serve_dir##*/}
+    echo "\"/${serve_path}\": \"tmstyle://${tm2project}\"," >> "$TESSERA_CONFIG"
+    echo "Serving ${tm2project##*/} at $serve_path"
+}
+
+function create_tessera_config() {
+    local mbtiles_file=$1
+    rm -f "$TESSERA_CONFIG"
+    echo '{' >> "$TESSERA_CONFIG"
+    for tm2project in "$DEST_DATA_DIR"/*.tm2; do
+        tessera_config_entry "${tm2project}"
+    done
+
+    # Always serve the vector tile source
+    echo "\"/\": \"mbtiles://${mbtiles_file}\"" >> "$TESSERA_CONFIG"
+    echo '}' >> "$TESSERA_CONFIG"
+}
+
+function replace_sources() {
+    mbtiles_file=$1
+    local vectortiles_name=${mbtiles_file%.mbtiles}
+
+    for project_dir in "$SOURCE_DATA_DIR"/*.tm2; do
+        local project_name="${project_dir##*/}"
+        local project_config_file="${project_dir%%/}/project.yml"
+
+        # project config will be copied to new folder because we
+        # modify the source configuration of the style and don't want
+        # that to effect the original file
+        dest_project_dir="${DEST_DATA_DIR%%/}/$project_name"
+        local dest_project_config_file="${dest_project_dir%%/}/project.yml"
+        cp -rf "$project_dir" "$dest_project_dir"
+
+        # replace external vector tile sources with mbtiles source
+        # this allows developing rapidyl with an external source and then use the
+        # mbtiles for dependency free deployment
+        echo "Associating $vectortiles_name with $project_name"
+        replace_expr="s|source: \".*\"|source: \"mbtiles://$mbtiles_file\"|g"
+        sed -e "$replace_expr" $project_config_file > $dest_project_config_file
+    done
+}
+
+function serve_config() {
+    exec tessera -c "$TESSERA_CONFIG" \
+        --PORT "$PORT" \
+        --cache-size "$CACHE_SIZE" \
+        --source-cache-size "$SOURCE_CACHE_SIZE"
+}
+
+function serve() {
+    local mbtiles_file=$(find_first_mbtiles)
+    local tm2project=$(find_first_tm2)
+    if [ -f "$mbtiles_file" ]; then
+        echo "Using $mbtiles_file as vector tile source"
+        if [ -d "$tm2project" ]; then
+            replace_sources "$mbtiles_file"
+            create_tessera_config "$mbtiles_file"
+            serve_config
+        else
+            echo "The mbtiles file is now served with X-Ray styles"
+            serve_xray "$mbtiles_file"
+        fi
+    else
+        echo "No MBTiles file found. Please add a .mbtiles file to your mounted folder."
+        exit 404
+    fi
+}
+
+function main() {
+    if [ "$(ls -A $SOURCE_DATA_DIR)" ]; then
+        serve
+    else
+        echo "No files found. Please mount the $SOURCE_DATA_DIR volume to a folder containing tm2 projects and mbtiles vector tile source."
+        exit 403
+    fi
+}
+
+main