Pre-release update. (#1657)

Update documentation before release v3.15.

.env

@@ -1,13 +1,46 @@
-POSTGRES_DB=openmaptiles
-POSTGRES_USER=openmaptiles
-POSTGRES_PASSWORD=openmaptiles
-POSTGRES_HOST=postgres
-POSTGRES_PORT=5432
-QUICKSTART_MIN_ZOOM=0
-QUICKSTART_MAX_ZOOM=7
-DIFF_MODE=false
-TOOLS_VERSION=3.1.0
+# This file defines default environment variables for all images
 
+# Layers definition and meta data
+TILESET_FILE=openmaptiles.yaml
+
+# Use 3-part patch version to ignore patch updates, e.g. 7.0.0
+TOOLS_VERSION=7.1
+
+# Make sure these values are in sync with the ones in .env-postgres file
+PGDATABASE=openmaptiles
+PGUSER=openmaptiles
+PGPASSWORD=openmaptiles
+PGHOST=postgres
+PGPORT=5432
+
+# BBOX may get overwritten by the computed bbox of the specific area:
+#   make generate-bbox-file
+# By default, the Makefile will use the content of data/$(area).bbox file if it exists.
 BBOX=-180.0,-85.0511,180.0,85.0511
+
+# Which zooms to generate with   make generate-tiles-pg
 MIN_ZOOM=0
-MAX_ZOOM=14
+MAX_ZOOM=7
+
+# `MID_ZOOM` setting only works with `make generate-tiles-pg` command.  Make sure MID_ZOOM < MAX_ZOOM.
+# See https://github.com/openmaptiles/openmaptiles-tools/pull/383
+# MID_ZOOM=11
+
+# Use  true  (case sensitive) to allow data updates
+DIFF_MODE=false
+
+# The current setup assumes this file is placed inside the data/ dir
+MBTILES_FILE=tiles.mbtiles
+
+# This is the current repl_config.json location, pre-configured in the tools Dockerfile
+# Makefile and quickstart replace it with the dynamically generated one, but we keep it here in case some other method is used to run.
+IMPOSM_CONFIG_FILE=/usr/src/app/config/repl_config.json
+
+# Number of parallel processes to use when importing sql files
+MAX_PARALLEL_PSQL=5
+
+# Number of parallel threads to use when generating vector map tiles
+COPY_CONCURRENCY=10
+
+# Variables for generate tiles using tilelive-pgquery
+PGHOSTS_LIST=

.github/workflows/build-test-data.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+#
+# Download several areas, combine them into a single file, and print --bbox params needed to run test-perf
+#
+
+# List of Geofabrik areas
+TEST_AREAS=(equatorial-guinea liechtenstein district-of-columbia greater-london)
+
+: "${DATA_DIR:=/tileset/data/test}"
+: "${DATA_FILE_SUFFIX:=-latest.osm.pbf}"
+: "${RESULT_FILE:=test${DATA_FILE_SUFFIX}}"
+
+mkdir -p "$DATA_DIR"
+cd "$DATA_DIR"
+
+
+echo -e $"\n=========== downloading areas" "${TEST_AREAS[@]}" "==========================="
+for area in "${TEST_AREAS[@]}"; do
+  file="${area}${DATA_FILE_SUFFIX}"
+  if [ -f "$file" ]; then
+    echo "File $file already exists, skipping download"
+  else
+    download-osm geofabrik "${area}" -- -d "$DATA_DIR"
+    if [ ! -f "$file" ]; then
+      echo "Unexpected error while downloading $file, aborting"
+      exit 1
+    fi
+  fi
+done
+
+
+echo -e $"\n=========== Merging" "${TEST_AREAS[@]}" "into ${RESULT_FILE} ====="
+rm -f "${RESULT_FILE}"
+OSMOSIS_ARG="--read-pbf ${TEST_AREAS[0]}${DATA_FILE_SUFFIX} $(printf " --read-pbf %s${DATA_FILE_SUFFIX} --merge" "${TEST_AREAS[@]:1}")"
+# shellcheck disable=SC2086
+( set -x; osmosis ${OSMOSIS_ARG} --write-pbf "${RESULT_FILE}" )
+
+
+echo -e $"\n=========== Computing test BBOXes ======================="
+echo -e $"\n  File ${RESULT_FILE} ($(du -b "$RESULT_FILE" | cut -f1)) has been generated with these test areas:\n"
+for area in "${TEST_AREAS[@]}"; do
+  file="${area}${DATA_FILE_SUFFIX}"
+  STATS=$(osmconvert --out-statistics "$file" )
+  LON_MIN=$( echo "$STATS" | grep "lon min:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  LON_MAX=$( echo "$STATS" | grep "lon max:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  LAT_MIN=$( echo "$STATS" | grep "lat min:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  LAT_MAX=$( echo "$STATS" | grep "lat max:" | cut -d":" -f 2 | awk '{gsub(/^ +| +$/,"")} {print $0}' )
+  BBOX="${LON_MIN},${LAT_MIN},${LON_MAX},${LAT_MAX}"
+  FILE_SIZE="$(du -b "$file" | cut -f1)"
+
+  cat <<EOF | (PYTHONPATH=/usr/src/app python)
+from openmaptiles.perfutils import TestCase
+tc = TestCase('${area}', 'a', bbox='${BBOX}')
+info = f"# {tc.id} {tc.size():,} tiles at z14, \
+{$FILE_SIZE/1024/1024:,.1f}MB, {$FILE_SIZE/tc.size():,.1f} bytes/tile \
+[{tc.start[0]}/{tc.start[1]}]x[{tc.before[0] - 1}/{tc.before[1] - 1}]"
+print(f"  --bbox {tc.bbox:46} {info}")
+EOF
+done
+echo ""

.github/workflows/integrity.yml

@@ -0,0 +1,71 @@
+# Workflow to run basic integrity checks on OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles Integrity CI
+
+on:
+  push:
+    branches: [ master, master-tools ]
+  pull_request:
+
+jobs:
+
+  integrity_test:
+    name: Run integrity test
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout the changes
+        uses: actions/checkout@v4
+
+      - name: Run quickstart for a small area
+        env:
+          area: monaco
+          QUIET: 1
+        run: |
+          echo MIN_ZOOM=0 >> .env
+          echo MAX_ZOOM=14 >> .env
+          ./quickstart.sh $area
+
+      - name: Save quickstart.log
+        uses: actions/upload-artifact@v4
+        with:
+          name: quickstart.log
+          path: quickstart.log
+
+      - name: Test etldoc images
+        run: |
+          export TEST_MODE=yes
+          make generate-devdoc
+
+      - name: Run quickstart and update in DIFF mode
+        env:
+          area: europe/monaco
+          QUIET: 1
+        run: |
+          echo MIN_ZOOM=0 >> .env
+          echo MAX_ZOOM=14 >> .env
+          echo DIFF_MODE=true >> .env
+          # Cleanup
+          rm -fr data build cache
+          # Create data/$area.repl.json
+          make download-geofabrik area=$area
+          # Download 2+ month old data
+          export old_date=$(date --date="$(date +%Y-%m-15) -2 month" +'%y%m01')
+          echo Downloading $old_date extract of $area
+          docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -O data/$area.osm.pbf http://download.geofabrik.de/$area-$old_date.osm.pbf"
+          # Initial import and tile generation
+          ./quickstart.sh $area
+          sleep 2
+          echo Downloading updates
+          # Loop to recover from potential "ERROR 429: Too Many Requests"
+          docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "
+            while ! osmupdate --keep-tempfiles --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz ; do 
+              sleep 2;
+              echo Sleeping...;
+              sleep 630;
+            done"
+          echo Downloading updates completed
+          echo Importing updates
+          make import-diff
+          echo Generating new tiles
+          make generate-tiles-pg

.github/workflows/performance.yml

@@ -0,0 +1,300 @@
+# Workflow to run performance tests OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles Performance CI
+
+on:
+  push:
+    branches: [ master, master-tools ]
+  pull_request:
+
+jobs:
+  performance:
+    name: Evaluate performance
+    runs-on: self-hosted
+    env:
+      ## Smaller tests (runs everything in about 30 minutes)
+      ## Two test areas:  equatorial-guinea  and  liechtenstein
+      #TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=12vw07f9W0MiAHIqMztRiIMwahJfqTi21"
+      #TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --bbox 5.4172943,-1.6732196,12.3733400,4.3475256 --bbox 9.0900979,46.9688169,9.6717077,47.5258072"
+
+      # Large test, size is 79,472,850
+      #  --bbox 5.4172943,-1.6732196,12.3733400,4.3475256       `# equatorial-guinea 87,768 tiles at z14, 2.1MB, 24.9 bytes/tile [8438/7993]x[8755/8268]` \
+      #  --bbox 9.0900979,46.9688169,9.6717077,47.5258072       `# liechtenstein 1,064 tiles at z14, 2.2MB, 2,217.0 bytes/tile [8605/5727]x[8632/5764]` \
+      #  --bbox -78.7749754,38.7820235,-76.8957735,39.6985009   `# district-of-columbia 4,785 tiles at z14, 16.0MB, 3,508.9 bytes/tile [4606/6220]x[4692/6274]` \
+      #  --bbox -0.6124681,51.2268449,0.3996690,51.7873570      `# greater-london 1,974 tiles at z14, 55.5MB, 29,458.3 bytes/tile [8164/5427]x[8210/5468]` \
+      TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=18nP3f06aBBiEKhUNmAkqq30gqQnU2_VJ"
+      TEST_PERF_PARAMS: >-
+        --minzoom 0 --maxzoom 14
+        --bbox 5.4172943,-1.6732196,12.3733400,4.3475256
+        --bbox 9.0900979,46.9688169,9.6717077,47.5258072
+        --bbox -78.7749754,38.7820235,-76.8957735,39.6985009
+        --bbox -0.6124681,51.2268449,0.3996690,51.7873570
+
+      ## Large test data -- we should switch to it after everything is working ok
+      # TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=1kw7XPDPd1Rc-Zi2XxGLTXdinUSq-S4pT"
+      # TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test hungary --test isle-of-man"
+    steps:
+      - name: Cleanup workdir
+        id: cleanup
+        run: |
+          set -euo pipefail
+          pwd
+          ls -al .
+          shopt -s dotglob
+          rm -rf *
+
+      - name: Cache test data download
+        id: cache-testdata
+        uses: actions/cache@v4
+        with:
+          path: ci_cache
+          key: "v2-${{ env.TEST_DATA_URL }}"
+
+      - name: Download test data on cache miss
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        run: |
+          echo "Data file does not exist, downloading $TEST_DATA_URL"
+          mkdir -p ci_cache
+          curl --silent --show-error --location --output ci_cache/perf-test-areas-latest.osm.pbf "$TEST_DATA_URL"
+
+      - name: Get code
+        uses: actions/checkout@v4
+        with:
+          # Fetch the last two commits in case this is a PR,
+          # and we need to profile the base branch first
+          fetch-depth: 2
+          path: code
+
+      - name: Compute git revision hash to cache
+        id: calc
+        run: |
+          # If this is a pull request, we should cache the parent (base) revision
+          # Otherwise cache the current one
+          cd code
+          REV_HASH=$(git log -1 --format="%H")
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            # Take the first parent of the grafted commit (cannot use HEAD^1 with shallow clones)
+            REV_HASH=$(git cat-file -p $REV_HASH | awk 'NR > 1 {if(/^parent/){print $2; exit}}')
+          fi
+          echo "hash=$REV_HASH" >> $GITHUB_OUTPUT
+
+      - name: Set up caching for the performance results
+        uses: actions/cache@v4
+        with:
+          path: perf_cache
+          # If profiling result cache has incompatible format, increase this "v" number
+          key: "v13-${{ steps.calc.outputs.hash }}-${{ env.TEST_DATA_URL }}"
+
+      - name: Load test data into DB and run performance test
+        id: main
+        env:
+          CACHE_SHA: "${{ steps.calc.outputs.hash }}"
+        run: |
+          profile() {
+            TIME_FORMAT="# %C
+          P\t%P\tPercentage of the CPU that this job got.  This is just user + system times divided by the total running time.  It also prints a percentage sign.
+          x\t%x\tExit status of the command.
+          -\t-----\t-- Time --------------------------------------------------------------------------------------------------
+          e\t%e\tElapsed real (wall clock) time used by the process, in seconds.
+          E\t%E\tElapsed real (wall clock) time used by the process, in [hours:]minutes:seconds.
+          U\t%U\tTotal number of CPU-seconds that the process used directly (in user mode), in seconds.
+          S\t%S\tTotal number of CPU-seconds used by the system on behalf of the process (in kernel mode), in seconds.
+          -\t-----\t-- Context Switches --------------------------------------------------------------------------------------
+          c\t%c\tNumber of times the process was context-switched involuntarily (because the time slice expired).
+          w\t%w\tNumber of times that the program was context-switched voluntarily, for instance while waiting for an I/O operation to complete.
+          -\t-----\t-- Page faults -------------------------------------------------------------------------------------------
+          F\t%F\tNumber of major, or I/O-requiring, page faults that occurred while the process was running.  These are faults where the page has actually migrated out of primary memory.
+          R\t%R\tNumber of minor, or recoverable, page faults.  These are pages that are not valid (so they fault) but which have not yet been claimed by other virtual pages.  Thus the data in the page is still valid but the system tables must be updated.
+          W\t%W\tNumber of times the process was swapped out of main memory.
+          -\t-----\t-- Memory ------------------------------------------------------------------------------------------------
+          K\t%K\tAverage total (data+stack+text) memory use of the process, in Kilobytes.
+          t\t%t\tAverage resident set size of the process, in Kilobytes.
+          p\t%p\tAverage unshared stack size of the process, in Kilobytes.
+          M\t%M\tMaximum resident set size of the process during its lifetime, in Kilobytes.
+          D\t%D\tAverage size of the process's unshared data area, in Kilobytes.
+          X\t%X\tAverage amount of shared text in the process, in Kilobytes.
+          Z\t%Z\tSystem's page size, in bytes.  This is a per-system constant, but varies between systems.
+          -\t-----\t-- Other -------------------------------------------------------------------------------------------------
+          I\t%I\tNumber of file system inputs by the process.
+          O\t%O\tNumber of file system outputs by the process.
+          k\t%k\tNumber of signals delivered to the process.
+          r\t%r\tNumber of socket messages received by the process.
+          s\t%s\tNumber of socket messages sent by the process.
+          "
+
+            # reset system for a more predictable results (hopefully)
+            sudo -- bash -c "
+              set -euo pipefail
+              echo 'Hardware reset $1 ...'
+              # Run the TRIM command (for SSDs)
+              /sbin/fstrim --all
+              # Run sync to minimize the number of dirty objects on the system
+              /bin/sync
+              # Give sync a little bit of time to finish. See https://linux.die.net/man/8/sync
+              sleep 5
+              # Free slab objects and pagecache
+              echo 3 > /proc/sys/vm/drop_caches
+              # Request compaction to reduce memory fragmentation
+              echo 1 > /proc/sys/vm/compact_memory
+              echo 'Hardware reset $1 done'
+            "
+
+            # Must use full path to get the full-featured version of time
+            # profile-*.tsv filenames are parsed using ${file:8:-4} below
+            /usr/bin/time --format "$TIME_FORMAT" --output "${PROFILE_DIR}/profile-${1}.tsv" "${@:2}"
+          }
+
+          create_db() {
+            make clean
+            make init-dirs
+            cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas.osm.pbf
+            make destroy-db
+            make all
+            make start-db
+            profile 1_data     make import-data
+            profile 2_osm      make import-osm
+
+            if [ -f ../ci_cache/wikidata-cache.json ]; then
+              cp ../ci_cache/wikidata-cache.json cache/wikidata-cache.json
+            fi
+            profile 3_wikidata make import-wikidata
+            profile 4_sql      make import-sql
+
+            # Get database total size, in MB
+            # Once Makefile has a few more improvements, we can use this approach instead:
+            # echo $'\\set QUIET on \\a \\x off \\t \\\\ select pg_database_size(current_database())/1024/1024;' | make -s psql
+            DB_SIZE_MB=$(docker-compose run --rm -u $(id -u):$(id -g) openmaptiles-tools psql.sh -qtAc 'select pg_database_size(current_database())/1024/1024;')
+            docker-compose run --rm -u $(id -u):$(id -g) openmaptiles-tools pg_dump --schema-only > "${PROFILE_DIR}/schema.sql"
+            echo "$DB_SIZE_MB" > "${PROFILE_DIR}/db_size.tsv"
+          }
+
+          echo "Ensuring we have the needed dirs"
+          pwd
+          mkdir -p perf_cache
+          mkdir -p artifacts
+          mkdir -p pr_message
+          cd code
+
+          CURRENT_SHA=$(git log -1 --format="%H")
+
+          if [ ! -f ../perf_cache/results.json ]; then
+            echo "We do not have cached performance results, create them..."
+            if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+              git reset --hard ${CURRENT_SHA}^1
+            fi
+
+            docker-compose pull
+            PROFILE_DIR=../perf_cache
+            create_db
+            if [ ! -f ../ci_cache/wikidata-cache.json ]; then
+              cp cache/wikidata-cache.json ../ci_cache/wikidata-cache.json
+            fi
+
+            (set -x; profile test-perf docker-compose run --rm -T openmaptiles-tools \
+              test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
+              --record /tileset/results.json)
+            echo "Done generating base perf results, moving them to ../perf_cache"
+            pwd
+            mv results.json ../perf_cache
+
+            if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+              # For pull requests, restore to the PR version before continuing
+              git reset --hard ${CURRENT_SHA}
+            fi
+          else
+            echo "Found cached performance results"
+          fi
+
+          docker-compose pull
+          pushd ../perf_cache
+          echo "Should be in perf_cache"
+          pwd
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            cp results.json ../artifacts/base-results.json
+            # Copy all tsv files, not just the ones with "profile-" prefix.
+            for file in *.tsv; do
+              cp "$file" "../artifacts/base-$file"
+            done
+          else
+            cp results.json ../artifacts/results.json
+            # Copy all tsv files, not just the ones with "profile-" prefix.
+            cp *.tsv ../artifacts
+          fi
+          popd
+
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            echo "Comparing pull request results with the base..."
+
+            PROFILE_DIR=../artifacts
+            create_db
+
+            echo "Copying existing perf_cache results to current dir"
+            pwd
+            cp ../perf_cache/results.json .
+            OUTPUT="$(set -x; profile test-perf docker-compose run --rm -T openmaptiles-tools \
+                        test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
+                        --compare /tileset/results.json --record /tileset/pr-results.json)"
+            rm results.json
+            mv pr-results.json ../artifacts/
+
+            # Split into two parts -- before and after the ===== SUMMARY =====
+            OUT_SUMMARY="${OUTPUT##*========}"
+            OUT_DETAILS="${OUTPUT%%========*}"
+
+            pushd ../artifacts
+            PROFILING_MSG="step | total time | change | user time | change
+            --- | --- | --- | --- | ---
+            "
+            for file in profile*.tsv; do
+              new_total_time_fmt=$(grep -E '^E' "$file"   | cut -d$'\t' -f 2)
+              old_total_time=$(grep -E '^e' "base-$file"  | cut -d$'\t' -f 2)
+              new_total_time=$(grep -E '^e' "$file"       | cut -d$'\t' -f 2)
+              old_user_time=$(grep -E '^U' "base-$file"   | cut -d$'\t' -f 2)
+              new_user_time=$(grep -E '^U' "$file"        | cut -d$'\t' -f 2)
+
+              PROFILING_MSG="${PROFILING_MSG}${file:8:-4} \
+              | ${new_total_time_fmt} | $( echo "$old_total_time $new_total_time" | awk '{ if ($1 == 0) print "-"; else printf  "%.1f%%", ($2-$1)*100/$1 }' ) \
+              | $( echo "$old_user_time $new_user_time" | awk '{ if ($1 == 0) print "%s | -", $2 ; else printf  "%s | %.1f%%", $2, ($2-$1)*100/$1 }' ) \
+              "$'\n'
+            done
+
+            DB_SIZE_CHANGE=$( echo "$(cat base-db_size.tsv) $(cat db_size.tsv)" | \
+              awk '{ printf "PostgreSQL DB size in MB: %s ⇒ %s", $1, $2; if ($1 != 0) printf " (%.1f%% change)", ($2-$1)*100/$1 }' )
+
+            popd
+
+            cat > ../pr_message/message.md <<EOF
+          Results evaluating commit $(git log HEAD^2 -1 --format="%H") (merged with base $CACHE_SHA as $GITHUB_SHA). See [run details](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID).
+
+          $DB_SIZE_CHANGE
+
+          $PROFILING_MSG
+
+          \`\`\`
+          $OUT_SUMMARY
+          \`\`\`
+
+          <details>
+          <summary>expand for details...</summary>
+
+          \`\`\`
+          $OUT_DETAILS
+          \`\`\`
+
+          </details>
+          EOF
+
+          fi
+
+      - name: Save performance artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance_results
+          path: artifacts
+
+      - name: Save PR message artifact
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr_message
+          path: pr_message

.github/workflows/pr-updater.yml

@@ -0,0 +1,246 @@
+name: Update PR comments
+
+on:
+  workflow_run:
+    workflows: ["OpenMapTiles Performance CI"]
+    types: [completed]
+
+jobs:
+  update_PRs:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+
+    steps:
+      - name: main
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          WORKFLOW_NAME: "OpenMapTiles Performance CI"
+          # the name of the artifact whose content comment published by PR. Must have a single markdown file inside.
+          MSG_ARTIFACT_NAME: "pr_message"
+          # How far back to look for finished runs, in minutes.
+          # Set to 10-20 minutes higher than cron's job frequency set above.
+          IGNORE_RUNS_OLDER_THAN: 80
+          # How far back to look for updated pull requests, in minutes.
+          # Should be bigger than IGNORE_RUNS_OLDER_THAN by the maximum time a pull request jobs may take
+          IGNORE_PRS_OLDER_THAN: 80
+        run: |
+          #
+          # Strategy:
+          #  * get all recently updated pull requests
+          #  * get all recent workflow runs
+          #  * match pull requests and their current SHA with the last workflow run for the same SHA
+          #  * for each found match of  <pull-request-number>  and  <workflow-run-id> :
+          #     * download artifact from the workflow run -- expects a single file with markdown content
+          #     * look through existing PR comments to see if we have posted a comment before
+          #       (uses a hidden magical header to identify our comment)
+          #     * either create or update the comment with the new text (if changed)
+          #
+
+          export GITHUB_API="https://api.github.com/repos/$GITHUB_REPOSITORY"
+          export COMMENT_MAGIC_HEADER='<!--'" Do not edit. This comment will be auto-updated with artifact '$MSG_ARTIFACT_NAME' created by action '$WORKFLOW_NAME' -->"
+
+          # A useful wrapper around CURL
+          crl() {
+            curl --silent --show-error --location --retry 1 "${@:2}" \
+              -H "Accept: application/vnd.github.antiope-preview+json, application/vnd.github.v3+json" \
+             "$1"
+          }
+
+          auth_crl() {
+            crl "$1" -H "authorization: Bearer $GITHUB_TOKEN" "${@:2}"
+          }
+
+          #
+          # Parse current pull requests
+          #
+
+          # Get all pull requests, most recently updated first
+          # (this way we don't need to page through all of them)
+          # Filter out PRs that are older than $IGNORE_PRS_OLDER_THAN minutes
+          # Result is an object, mapping a "key" to the pull request number:
+          # {
+          #   "nyurik/openmaptiles/nyurik-patch-1/4953dd2370b9988a7832d090b5e47b3cd867f594": 6,
+          #   ...
+          # }
+          PULL_REQUESTS_RAW="$( crl "$GITHUB_API/pulls?sort=updated&direction=desc" )"
+          if ! PULL_REQUESTS="$(jq --arg IGNORE_PRS_OLDER_THAN "$IGNORE_PRS_OLDER_THAN" '
+              map(
+                  # Only select unlocked pull requests updated within last $IGNORE_PRS_OLDER_THAN minutes
+                  select(.locked==false
+                         and (now - (.updated_at|fromdate)) / 60 < ($IGNORE_PRS_OLDER_THAN | tonumber))
+                  # Prepare for "from_entries" by creating a key/value object
+                  # The key is a combination of repository name, branch name, and latest SHA
+                  | { key: (.head.repo.full_name + "/" + .head.ref + "/" + .head.sha), value: .number }
+              )
+              | from_entries
+            ' <( echo "$PULL_REQUESTS_RAW" ) )"; then
+
+              echo "Error parsing pull requests"
+              echo "$PULL_REQUESTS_RAW"
+              exit 1
+          fi
+
+          # Count how many pull requests we should process, and exit early if there are none
+          PR_COUNT="$(jq 'length' <( echo "$PULL_REQUESTS" ) )"
+          if [ "$PR_COUNT" -eq 0 ]; then
+            echo "There are no pull requests updated in the last $IGNORE_PRS_OLDER_THAN minutes. Exiting."
+            exit
+          else
+            echo "$PR_COUNT pull requests have been updated in the last $IGNORE_PRS_OLDER_THAN minutes"
+            echo "$PULL_REQUESTS" | jq -r 'keys|.[]|" * " + .'
+          fi
+
+
+
+          #
+          # Resolve workflow name into workflow ID
+          #
+          WORKFLOW_ID="$(crl "$GITHUB_API/actions/workflows" \
+              | jq --arg WORKFLOW_NAME "$WORKFLOW_NAME" '
+              .workflows[] | select(.name == $WORKFLOW_NAME) | .id
+              ')"
+          if [ -z "$WORKFLOW_ID" ]; then
+            echo "Unable to find workflow '$WORKFLOW_NAME' in $GITHUB_REPOSITORY"
+            exit 1
+          else
+            echo "Resolved workflow '$WORKFLOW_NAME' to ID $WORKFLOW_ID"
+          fi
+
+
+
+          #
+          # Match pull requests with the workflow runs
+          #
+
+          # Get all workflow runs that were triggered by pull requests
+          WORKFLOW_PR_RUNS="$(crl "$GITHUB_API/actions/workflows/${WORKFLOW_ID}/runs?event=pull_request")"
+
+          # For each workflow run, match it with the pull request to get the PR number
+          # A match is based on "source repository + branch + SHA" key
+          # In rare cases (e.g. force push to an older revision), there could be more than one match
+          # for a given PR number, so just use the most recent one.
+          # Result is a table (list of lists) - each row with PR number, JOB ID, and the above key
+          PR_JOB_MAP="$(jq --arg IGNORE_RUNS_OLDER_THAN "$IGNORE_RUNS_OLDER_THAN" '
+              # second input is the pull request map - use it to lookup PR numbers
+              input as $PULL_REQUESTS
+              | .workflow_runs
+              | map(
+                  # Create a new object with the relevant values
+                  {
+                      id,
+                      updated_at,
+                      # create lookup key based on source repository + branch + SHA
+                      key: (.head_repository.full_name + "/" + .head_branch + "/" + .head_sha),
+                      # was this a successful run?
+                      # do not include  .conclusion=="success"  because errors could also post messages
+                      success: (.status=="completed")
+                  }
+                  # lookup PR number from $PULL_REQUESTS using the above key
+                  | . += { pr_number: $PULL_REQUESTS[.key] }
+                  # Remove runs that were not in the list of the PRs
+                  | select(.pr_number)
+              )
+              # Keep just the most recent run per pull request
+              | group_by(.pr_number)
+              | map(
+                  sort_by(.updated_at)
+                  | last
+                  # If the most recent run did not succeed, or if the run is too old, ignore it
+                  | select(.success and (now - (.updated_at|fromdate)) / 60 < ($IGNORE_RUNS_OLDER_THAN | tonumber))
+                  # Keep just the pull request number mapping to run ID
+                  | { pr_number, id, key }
+              )
+              ' <( echo "$WORKFLOW_PR_RUNS" ) <( echo "$PULL_REQUESTS" ) )"
+
+
+          # Count how many jobs we should process, and exit early if there are none
+          JOBS_COUNT="$(jq 'length' <( echo "$PR_JOB_MAP" ) )"
+          if [ "$JOBS_COUNT" -eq 0 ]; then
+            echo "There are no recent workflow job runs in the last $IGNORE_RUNS_OLDER_THAN minutes. Exiting."
+            exit
+          else
+            echo "$JOBS_COUNT '$WORKFLOW_NAME' jobs have been updated in the last $IGNORE_RUNS_OLDER_THAN minutes"
+            echo "$PR_JOB_MAP" | jq -r '.[] | " * PR #\(.pr_number) Job #\(.id) -- \(.key) "'
+          fi
+
+          #
+          # Iterate over the found pairs of PR number + run ID, and update them all
+          #
+          echo "$PR_JOB_MAP" | jq -r '.[] | [ .pr_number, .id, .key ] | @sh' | \
+          while read -r PR_NUMBER RUN_ID RUN_KEY; do
+
+            echo "Processing '$WORKFLOW_NAME' run #$RUN_ID for pull request #$PR_NUMBER $RUN_KEY..."
+            ARTIFACTS="$(crl "$GITHUB_API/actions/runs/$RUN_ID/artifacts")"
+
+            # Find the artifact download URL for the artifact with the expected name
+            ARTIFACT_URL="$(jq -r --arg MSG_ARTIFACT_NAME "$MSG_ARTIFACT_NAME" '
+                .artifacts
+                | map(select(.name == $MSG_ARTIFACT_NAME and .expired == false))
+                | first
+                | .archive_download_url
+                | select(.!=null)
+                ' <( echo "$ARTIFACTS" ) )"
+
+            if [ -z "$ARTIFACT_URL" ]; then
+              echo "Unable to find an artifact named '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
+              continue
+            fi
+
+            echo "Downloading artifact $ARTIFACT_URL (assuming single text file per artifact)..."
+            if ! MESSAGE="$(auth_crl "$ARTIFACT_URL" | gunzip)"; then
+              echo "Unable to download or parse message from artifact '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
+              continue
+            fi
+            if [ -z "$MESSAGE" ]; then
+              echo "Empty message in artifact '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
+              continue
+            fi
+
+            # Create a message body by appending a magic header
+            # and stripping any starting and ending whitespace from the original message
+            MESSAGE_BODY="$(jq -n \
+                --arg COMMENT_MAGIC_HEADER "$COMMENT_MAGIC_HEADER" \
+                --arg MESSAGE "$MESSAGE" \
+                '{ body: ($COMMENT_MAGIC_HEADER + "\n" + ($MESSAGE | sub( "^[\\s\\p{Cc}]+"; "" ) | sub( "[\\s\\p{Cc}]+$"; "" ))) }' \
+            )"
+
+            EXISTING_PR_COMMENTS="$(crl "$GITHUB_API/issues/$PR_NUMBER/comments")"
+
+            # Get the comment URL for the first comment that begins with the magic header, or empty string
+            OLD_COMMENT="$(jq --arg COMMENT_MAGIC_HEADER "$COMMENT_MAGIC_HEADER" '
+                map(select(.body | startswith($COMMENT_MAGIC_HEADER)))
+                | first
+                | select(.!=null)
+                ' <( echo "$EXISTING_PR_COMMENTS" ) )"
+
+            if [ -z "$OLD_COMMENT" ]; then
+              COMMENT_HTML_URL="$(auth_crl "$GITHUB_API/issues/$PR_NUMBER/comments" \
+                  -X POST \
+                  -H "Content-Type: application/json" \
+                  --data "$MESSAGE_BODY" \
+                | jq -r '.html_url' )"
+              COMMENT_INFO="New comment $COMMENT_HTML_URL was created"
+            else
+              # Make sure the content of the message has changed
+              COMMENT_URL="$(jq -r '
+                  (input | .body) as $body
+                  | select(.body | . != $body)
+                  | .url
+                  ' <( echo "$OLD_COMMENT" ) <( echo "$MESSAGE_BODY" ) )"
+
+              if [ -z "$COMMENT_URL" ]; then
+                echo "The message has already been posted from artifact '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
+                continue
+              fi
+
+              COMMENT_HTML_URL="$(auth_crl "$COMMENT_URL" \
+                  -X PATCH \
+                  -H "Content-Type: application/json" \
+                  --data "$MESSAGE_BODY" \
+                | jq -r '.html_url' )"
+
+              COMMENT_INFO="Existing comment $COMMENT_HTML_URL was updated"
+            fi
+
+            echo "$COMMENT_INFO from workflow $WORKFLOW_NAME #$RUN_ID"
+          done

.github/workflows/sql-tests.yml

@@ -0,0 +1,22 @@
+# Workflow to run unit tests on OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles SQL Test CI
+
+on:
+  push:
+    branches: [ master, master-tools ]
+  pull_request:
+
+jobs:
+
+  unit_tests:
+    name: Run unit test
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout the changes
+        uses: actions/checkout@v4
+
+      - name: Run unit tests
+        run: |
+          make clean && make test-sql

.gitignore

@@ -11,12 +11,12 @@ quickstart.log
 # imput / output data
 data/*
 
-# input wikidata
-wikidata/*
-
 # generated source files
 build/*
 
+# Import cache
+cache/*
+
 # any IDE files
 .idea/
 .vscode/

.travis.yml

@@ -1,11 +0,0 @@
-sudo: required
-
-language: bash
-
-services:
-  - docker
-
-script:
-  - set -e
-  - TEST_MODE=yes make generate-devdoc
-  - ./quickstart.sh

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Introduction
 
-Thank you for considering contributing to OpenMapTiles. It's people like you that make OpenMapTiles such a great project. Talk to us at the OSM Slack **#openmaptiles** channel ([join](https://osmus-slack.herokuapp.com/)).
+Thank you for considering contributing to OpenMapTiles. It's people like you that make OpenMapTiles such a great project. Talk to us at the OSM Slack **#openmaptiles** channel ([join](https://slack.openstreetmap.us/)).
 
 Following these guidelines helps to communicate that you respect the time of the developers managing and developing this open source project. In return, they should reciprocate that respect in addressing your issue, assessing changes, and helping you finalize your pull requests.
 
@@ -32,16 +32,57 @@ Your pull request must:
  * Use clear commit messages.
  * Be possible to merge automatically.
 
-When you modify rules of importing data in `mapping.yaml` or `*.sql`, please update also:
+When you modify import data rules in `mapping.yaml` or `*.sql`, please update:
 
 1. field description in `[layer].yaml`
 2. comments starting with `#etldoc`
-3. if needed, generate new `mapping_diagram.png` or `etl_diagram.png` using commands below:
-```
-make mapping-graph-[layer]
-make etl-graph-[layer]
-```
+3. regenerate documentation graphs with `make generate-devdoc`
 4. update layer description on https://openmaptiles.org/schema/ (https://github.com/openmaptiles/www.openmaptiles.org/tree/master/layers)
 5. check if OMT styles are affected by the PR and if there is a need for style updates
 
 When you are making PR that adds new spatial features to OpenMapTiles schema, please make also PR for at least one of our GL styles to show it on the map. Visual check is crucial.
+
+# SQL unit testing
+
+It is recommended that you create a [unit test](TESTING.md) when modifying the behavior of the SQL layer.  This will ensure that your changes are working as expected when importing or updating OSM data into an OpenMapTiles database.
+
+# Verifying that updates still work
+
+When testing a PR, you should also verify that the update process completes without an error. Please modify, if necessary, and run the script below.
+
+**Note:**
+
+The verification requires the script to append temporary changes to the `.env` file. Please restore the original version from git using `git checkout .env` or remove these changes before submitting a PR.
+
+```
+(
+set -e
+
+cat >> .env << EOM
+
+# temporary changes for verifying that updates still work
+# Ensure DIFF_MODE is active
+DIFF_MODE=true
+# Ensure all zoom levels are tested
+MAX_ZOOM=14
+EOM
+
+# Set the test area to the appropriate geofabrik extract
+export area=north-america/us/indiana
+
+# Build 1-month-old tiles
+rm -fr data build cache
+make destroy-db
+make download-geofabrik area=$area
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -nv -O data/$area.osm.pbf http://download.geofabrik.de/$area-$(date --date="$(date +%Y-%m-15) -1 month" +'%y%m01').osm.pbf"
+./quickstart.sh $area
+cat << EOM
+
+# Update with the changes since a month+ ago
+
+EOM
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "osmupdate --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz"
+make import-diff
+make generate-tiles-pg
+) < /dev/null
+```

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors.  
+Copyright (c) 2024, MapTiler.com & OpenMapTiles contributors.  
 All rights reserved.  
 
 The vector tile schema has been developed by Klokan Technologies GmbH and
@@ -55,6 +55,6 @@ For printed and static maps a similar attribution should be made in a textual
 description near the image, in the same fashion as if you cite a photograph.
 
 Exceptions to OpenMapTiles attribution requirement can be in a written form granted
-by Klokan Technologies GmbH (info@klokantech.com).  
-The project contributors grant Klokan Technologies GmbH the license to give such
+by MapTiler (info@maptiler.com).  
+The project contributors grant MapTiler AG the license to give such
 exceptions on a commercial basis.

Makefile

@@ -1,286 +1,697 @@
-# Options to run with docker and docker-compose - ensure the container is destroyed on exit
-DC_OPTS?=--rm
+#
+# First section - common variable initialization
+#
 
-# container runs as the current user rather than root (so that created files are not root-owned)
-DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+# Ensure that errors don't hide inside pipes
+SHELL         = /bin/bash
+.SHELLFLAGS   = -o pipefail -c
+
+# Layers definition and meta data
+TILESET_FILE := $(or $(TILESET_FILE),$(shell (. .env; echo $${TILESET_FILE})),openmaptiles.yaml)
+
+# Options to run with docker and docker-compose - ensure the container is destroyed on exit
+# Containers run as the current user rather than root (so that created files are not root-owned)
+DC_OPTS ?= --rm --user=$(shell id -u):$(shell id -g)
+
+# If set to a non-empty value, will use postgis-preloaded instead of postgis docker image
+USE_PRELOADED_IMAGE ?=
+
+# Local port to use with postserve
+PPORT ?= 8090
+export PPORT
+# Local port to use with tileserver
+TPORT ?= 8080
+export TPORT
+STYLE_FILE := build/style/style.json
+STYLE_HEADER_FILE := style/style-header.json
+
+# Support newer `docker compose` syntax in addition to `docker-compose`
+
+ifeq (, $(shell which docker-compose))
+  DOCKER_COMPOSE_COMMAND := docker compose
+  $(info Using docker compose V2 (docker compose))
+else
+  DOCKER_COMPOSE_COMMAND := docker-compose
+  $(info Using docker compose V1 (docker-compose))
+endif
+
+# Allow a custom docker-compose project name
+DC_PROJECT := $(or $(DC_PROJECT),$(shell (. .env; echo $${DC_PROJECT})))
+ifeq ($(DC_PROJECT),)
+  DC_PROJECT := $(notdir $(shell pwd))
+  DOCKER_COMPOSE := $(DOCKER_COMPOSE_COMMAND)
+else
+  DOCKER_COMPOSE := $(DOCKER_COMPOSE_COMMAND) --project-name $(DC_PROJECT)
+endif
+
+# Make some operations quieter (e.g. inside the test script)
+ifeq ($(or $(QUIET),$(shell (. .env; echo $${QUIET})))),)
+  QUIET_FLAG :=
+else
+  QUIET_FLAG := --quiet
+endif
+
+# Use `xargs --no-run-if-empty` flag, if supported
+XARGS := xargs $(shell xargs --no-run-if-empty </dev/null 2>/dev/null && echo --no-run-if-empty)
 
 # If running in the test mode, compare files rather than copy them
 TEST_MODE?=no
 ifeq ($(TEST_MODE),yes)
-  COPY_TO_GIT=diff
+  # create images in ./build/devdoc and compare them to ./layers
+  GRAPH_PARAMS=./build/devdoc ./layers
 else
-  COPY_TO_GIT=cp
+  # update graphs in the ./layers dir
+  GRAPH_PARAMS=./layers
 endif
 
+# Set OpenMapTiles host
+export OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
+
+# This defines an easy $(newline) value to act as a "\n". Make sure to keep exactly two empty lines after newline.
+define newline
+
+
+endef
+
+# Use the old Postgres connection values as a fallback
+PGHOST := $(or $(PGHOST),$(shell (. .env; echo $${PGHOST})),$(POSTGRES_HOST),$(shell (. .env; echo $${POSTGRES_HOST})),postgres)
+PGPORT := $(or $(PGPORT),$(shell (. .env; echo $${PGPORT})),$(POSTGRES_PORT),$(shell (. .env; echo $${POSTGRES_PORT})),postgres)
+PGDATABASE := $(or $(PGDATABASE),$(shell (. .env; echo $${PGDATABASE})),$(POSTGRES_DB),$(shell (. .env; echo $${POSTGRES_DB})),postgres)
+PGUSER := $(or $(PGUSER),$(shell (. .env; echo $${PGUSER})),$(POSTGRES_USER),$(shell (. .env; echo $${POSTGRES_USER})),postgres)
+PGPASSWORD := $(or $(PGPASSWORD),$(shell (. .env; echo $${PGPASSWORD})),$(POSTGRES_PASSWORD),$(shell (. .env; echo $${POSTGRES_PASSWORD})),postgres)
+
+#
+# Determine area to work on
+# If $(area) parameter is not set, and only one *.osm.pbf file is found in ./data, use it as $(area).
+# Otherwise, all make targets requiring an area will show an error.
+# Note: If no *.osm.pbf files are found, once the users call  "make download area=..."
+#       they will not need to use an "area=" parameter again because there will be just a single file.
+#
+
+# historically we have been using $(area) rather than $(AREA), so make both work
+area ?= $(AREA)
+# Ensure the $(area) param is set, or try to automatically determine it based on available data files
+ifeq ($(area),)
+  # An $(area) parameter is not set. If only one *.osm.pbf file is found in ./data, use it as $(area).
+  data_files := $(shell find data -name '*.osm.pbf' 2>/dev/null)
+  ifneq ($(word 2,$(data_files)),)
+    define assert_area_is_given
+	  @echo ""
+	  @echo "ERROR: The 'area' parameter or environment variable have not been set, and there several 'area' options:"
+	  @$(patsubst data/%.osm.pbf,echo "  '%'";,$(data_files))
+	  @echo ""
+	  @echo "To specify an area use:"
+	  @echo "  make $@ area=<area-id>"
+	  @echo ""
+	  @exit 1
+    endef
+  else
+    ifeq ($(word 1,$(data_files)),)
+      define assert_area_is_given
+        @echo ""
+        @echo "ERROR: The 'area' parameter (or env var) has not been set, and there are no data/*.osm.pbf files"
+        @echo ""
+        @echo "To specify an area use"
+        @echo "  make $@ area=<area-id>"
+        @echo ""
+        @echo "To download an area, use   make download area=<area-id>"
+        @echo "To list downloadable areas, use   make list-geofabrik   and/or   make list-bbbike"
+        @exit 1
+        @echo ""
+      endef
+    else
+      # Keep just the name of the data file, without the .osm.pbf extension
+      area := $(patsubst data/%.osm.pbf,%,$(data_files))
+      # Rename area-latest.osm.pbf to area.osm.pbf
+      # TODO: This if statement could be removed in a few months once everyone is using the file without the `-latest`?
+      ifneq ($(area),$(area:-latest=))
+        $(shell mv "data/$(area).osm.pbf" "data/$(area:-latest=).osm.pbf")
+        area := $(area:-latest=)
+        $(warning ATTENTION: File data/$(area)-latest.osm.pbf was renamed to $(area).osm.pbf.)
+        AREA_INFO := Detected area=$(area) based on finding a 'data/$(area)-latest.osm.pbf' file - renamed to '$(area).osm.pbf'. Use 'area' parameter or environment variable to override.
+      else
+        AREA_INFO := Detected area=$(area) based on finding a 'data/$(area).osm.pbf' file. Use 'area' parameter or environment variable to override.
+      endif
+    endif
+  endif
+endif
+
+ifneq ($(AREA_INFO),)
+  define assert_area_is_given
+      @echo "$(AREA_INFO)"
+  endef
+endif
+
+# If set, this file will be downloaded in download-osm and imported in the import-osm targets
+PBF_FILE ?= data/$(area).osm.pbf
+
+# For download-osm, allow URL parameter to download file from a given URL. Area param must still be provided.
+DOWNLOAD_AREA := $(or $(url), $(area))
+
+# The mbtiles file is placed into the $EXPORT_DIR=/export (mapped to ./data)
+MBTILES_FILE := $(or $(MBTILES_FILE),$(shell (. .env; echo $${MBTILES_FILE})),$(area).mbtiles)
+MBTILES_LOCAL_FILE = data/$(MBTILES_FILE)
+
+DIFF_MODE := $(or $(DIFF_MODE),$(shell (. .env; echo $${DIFF_MODE})))
+ifeq ($(DIFF_MODE),true)
+  # import-osm implementation requires IMPOSM_CONFIG_FILE to be set to a valid file
+  # For one-time only imports, the default value is fine.
+  # For diff mode updates, use the dynamically-generated area-based config file
+  export IMPOSM_CONFIG_FILE = data/$(area).repl.json
+endif
+
+# Load area-specific bbox file that gets generated by the download-osm --bbox
+AREA_BBOX_FILE ?= data/$(area).bbox
+ifneq (,$(wildcard $(AREA_BBOX_FILE)))
+  cat := $(if $(filter $(OS),Windows_NT),type,cat)
+  BBOX := $(shell $(cat) ${AREA_BBOX_FILE})
+  export BBOX
+endif
+
+# Consult .env if needed
+MIN_ZOOM := $(or $(MIN_ZOOM),$(shell (. .env; echo $${MIN_ZOOM})),0)
+MAX_ZOOM := $(or $(MAX_ZOOM),$(shell (. .env; echo $${MAX_ZOOM})),7)
+PPORT := $(or $(PPORT),$(shell (. .env; echo $${PPORT})),7)
+TPORT := $(or $(TPORT),$(shell (. .env; echo $${TPORT})),7)
+
+define HELP_MESSAGE
+==============================================================================
+OpenMapTiles  https://github.com/openmaptiles/openmaptiles
+
+Hints for testing areas
+  make list-geofabrik                  # list actual geofabrik OSM extracts for download -> <<your-area>>
+  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar
+
+Hints for designers:
+  make start-maputnik                  # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]
+  make stop-maputnik                   # stop Maputnik Editor + dynamic tile server
+  make start-postserve                 # start dynamic tile server                   [ see $(OMT_HOST):$(PPORT) ]
+  make stop-postserve                  # stop dynamic tile server
+  make start-tileserver                # start maptiler/tileserver-gl                [ see $(OMT_HOST):$(TPORT) ]
+  make stop-tileserver                 # stop maptiler/tileserver-gl
+
+Hints for developers:
+  make                                 # build source code
+  make bash                            # start openmaptiles-tools /bin/bash terminal
+  make generate-bbox-file              # compute bounding box of a data file and store it in a file
+  make generate-devdoc                 # generate devdoc including graphs for all layers [./layers/...]
+  make generate-qa                     # statistics for a given layer's field
+  make generate-tiles-pg               # generate vector tiles based on .env settings using PostGIS ST_MVT()
+  make generate-tiles                  # generate vector tiles based on .env settings using Mapnik (obsolete)
+  make generate-changed-tiles          # Generate tiles changed by import-diff
+  make test-sql                        # run unit tests on the OpenMapTiles SQL schema
+  cat  .env                            # list PG database and MIN_ZOOM and MAX_ZOOM information
+  cat  quickstart.log                  # transcript of the last ./quickstart.sh run
+  make help                            # help about available commands
+
+Hints for downloading & importing data:
+  make list-geofabrik                  # list actual geofabrik OSM extracts for download
+  make list-bbbike                     # list actual BBBike OSM extracts for download
+  make download area=albania           # download OSM data from any source       and create config file
+  make download-geofabrik area=albania # download OSM data from geofabrik.de     and create config file
+  make download-osmfr area=asia/qatar  # download OSM data from openstreetmap.fr and create config file
+  make download-bbbike area=Amsterdam  # download OSM data from bbbike.org       and create config file
+  make import-data                     # Import data from OpenStreetMapData, Natural Earth and OSM Lake Labels.
+  make import-osm                      # Import OSM data with the mapping rules from build/mapping.yaml
+  make import-diff                     # Import OSM updates from data/changes.osc.gz
+  make import-wikidata                 # Import labels from Wikidata
+  make import-sql                      # Import layers (run this after modifying layer SQL)
+
+Hints for database management:
+  make psql                            # start PostgreSQL console
+  make psql-list-tables                # list all PostgreSQL tables
+  make list-views                      # list PostgreSQL public schema views
+  make list-tables                     # list PostgreSQL public schema tables
+  make vacuum-db                       # PostgreSQL: VACUUM ANALYZE
+  make analyze-db                      # PostgreSQL: ANALYZE
+  make destroy-db                      # remove docker containers and PostgreSQL data volume
+  make start-db                        # start PostgreSQL, creating it if it doesn't exist
+  make start-db-preloaded              # start PostgreSQL, creating data-prepopulated one if it doesn't exist
+  make stop-db                         # stop PostgreSQL database without destroying the data
+
+Hints for Docker management:
+  make clean-unnecessary-docker        # clean unnecessary docker image(s) and container(s)
+  make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB
+  make remove-docker-images            # remove openmaptiles docker images
+  make list-docker-images              # show a list of available docker images
+==============================================================================
+endef
+export HELP_MESSAGE
+
+#
+#  TARGETS
+#
+
 .PHONY: all
-all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
+all: init-dirs build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql build-style
 
 .PHONY: help
 help:
-	@echo "=============================================================================="
-	@echo " OpenMapTiles  https://github.com/openmaptiles/openmaptiles "
-	@echo "Hints for testing areas                "
-	@echo "  make download-geofabrik-list         # list actual geofabrik OSM extracts for download -> <<your-area>> "
-	@echo "  make list                            # list actual geofabrik OSM extracts for download -> <<your-area>> "
-	@echo "  ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar "
-	@echo "  "
-	@echo "Hints for designers:"
-	@echo "  make start-postserve                 # start Postserver + Maputnik Editor [ see localhost:8088 ] "
-	@echo "  make start-tileserver                # start klokantech/tileserver-gl [ see localhost:8080 ] "
-	@echo "  "
-	@echo "Hints for developers:"
-	@echo "  make                                 # build source code"
-	@echo "  make download-geofabrik area=albania # download OSM data from geofabrik, and create config file"
-	@echo "  make psql                            # start PostgreSQL console"
-	@echo "  make psql-list-tables                # list all PostgreSQL tables"
-	@echo "  make psql-vacuum-analyze             # PostgreSQL: VACUUM ANALYZE"
-	@echo "  make psql-analyze                    # PostgreSQL: ANALYZE"
-	@echo "  make generate-qareports              # generate reports [./build/qareports]"
-	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers  [./build/devdoc]"
-	@echo "  make etl-graph                       # hint for generating a single etl graph"
-	@echo "  make mapping-graph                   # hint for generating a single mapping graph"
-	@echo "  make import-sql-dev                  # start import-sql /bin/bash terminal"
-	@echo "  make import-osm-dev                  # start import-osm /bin/bash terminal (imposm3)"
-	@echo "  make clean-docker                    # remove docker containers, PG data volume"
-	@echo "  make forced-clean-sql                # drop all PostgreSQL tables for clean environment"
-	@echo "  make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)"
-	@echo "  make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB"
-	@echo "  make remove-docker-images            # remove openmaptiles docker images"
-	@echo "  make pgclimb-list-views              # list PostgreSQL public schema views"
-	@echo "  make pgclimb-list-tables             # list PostgreSQL public schema tables"
-	@echo "  cat  .env                            # list PG database and MIN_ZOOM and MAX_ZOOM information"
-	@echo "  cat  quickstart.log                  # backup of the last ./quickstart.sh"
-	@echo "  make help                            # help about available commands"
-	@echo "=============================================================================="
+	@echo "$$HELP_MESSAGE" | less
 
-.PHONY: build
-build:
-	mkdir -p build
+define win_fs_error
+	( \
+	echo "" ;\
+	echo "ERROR: Windows native filesystem" ;\
+	echo "" ;\
+	echo "Please avoid running OpenMapTiles in a Windows filesystem." ;\
+	echo "See https://github.com/openmaptiles/openmaptiles/issues/1095#issuecomment-817095465" ;\
+	echo "" ;\
+	exit 1 ;\
+	)
+endef
 
-build/openmaptiles.tm2source/data.yml: build
-	mkdir -p build/openmaptiles.tm2source
-	docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml
+.PHONY: init-dirs
+init-dirs:
+	@mkdir -p build/sql/parallel
+	@mkdir -p build/openmaptiles.tm2source
+	@mkdir -p build/style
+	@mkdir -p data
+	@mkdir -p cache
+	@ ! ($(DOCKER_COMPOSE) 2>/dev/null run $(DC_OPTS) openmaptiles-tools df --output=fstype /tileset| grep -q 9p) < /dev/null || ($(win_fs_error))
 
-build/mapping.yaml: build
-	docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > build/mapping.yaml
+build/openmaptiles.tm2source/data.yml: init-dirs
+ifeq (,$(wildcard build/openmaptiles.tm2source/data.yml))
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
+		'generate-tm2source $(TILESET_FILE) > $@'
+endif
 
-build/tileset.sql: build
-	docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql
+build/mapping.yaml: init-dirs
+ifeq (,$(wildcard build/mapping.yaml))
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
+		'generate-imposm3 $(TILESET_FILE) > $@'
+endif
+
+.PHONY: build-sql
+build-sql: init-dirs
+ifeq (,$(wildcard build/sql/run_last.sql))
+	@mkdir -p build/sql/parallel
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
+		'generate-sql $(TILESET_FILE) --dir ./build/sql \
+		&& generate-sqltomvt $(TILESET_FILE) \
+							 --key --gzip --postgis-ver 3.3.4 \
+							 --function --fname=getmvt >> ./build/sql/run_last.sql'
+endif
+
+.PHONY: build-sprite
+build-sprite: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c 'spreet /style/icons build/style/sprite && \
+		spreet --retina /style/icons build/style/sprite@2x'
+
+.PHONY: build-style
+build-style: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c 'style-tools recompose $(TILESET_FILE) $(STYLE_FILE) \
+		$(STYLE_HEADER_FILE) && \
+		spreet /style/icons build/style/sprite && spreet --retina /style/icons build/style/sprite@2x'
+
+.PHONY: download-fonts
+download-fonts:
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c '[ ! -d "/export/fonts" ] && mkdir /export/fonts && \
+		echo "Downloading fonts..." && wget -qO /export/noto-sans.zip --show-progress \
+		https://github.com/openmaptiles/fonts/releases/download/v2.0/noto-sans.zip && \
+		echo "Unzipping fonts..." && unzip -q /export/noto-sans.zip -d /export/fonts && rm /export/noto-sans.zip || \
+		echo "Fonts already exist."'
 
 .PHONY: clean
-clean:
-	rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql
+clean: clean-test-data
+	rm -rf build
 
-.PHONY: clean-docker
-clean-docker:
-	docker-compose down -v --remove-orphans
-	docker-compose rm -fv
-	docker volume ls -q | grep openmaptiles  | xargs -r docker volume rm || true
+clean-test-data:
+	rm -rf data/changes.state.txt
+	rm -rf data/last.state.txt
+	rm -rf data/changes.repl.json
 
-.PHONY: db-start
-db-start:
-	docker-compose up -d postgres
+.PHONY: destroy-db
+DOCKER_PROJECT = $(shell echo $(DC_PROJECT) | tr A-Z a-z | tr -cd '[:alnum:]')
+destroy-db:
+	$(DOCKER_COMPOSE) down -v --remove-orphans
+	$(DOCKER_COMPOSE) rm -fv
+	docker volume ls -q -f "name=^$(DOCKER_PROJECT)_" | $(XARGS) docker volume rm
+	rm -rf cache
+	mkdir cache
+
+.PHONY: start-db-nowait
+start-db-nowait: init-dirs
+	@echo "Starting postgres docker compose target using $${POSTGIS_IMAGE:-default} image (no recreate if exists)" && \
+	$(DOCKER_COMPOSE) up --no-recreate -d postgres
+
+.PHONY: start-db
+start-db: start-db-nowait
 	@echo "Wait for PostgreSQL to start..."
-	docker-compose run $(DC_OPTS) import-osm  ./pgwait.sh
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools pgwait
 
-.PHONY: download-geofabrik
-download-geofabrik:
-	@echo ===============  download-geofabrik =======================
-	@echo Download area :   $(area)
-	@echo [[ example: make download-geofabrik  area=albania ]]
-	@echo [[ list areas:  make download-geofabrik-list       ]]
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik.sh $(area)
-	ls -la ./data/$(area).*
-	@echo "Generated config file: ./data/docker-compose-config.yml"
-	@echo " "
-	cat ./data/docker-compose-config.yml
-	@echo " "
+# Wrap start-db target but use the preloaded image
+.PHONY: start-db-preloaded
+start-db-preloaded: export POSTGIS_IMAGE=openmaptiles/postgis-preloaded
+start-db-preloaded: export COMPOSE_HTTP_TIMEOUT=180
+start-db-preloaded: start-db
+
+.PHONY: stop-db
+stop-db:
+	@echo "Stopping PostgreSQL..."
+	$(DOCKER_COMPOSE) stop postgres
+
+.PHONY: list-geofabrik
+list-geofabrik: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik
+
+.PHONY: list-bbbike
+list-bbbike: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike
+
+#
+# download, download-geofabrik, download-osmfr, and download-bbbike are handled here
+# The --imposm-cfg will fail for some of the sources, but we ignore that error -- only needed for diff mode
+#
+OSM_SERVERS := geofabrik osmfr bbbike
+ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) download
+OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@))
+.PHONY: $(ALL_DOWNLOADS)
+$(ALL_DOWNLOADS): init-dirs
+	@$(assert_area_is_given)
+ifneq ($(url),)
+	$(if $(OSM_SERVER),$(error url parameter can only be used with non-specific download target:$(newline)       make download area=$(area) url="$(url)"$(newline)))
+endif
+ifeq (,$(wildcard $(PBF_FILE)))
+ ifeq ($(DIFF_MODE),true)
+	@echo "Downloading $(DOWNLOAD_AREA) with replication support into $(PBF_FILE) and $(IMPOSM_CONFIG_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--imposm-cfg "$(IMPOSM_CONFIG_FILE)" \
+				--bbox "$(AREA_BBOX_FILE)" \
+				--output "$(PBF_FILE)"
+ else
+	@echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--bbox "$(AREA_BBOX_FILE)" \
+				--output "$(PBF_FILE)"
+ endif
+	@echo ""
+else
+ ifeq ($(DIFF_MODE),true)
+  ifeq (,$(wildcard $(IMPOSM_CONFIG_FILE)))
+	$(error \
+		$(newline)   Data files $(PBF_FILE) already exists, but $(IMPOSM_CONFIG_FILE) does not. \
+		$(newline)   You probably downloaded the data file before setting DIFF_MODE=true. \
+		$(newline)   You can delete the data file  $(PBF_FILE) and re-run  make download \
+		$(newline)   to re-download and generate config, or manually create  $(IMPOSM_CONFIG_FILE) \
+		$(newline)   See example    https://github.com/openmaptiles/openmaptiles-tools/blob/v5.2/bin/config/repl_config.json \
+		$(newline))
+  else
+	@echo "Data files $(PBF_FILE) and replication config $(IMPOSM_CONFIG_FILE) already exists, skipping the download."
+  endif
+ else
+	@echo "Data files $(PBF_FILE) already exists, skipping the download."
+ endif
+endif
+
+.PHONY: generate-bbox-file
+generate-bbox-file:
+	@$(assert_area_is_given)
+ifeq (,$(wildcard $(AREA_BBOX_FILE)))
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm bbox "$(PBF_FILE)" "$(AREA_BBOX_FILE)"
+else
+	@echo "Configuration file $(AREA_BBOX_FILE) already exists, no need to regenerate.  BBOX=$(BBOX)"
+endif
 
 .PHONY: psql
-psql: db-start
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh
+psql: start-db-nowait
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && psql.sh'
+
+# Special cache handling for Docker Toolbox on Windows
+ifeq ($(MSYSTEM),MINGW64)
+  DC_CONFIG_CACHE := -f docker-compose.yml -f docker-compose-$(MSYSTEM).yml
+  DC_OPTS_CACHE := $(filter-out --user=%,$(DC_OPTS))
+else
+  DC_OPTS_CACHE := $(DC_OPTS)
+endif
 
 .PHONY: import-osm
-import-osm: db-start all
-	docker-compose run $(DC_OPTS) import-osm
+import-osm: all start-db-nowait
+	@$(assert_area_is_given)
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)'
+
+.PHONY: start-update-osm
+start-update-osm: start-db
+	@$(assert_area_is_given)
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) up -d update-osm
+
+.PHONY: stop-update-osm
+stop-update-osm:
+	$(DOCKER_COMPOSE) stop update-osm
+
+.PHONY: import-diff
+import-diff: start-db-nowait
+	@$(assert_area_is_given)
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-diff'
+
+.PHONY: import-data
+import-data: start-db
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) import-data
 
 .PHONY: import-sql
-import-sql: db-start all
-	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
-
-.PHONY: import-osmsql
-import-osmsql: db-start all
-	docker-compose run $(DC_OPTS) import-osm
-	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
+import-sql: all start-db-nowait
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
+    	awk -v s=": WARNING:" '1{print; fflush()} $$0~s{print "\n*** WARNING detected, aborting"; exit(1)}' | \
+    	awk '1{print; fflush()} $$0~".*ERROR" {txt=$$0} END{ if(txt){print "\n*** ERROR detected, aborting:"; print txt; exit(1)} }'
 
 .PHONY: generate-tiles
-generate-tiles: db-start all
-	rm -rf data/tiles.mbtiles
-	if [ -f ./data/docker-compose-config.yml ]; then \
-		docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $(DC_OPTS) generate-vectortiles; \
-	else \
-		docker-compose run $(DC_OPTS) generate-vectortiles; \
+generate-tiles: all start-db
+	@echo "WARNING: This Mapnik-based method of tile generation is obsolete. Use generate-tiles-pg instead."
+	@echo "Generating tiles into $(MBTILES_LOCAL_FILE) (will delete if already exists)..."
+	@rm -rf "$(MBTILES_LOCAL_FILE)"
+	$(DOCKER_COMPOSE) run $(DC_OPTS) generate-vectortiles
+	@echo "Updating generated tile metadata ..."
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
+			mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" $(TILESET_FILE) --auto-minmax --show-ranges
+
+.PHONY: generate-tiles-pg
+generate-tiles-pg: all start-db
+	@echo "Generating tiles into $(MBTILES_LOCAL_FILE) (will delete if already exists) using PostGIS ST_MVT()..."
+	@rm -rf "$(MBTILES_LOCAL_FILE)"
+# For some reason Ctrl+C doesn't work here without the -T. Must be pressed twice to stop.
+	$(DOCKER_COMPOSE) run -T $(DC_OPTS) openmaptiles-tools generate-tiles
+	@echo "Updating generated tile metadata ..."
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
+			mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" $(TILESET_FILE) --auto-minmax --show-ranges
+
+.PHONY: data/tiles.txt
+data/tiles.txt:
+	find ./data -name "*.tiles" -exec cat {} \; -exec rm {} \; | \
+	  $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
+	    tile_multiplier $(MIN_ZOOM) $(MAX_ZOOM) >> data/tiles.txt
+
+.PHONY: generate-changed-tiles
+generate-changed-tiles: data/tiles.txt
+	# Re-generating updated tiles, if needed
+	if [ -s data/tiles.txt ] ; then \
+	  $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools refresh-views; \
+	  $(DOCKER_COMPOSE) run $(DC_OPTS) -e LIST_FILE=data/tiles.txt openmaptiles-tools generate-tiles; \
+	  rm data/tiles.txt; \
 	fi
-	docker-compose run $(DC_OPTS) openmaptiles-tools  generate-metadata ./data/tiles.mbtiles
-	docker-compose run $(DC_OPTS) openmaptiles-tools  chmod 666         ./data/tiles.mbtiles
 
 .PHONY: start-tileserver
-start-tileserver:
+start-tileserver: init-dirs build-style download-fonts
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
-	@echo "* Download/refresh klokantech/tileserver-gl docker image"
-	@echo "* see documentation: https://github.com/klokantech/tileserver-gl"
+	@echo "* Download/refresh maptiler/tileserver-gl docker image"
+	@echo "* see documentation: https://github.com/maptiler/tileserver-gl"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
-	docker pull klokantech/tileserver-gl
+	$(DOCKER_COMPOSE_COMMAND) pull tileserver-gl
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
-	@echo "* Start klokantech/tileserver-gl "
-	@echo "*       ----------------------------> check localhost:8080 "
+	@echo "* Start maptiler/tileserver-gl "
+	@echo "*       ----------------------------> check $(OMT_HOST):$(TPORT) "
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
-	docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl
+	$(DOCKER_COMPOSE) up -d tileserver-gl
+
+.PHONY: stop-tileserver
+stop-tileserver:
+	$(DOCKER_COMPOSE) stop tileserver-gl
 
 .PHONY: start-postserve
-start-postserve: db-start
+start-postserve: start-db
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
-	@echo "* Bring up postserve at localhost:8090"
+	@echo "* Bring up postserve at $(OMT_HOST):$(PPORT)"
+	@echo "*     --> can view it locally (use make start-maputnik)"
+	@echo "*     --> or can use https://maputnik.github.io/editor"
+	@echo "* "
+	@echo "*  set data source / TileJSON URL to $(OMT_HOST):$(PPORT)"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
-	docker-compose up -d postserve
-	docker pull maputnik/editor
+	$(DOCKER_COMPOSE) up -d postserve
+
+.PHONY: stop-postserve
+stop-postserve:
+	$(DOCKER_COMPOSE) stop postserve
+
+.PHONY: start-maputnik
+start-maputnik: stop-maputnik start-postserve
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
 	@echo "* Start maputnik/editor "
-	@echo "*       ---> go to http://localhost:8088"
-	@echo "*       ---> set 'data source' to  http://localhost:8090"
+	@echo "*       ---> go to $(OMT_HOST):8088 "
+	@echo "*       ---> set data source / TileJSON URL to $(OMT_HOST):$(PPORT)"
 	@echo "* "
 	@echo "***********************************************************"
 	@echo " "
-	docker rm -f maputnik_editor || true
-	docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor
+	$(DOCKER_COMPOSE) up -d maputnik_editor
 
-.PHONY: generate-qareports
-generate-qareports:
-	./qa/run.sh
+.PHONY: stop-maputnik
+stop-maputnik:
+	-$(DOCKER_COMPOSE) stop maputnik_editor
 
-build/devdoc:
-	mkdir -p ./build/devdoc
-
-
-layers = $(notdir $(wildcard layers/*)) # all layers
-
-.PHONY: etl-graph
-etl-graph:
-	@echo 'Use'
-	@echo '   make etl-graph-[layer]	to generate etl graph for [layer]'
-	@echo '   example: make etl-graph-poi'
-	@echo 'Valid layers: $(layers)'
-
-# generate etl graph for a certain layer, e.g. etl-graph-building, etl-graph-place
-etl-graph-%: layers/% build/devdoc
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-etlgraph layers/$*/$*.yaml ./build/devdoc
-	@$(COPY_TO_GIT) ./build/devdoc/etl_$*.png layers/$*/etl_diagram.png
-
-
-mappingLayers = $(notdir $(patsubst %/mapping.yaml,%, $(wildcard layers/*/mapping.yaml))) # layers with mapping.yaml
-
-# generate mapping graph for a certain layer, e.g. mapping-graph-building, mapping-graph-place
-.PHONY: mapping-graph
-mapping-graph:
-	@echo 'Use'
-	@echo '   make mapping-graph-[layer]	to generate mapping graph for [layer]'
-	@echo '   example: make mapping-graph-poi'
-	@echo 'Valid layers: $(mappingLayers)'
-
-mapping-graph-%: ./layers/%/mapping.yaml build/devdoc
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
-	@$(COPY_TO_GIT) ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png
+# STAT_FUNCTION=frequency|toplength|variance
+.PHONY: generate-qa
+generate-qa: all start-db-nowait
+	@echo " "
+	@echo "e.g. make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class"
+	@echo " "
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
+		layer-stats $(STAT_FUNCTION) $(TILESET_FILE) $(LAYER) $(ATTRIBUTE) -m 0 -n 14 -v
 
 # generate all etl and mapping graphs
-generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(mappingLayers))
+.PHONY: generate-devdoc
+generate-devdoc: init-dirs
+	mkdir -p ./build/devdoc && \
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
+			'generate-etlgraph $(TILESET_FILE) $(GRAPH_PARAMS) && \
+			 generate-mapping-graph $(TILESET_FILE) $(GRAPH_PARAMS)'
 
-.PHONY: import-sql-dev
-import-sql-dev:
-	docker-compose run $(DC_OPTS) openmaptiles-tools bash
+.PHONY: bash
+bash: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash
 
-.PHONY: import-osm-dev
-import-osm-dev:
-	docker-compose run $(DC_OPTS) import-osm /bin/bash
+.PHONY: import-wikidata
+import-wikidata: init-dirs
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json $(TILESET_FILE)
 
-# the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command
-.PHONY: list
-list:
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
+.PHONY: reset-db-stats
+reset-db-stats: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();'
 
-# same as a `make list`
-.PHONY: download-geofabrik-list
-download-geofabrik-list:
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
+.PHONY: list-views
+list-views: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+		-c "select viewname from pg_views where schemaname='public' order by viewname;"
 
-.PHONY: download-wikidata
-download-wikidata:
-	mkdir -p wikidata && docker-compose run $(DC_OPTS) --entrypoint /usr/src/app/download-gz.sh import-wikidata
+.PHONY: list-tables
+list-tables: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+		-c "select tablename from pg_tables where schemaname='public' order by tablename;"
 
 .PHONY: psql-list-tables
-psql-list-tables:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c "\d+"
+psql-list-tables: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+"
 
-.PHONY: psql-pg-stat-reset
-psql-pg-stat-reset:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'SELECT pg_stat_statements_reset();'
-
-.PHONY: forced-clean-sql
-forced-clean-sql:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; "
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';"
-
-.PHONY: pgclimb-list-views
-pgclimb-list-views:
-	docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv
-
-.PHONY: pgclimb-list-tables
-pgclimb-list-tables:
-	docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv
-
-.PHONY: psql-vacuum-analyze
-psql-vacuum-analyze:
+.PHONY: vacuum-db
+vacuum-db: init-dirs
 	@echo "Start - postgresql: VACUUM ANALYZE VERBOSE;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'VACUUM ANALYZE VERBOSE;'
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;'
 
-.PHONY: psql-analyze
-psql-analyze:
-	@echo "Start - postgresql: ANALYZE VERBOSE ;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'ANALYZE VERBOSE;'
+.PHONY: analyze-db
+analyze-db: init-dirs
+	@echo "Start - postgresql: ANALYZE VERBOSE;"
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;'
 
 .PHONY: list-docker-images
 list-docker-images:
 	docker images | grep openmaptiles
 
 .PHONY: refresh-docker-images
-refresh-docker-images:
-	docker-compose pull --ignore-pull-failures
+refresh-docker-images: init-dirs
+ifneq ($(NO_REFRESH),)
+	@echo "Skipping docker image refresh"
+else
+	@echo ""
+	@echo "Refreshing docker images... Use NO_REFRESH=1 to skip."
+ifneq ($(USE_PRELOADED_IMAGE),)
+	POSTGIS_IMAGE=openmaptiles/postgis-preloaded \
+		$(DOCKER_COMPOSE_COMMAND) pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres
+else
+	$(DOCKER_COMPOSE_COMMAND) pull --ignore-pull-failures $(QUIET_FLAG) openmaptiles-tools generate-vectortiles postgres import-data
+endif
+endif
 
 .PHONY: remove-docker-images
 remove-docker-images:
 	@echo "Deleting all openmaptiles related docker image(s)..."
-	@docker-compose down
-	@docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f
-	@docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f
-	@docker images | grep "klokantech/tileserver-gl"      | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f
+	@$(DOCKER_COMPOSE) down
+	@docker images "openmaptiles/*" -q                | $(XARGS) docker rmi -f
+	@docker images "maputnik/editor" -q               | $(XARGS) docker rmi -f
+	@docker images "maptiler/tileserver-gl" -q        | $(XARGS) docker rmi -f
 
-.PHONY: docker-unnecessary-clean
-docker-unnecessary-clean:
+.PHONY: clean-unnecessary-docker
+clean-unnecessary-docker:
 	@echo "Deleting unnecessary container(s)..."
-	@docker ps -a  | grep Exited | awk -F" " '{print $$1}' | xargs  --no-run-if-empty docker rm
+	@docker ps -a -q --filter "status=exited" | $(XARGS) docker rm
 	@echo "Deleting unnecessary image(s)..."
-	@docker images | grep \<none\> | awk -F" " '{print $$3}' | xargs  --no-run-if-empty  docker rmi
+	@docker images | awk -F" " '/<none>/{print $$3}' | $(XARGS) docker rmi
+
+.PHONY: test-perf-null
+test-perf-null: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools test-perf $(TILESET_FILE) --test null --no-color
+
+.PHONY: build-test-pbf
+build-test-pbf: init-dirs
+	$(DOCKER_COMPOSE_COMMAND) run $(DC_OPTS) openmaptiles-tools /tileset/.github/workflows/build-test-data.sh
+
+.PHONY: debug
+debug:  ## Use this target when developing Makefile itself to verify loaded environment variables
+	@$(assert_area_is_given)
+	@echo file_exists = $(wildcard $(AREA_BBOX_FILE))
+	@echo AREA_BBOX_FILE = $(AREA_BBOX_FILE) , $$AREA_ENV_FILE
+	@echo BBOX = $(BBOX) , $$BBOX
+	@echo MIN_ZOOM = $(MIN_ZOOM) , $$MIN_ZOOM
+	@echo MAX_ZOOM = $(MAX_ZOOM) , $$MAX_ZOOM
+
+build/import-tests.osm.pbf: init-dirs
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'osmconvert tests/import/*.osm -o=build/import-tests.osm.pbf'
+
+data/changes.state.txt:
+	cp -f tests/changes.state.txt data/
+
+data/last.state.txt:
+	cp -f tests/last.state.txt data/
+
+data/changes.repl.json:
+	cp -f tests/changes.repl.json data/
+
+data/changes.osc.gz: init-dirs
+	@echo " UPDATE unit test data..."
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'osmconvert tests/update/*.osc --merge-versions -o=data/changes.osc && gzip -f data/changes.osc'
+
+test-sql: clean refresh-docker-images destroy-db start-db-nowait build/import-tests.osm.pbf data/changes.state.txt data/last.state.txt data/changes.repl.json build/mapping.yaml data/changes.osc.gz build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
+	$(eval area := changes)
+
+	@echo "Load IMPORT test data"
+	sed -ir "s/^[#]*\s*MAX_ZOOM=.*/MAX_ZOOM=14/" .env
+	sed -ir "s/^[#]*\s*DIFF_MODE=.*/DIFF_MODE=false/" .env
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-osm build/import-tests.osm.pbf'
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) import-data
+
+	@echo "Apply OpenMapTiles SQL schema to test data @ Zoom 14..."
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
+    	awk -v s=": WARNING:" '1{print; fflush()} $$0~s{print "\n*** WARNING detected, aborting"; exit(1)}' | \
+    	awk '1{print; fflush()} $$0~".*ERROR" {txt=$$0} END{ if(txt){print "\n*** ERROR detected, aborting:"; print txt; exit(1)} }'
+
+	@echo "Test SQL output for Import Test Data"
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && psql.sh < tests/test-post-import.sql' 2>&1 | \
+		awk -v s="ERROR:" '1{print; fflush()} $$0~s{print "*** ERROR detected, aborting"; exit(1)}'
+
+	@echo "Run UPDATE process on test data..."
+	sed -ir "s/^[#]*\s*DIFF_MODE=.*/DIFF_MODE=true/" .env
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-diff'
+
+	@echo "Test SQL output for Update Test Data"
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && psql.sh < tests/test-post-update.sql' 2>&1 | \
+		awk -v s="ERROR:" '1{print; fflush()} $$0~s{print "*** ERROR detected, aborting"; exit(1)}'

QUICKSTART.md

@@ -2,7 +2,7 @@
 
 ### Req:
 * CPU: AMD64 ( = Intel 64 bit)
-    *  The base docker debian images are x86_64 based, so the ARM,MIPS currently not supported!
+    *  The base docker debian images are x86_64 based, so the ARM, MIPS currently not supported!
 * Operating system
     * Linux is suggested
         * The development and the testing platform is Linux.
@@ -59,7 +59,7 @@ If you have problems with the quickstart
 IF the previous step is working,
 THEN you can test other available quickstart extracts ( based on [Geofabrik extracts](http://download.geofabrik.de/index.html) ) !
  * We are using https://github.com/julien-noblet/download-geofabrik tool
- * The current extract list, and more information  ->  `make list`
+ * The current extract list, and more information  ->  `make list-geofabrik` or `make list-bbbike`
 
 This is generating `.mbtiles` for your area :  [ MIN_ZOOM: "0"  - MAX_ZOOM: "7" ]
 
@@ -363,12 +363,13 @@ This is generating `.mbtiles` for your area :  [ MIN_ZOOM: "0"  - MAX_ZOOM: "7"
 ./quickstart.sh yukon                        # Yukon, Canada
 ```
 ### Using your own OSM data
-Mbtiles can be generated from an arbitrary osm.pbf (e.g. for a region that is not covered by an existing extract) by making the `data/` directory and placing an osm.pbf inside/
+Mbtiles can be generated from an arbitrary osm.pbf (e.g. for a region that is not covered by an existing extract) by making the `data/` directory and placing an *.osm.pbf (e.g. `mydata.osm.pbf`) inside.
 
 ```
 mkdir -p data
-mv my.osm.pbf data/
-./quickstart.sh my
+mv mydata.osm.pbf data/
+make generate-bbox-file area=mydata
+./quickstart.sh mydata
 ```
 
 ### Check postserve
@@ -384,20 +385,26 @@ and the generated maps are going to be available in webbrowser on [localhost:808
 This is only a quick preview, because your mbtiles only generated to zoom level 7 !
 
 
-### Change MIN_ZOOM and MAX_ZOOM
+### Set which zooms to generate
 
-modify the settings in the `.env`  file, the defaults :
-* QUICKSTART_MIN_ZOOM=0
-* QUICKSTART_MAX_ZOOM=7
-
-and re-start  `./quickstart.sh `
-*  the new config file re-generating to here  ./data/docker-compose-config.yml
-*  Known problems:
-    * If you use same area - then the ./data/docker-compose-config.yml not re-generating, so you have to modify by hand!
+modify the settings in the `.env` file, the defaults:
+* `MIN_ZOOM=0`
+* `MAX_ZOOM=7`
 
 Hints:
-* Small increments! Never starts with the MAX_ZOOM = 14
-* The suggested  MAX_ZOOM = 14  - use only with small extracts
+* Small increments! Never starts with the `MAX_ZOOM = 14`
+* The suggested  `MAX_ZOOM = 14`  - use only with small extracts
+
+### Set the bounding box to generate
+
+By default, tile generation is done for the full extent of the area.
+If you want to generate a tiles for a smaller extent, modify the settings in the `.env` file, the default:
+* `BBOX=-180.0,-85.0511,180.0,85.0511`
+
+Delete the `./data/<area>.bbox` file, and re-start `./quickstart.sh <area>`
+
+Hint:
+* The [boundingbox.klokantech.com](https://boundingbox.klokantech.com/) site can be used to find a bounding box (CSV format) using a map.
 
 ### Check other commands
 
@@ -408,36 +415,63 @@ the current output:
 
 ```
 ==============================================================================
- OpenMapTiles  https://github.com/openmaptiles/openmaptiles
+OpenMapTiles  https://github.com/openmaptiles/openmaptiles
+
 Hints for testing areas
-  make download-geofabrik-list         # list actual geofabrik OSM extracts for download -> <<your-area>>
-  make list                            # list actual geofabrik OSM extracts for download -> <<your-area>>
+  make list-geofabrik                  # list actual geofabrik OSM extracts for download -> <<your-area>>
   ./quickstart.sh <<your-area>>        # example:  ./quickstart.sh madagascar
 
 Hints for designers:
-  make start-postserve                 # start Postserver + Maputnik Editor [ see localhost:8088 ]
-  make start-tileserver                # start klokantech/tileserver-gl [ see localhost:8080 ]
+  make start-maputnik                  # start Maputnik Editor + dynamic tile server [ see http://localhost:8088 ]
+  make stop-maputnik                   # stop Maputnik Editor + dynamic tile server
+  make start-postserve                 # start dynamic tile server                   [ see http://localhost:8090 ]
+  make stop-postserve                  # stop dynamic tile server
+  make start-tileserver                # start maptiler/tileserver-gl                [ see http://localhost:8080 ]
+  make stop-tileserver                 # stop maptiler/tileserver-gl
 
 Hints for developers:
   make                                 # build source code
-  make download-geofabrik area=albania # download OSM data from geofabrik, and create config file
+  make bash                            # start openmaptiles-tools /bin/bash terminal
+  make generate-bbox-file              # compute bounding box of a data file and store it in a file
+  make generate-devdoc                 # generate devdoc including graphs for all layers [./layers/...]
+  make generate-qa                     # statistics for a given layer's field
+  make generate-tiles-pg               # generate vector tiles based on .env settings using PostGIS ST_MVT()
+  make generate-tiles                  # generate vector tiles based on .env settings using Mapnik (obsolete)
+  make generate-changed-tiles          # Generate tiles changed by import-diff
+  make test-sql                        # run unit tests on the OpenMapTiles SQL schema
+  cat  .env                            # list PG database and MIN_ZOOM and MAX_ZOOM information
+  cat  quickstart.log                  # transcript of the last ./quickstart.sh run
+  make help                            # help about available commands
+
+Hints for downloading & importing data:
+  make list-geofabrik                  # list actual geofabrik OSM extracts for download
+  make list-bbbike                     # list actual BBBike OSM extracts for download
+  make download area=albania           # download OSM data from any source       and create config file
+  make download-geofabrik area=albania # download OSM data from geofabrik.de     and create config file
+  make download-osmfr area=asia/qatar  # download OSM data from openstreetmap.fr and create config file
+  make download-bbbike area=Amsterdam  # download OSM data from bbbike.org       and create config file
+  make import-data                     # Import data from OpenStreetMapData, Natural Earth and OSM Lake Labels.
+  make import-osm                      # Import OSM data with the mapping rules from build/mapping.yaml
+  make import-diff                     # Import OSM updates from data/changes.osc.gz
+  make import-wikidata                 # Import labels from Wikidata
+  make import-sql                      # Import layers (run this after modifying layer SQL)
+
+Hints for database management:
   make psql                            # start PostgreSQL console
   make psql-list-tables                # list all PostgreSQL tables
-  make psql-vacuum-analyze             # PostgreSQL: VACUUM ANALYZE
-  make psql-analyze                    # PostgreSQL: ANALYZE
-  make generate-qareports              # generate reports [./build/qareports]
-  make generate-devdoc                 # generate devdoc  [./build/devdoc]
-  make import-sql-dev                  # start import-sql  /bin/bash terminal
-  make import-osm-dev                  # start import-osm  /bin/bash terminal (imposm3)
-  make clean-docker                    # remove docker containers, PG data volume
-  make forced-clean-sql                # drop all PostgreSQL tables for clean environment
-  make docker-unnecessary-clean        # clean unnecessary docker image(s) and container(s)
+  make list-views                      # list PostgreSQL public schema views
+  make list-tables                     # list PostgreSQL public schema tables
+  make vacuum-db                       # PostgreSQL: VACUUM ANALYZE
+  make analyze-db                      # PostgreSQL: ANALYZE
+  make destroy-db                      # remove docker containers and PostgreSQL data volume
+  make start-db                        # start PostgreSQL, creating it if it doesn't exist
+  make start-db-preloaded              # start PostgreSQL, creating data-prepopulated one if it doesn't exist
+  make stop-db                         # stop PostgreSQL database without destroying the data
+
+Hints for Docker management:
+  make clean-unnecessary-docker        # clean unnecessary docker image(s) and container(s)
   make refresh-docker-images           # refresh openmaptiles docker images from Docker HUB
   make remove-docker-images            # remove openmaptiles docker images
-  make pgclimb-list-views              # list PostgreSQL public schema views
-  make pgclimb-list-tables             # list PostgreSQL public schema tables
-  cat  .env                            # list PG database and MIN_ZOOM and MAX_ZOOM information
-  cat ./quickstart.log                 # backup  of the last ./quickstart.sh
-  make help                            # help about available commands
+  make list-docker-images              # show a list of available docker images
 ==============================================================================
 ```

README.md

@@ -1,6 +1,6 @@
-## OpenMapTiles [![Build Status](https://travis-ci.org/openmaptiles/openmaptiles.svg?branch=master)](https://travis-ci.org/openmaptiles/openmaptiles)
+## OpenMapTiles [![Build Status](https://github.com/openmaptiles/openmaptiles/workflows/OpenMapTiles%20Integrity%20CI/badge.svg?branch=master)](https://github.com/openmaptiles/openmaptiles/actions)
 
-OpenMapTiles is an extensible and open tile schema based on the OpenStreetMap. This project is used to generate vector tiles for online zoomable maps. OpenMapTiles is about creating a beautiful basemaps with general layers containing topographic information. More information [openmaptiles.org](https://openmaptiles.org/) and [openmaptiles.com](https://openmaptiles.com/).
+OpenMapTiles is an extensible and open tile schema based on the OpenStreetMap. This project is used to generate vector tiles for online zoomable maps. OpenMapTiles is about creating a beautiful basemaps with general layers containing topographic information. More information [openmaptiles.org](https://openmaptiles.org/) and [maptiler.com/data/](https://www.maptiler.com/data/).
 
 We encourage you to collaborate, reuse and adapt existing layers, or add your own layers. You may use our approach for your own vector tile project. Feel free to fork the repo and experiment. The repository is built on top of the [openmaptiles/openmaptiles-tools](https://github.com/openmaptiles/openmaptiles-tools) to simplify vector tile creation.
 
@@ -8,10 +8,11 @@ Please keep in mind that OpenMapTiles schema should display general topographic
 
 - :link: Schema https://openmaptiles.org/schema
 - :link: Docs https://openmaptiles.org/docs
-- :link: Production package: https://openmaptiles.com/production-package/
+- :link: Data for download: https://www.maptiler.com/data/
 - :link: Hosting https://www.maptiler.com/cloud/
 - :link: Create own layer https://github.com/openmaptiles/openmaptiles-skiing
-- :link: Discuss at the #openmaptiles channel at [OSM Slack](https://osmus-slack.herokuapp.com/)
+- :link: Practical usage of OpenMapTiles https://github.com/maptiler/foss4g-workshop
+- :link: Discuss at the #openmaptiles channel at [OSM Slack](https://slack.openstreetmap.us/)
 
 ## Styles
 
@@ -20,14 +21,15 @@ You can start from several GL styles supporting the OpenMapTiles vector schema.
 :link: [Learn how to create Mapbox GL styles with Maputnik and OpenMapTiles](http://openmaptiles.org/docs/style/maputnik/).
 
 
+- [OSM OpenMapTiles](./style/README.md)
 - [OSM Bright](https://github.com/openmaptiles/osm-bright-gl-style)
+- [MapTiler Basic](https://github.com/openmaptiles/maptiler-basic-gl-style)
+- [MapTiler 3D](https://github.com/openmaptiles/maptiler-3d-gl-style)
+- [Fiord Color](https://github.com/openmaptiles/fiord-color-gl-style)
+- [MapTiler Toner](https://github.com/openmaptiles/maptiler-toner-gl-style)
+- [OSM Liberty](https://github.com/maputnik/osm-liberty)
 - [Positron](https://github.com/openmaptiles/positron-gl-style)
 - [Dark Matter](https://github.com/openmaptiles/dark-matter-gl-style)
-- [Klokantech Basic](https://github.com/openmaptiles/klokantech-basic-gl-style)
-- [Klokantech 3D](https://github.com/openmaptiles/klokantech-3d-gl-style)
-- [Fiord Color](https://github.com/openmaptiles/fiord-color-gl-style)
-- [Toner](https://github.com/openmaptiles/toner-gl-style)
-- [OSM Liberty](https://github.com/maputnik/osm-liberty)
 
 We also ported over our favorite old raster styles (TM2).
 
@@ -71,6 +73,10 @@ To work on OpenMapTiles you need Docker.
 - Install [Docker](https://docs.docker.com/engine/installation/). Minimum version is 1.12.3+.
 - Install [Docker Compose](https://docs.docker.com/compose/install/). Minimum version is 1.7.1+.
 
+### Microsoft Windows Subsystem for Linux (WSL)
+
+Please use Linux `/home/user/` directory, not Windows e.g. `/mnt/c` directory.
+
 ### Build
 
 Build the tileset.
@@ -83,10 +89,10 @@ make
 ```
 
 You can execute the following manual steps (for better understanding)
-or use the provided `quickstart.sh` script.
+or use the provided `quickstart.sh` script to automatically download and import given area. If area is not given, Albania will be imported. List of available areas `make list-geofabrik`.
 
 ```
-./quickstart.sh
+./quickstart.sh <area>
 ```
 
 ### Prepare the Database
@@ -94,43 +100,44 @@ or use the provided `quickstart.sh` script.
 Now start up the database container.
 
 ```bash
-docker-compose up -d postgres
+make start-db
 ```
 
-Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).
+Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/openmaptiles/osm-lakelines). Natural Earth country boundaries are used in the few lowest zoom levels.
 
 ```bash
-docker-compose run import-water
-docker-compose run import-natural-earth
-docker-compose run import-lakelines
-docker-compose run import-osmborder
+make import-data
 ```
 
-**[Optional]**
-Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field.
-
-Beware that current [Wikidata dump](https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.gz) is more than 55GB, it takes time to download and import it. If you just want to have a quickstart on OpenMapTiles, just skip this step.
+Download OpenStreetMap data extracts from any source like [Geofabrik](http://download.geofabrik.de/), and store the PBF file in the `./data` directory. To use a specific download source, use `download-geofabrik`, `download-bbbike`, or `download-osmfr`, or use `download` to make it auto-pick the area. You can use `area=planet` for the entire OSM dataset (very large).  Note that if you have more than one `data/*.osm.pbf` file, every `make` command will always require `area=...` parameter (or you can just `export area=...` first).
 
 ```bash
-make download-wikidata
-docker-compose run import-wikidata
+make download area=albania
 ```
 
-[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
+[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-osm) with the mapping rules from
+`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition (any change in `mapping.yaml`).
 
 ```bash
-cd data
-wget http://download.geofabrik.de/europe/albania-latest.osm.pbf
+make import-osm
 ```
 
-[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
-`build/mapping.yaml` (which has been created by `make`).
+Import labels from Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field.
+
+This step uses [Wikidata Query Service](https://query.wikidata.org) to download just the Wikidata IDs that already exist in the database.
 
 ```bash
-docker-compose run import-osm
+make import-wikidata
 ```
 
 ### Work on Layers
+Each time you modify a layer's `mapping.yaml` file or add new OSM tags, run `make` and `make import-osm` to recreate tables (potentially with additional data) in PostgreSQL. With the new data, there can be new Wikidata records also.
+```
+make clean
+make
+make import-osm
+make import-wikidata
+```
 
 Each time you modify layer SQL code run `make` and `make import-sql`.
 
@@ -140,18 +147,47 @@ make
 make import-sql
 ```
 
-Now you are ready to **generate the vector tiles**. Using environment variables
-you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`).
+Each time you make a modification that adds a new feature to vector tiles e.g. adding new OSM tags, modify the layer 
+style snippet by adding new style layer so the changes are propagated visually into the style.
+All new style layers must have the `order` value which determines the order or rendering in the map style. 
+After the layer style snippet is modified run:
+```bash
+make build-style
+```
+
+
+
+Now you are ready to **generate the vector tiles**. By default, `./.env` specifies the entire planet BBOX for zooms 0-7, but running `generate-bbox-file` will analyze the data file and set the `BBOX` param to limit tile generation.
 
 ```
-docker-compose run generate-vectortiles
+make generate-bbox-file  # compute data bbox -- not needed for the whole planet or for downloaded area by `make download`
+make generate-tiles-pg   # generate tiles
 ```
 
+### Workflow to generate tiles
+If you go from top to bottom you can be sure that it will generate a .mbtiles file out of a .osm.pbf file
+```
+make clean                  # clean / remove existing build files
+make                        # generate build files
+make start-db               # start up the database container.
+make import-data            # Import external data from OpenStreetMapData, Natural Earth and OpenStreetMap Lake Labels.
+make download area=albania  # download albania .osm.pbf file -- can be skipped if a .osm.pbf file already existing
+make import-osm             # import data into postgres
+make import-wikidata        # import Wikidata
+make import-sql             # create / import sql functions 
+make generate-bbox-file     # compute data bbox -- not needed for the whole planet or for downloaded area by `make download`
+make generate-tiles-pg      # generate tiles
+```
+Instead of calling `make download area=albania` you can add a .osm.pbf file in the `data` folder `openmaptiles/data/your_area_file.osm.pbf`
+
+To change the name of the output filename, you can modify the variable `MBTILES_FILE` in the `.env` file or set up the environment variable `MBTILES_FILE` before running `./quickstart.sh` or `make generate-tiles-pg` (e.g., `MBTILES_FILENAME=monaco.mbtiles ./quickstart.sh monaco`).
+
+
 ## License
 
-All code in this repository is under the [BSD license](./LICENSE.md) and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md).
+All code in this repository is under the [BSD license](./LICENSE.md). Design and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md).
 
-Products or services using maps derived from OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference "OpenMapTiles" with a link to https://openmaptiles.org/. Exceptions to attribution requirement can be granted on request.
+Products or services using maps derived from OpenMapTiles schema need to **visibly credit "OpenMapTiles.org"** or **reference "OpenMapTiles"** with a link to https://openmaptiles.org/. Exceptions to attribution requirement can be granted on request.
 
 For a browsable electronic map based on OpenMapTiles and OpenStreetMap data, the
 credit should appear in the corner of the map. For example:

TESTING.md

@@ -0,0 +1,18 @@
+
+# OpenMapTiles SQL Testing
+
+The OpenMapTiles SQL tests ensure that OSM data is properly imported and updated in the OpenMapTiles data schema.  The tests work by injecting test OSM data into the database and checking to ensure that the data is properly reflected in the SQL output. 
+
+Usage:
+
+`make clean && make test-sql`
+
+## How it works
+
+The SQL tests consist of the following parts:
+
+ 1. **Test import data**, located in `tests/import`.  This test data is in the [OSM XML](https://wiki.openstreetmap.org/wiki/OSM_XML) format and contains the data that should be initially injected into the database.  The files are numbered in order to ensure that each test data file OSM id numbers that are unique from the other files.  For example, the file starting with `100` will use node ids from 100000-199999, way ids from 1000-1999, and relation ids from 100-199.
+ 2. **Test update data**, located in `tests/update`.  This test data is in the [osmChange XML](https://wiki.openstreetmap.org/wiki/OsmChange) format, and contains the data that will be used to update the test import data (in order to verify that the update process is working correctly.  These files are also numbered using the same scheme as the test import data.
+ 3. **Import SQL test script**, located at `tests/test-post-import.sql`.  This script is executed after the test import data has been injected, and runs SQL-based checks to ensure that the import data was properly imported.  If there are failures in the tests, an entry will be added to the table `omt_test_failures`, with one record per error that occurs during the import process.  A test failure will also fail the build.  To inspect the test failure messages, run `make psql` and issue the comment `SELECT * FROM omt_test_failures`.
+ 4. **Update SQL test script**, located at `tests/test-post-update.sql`.  This script performs the same function as the import test script, except that it occurs after the test update data has been applied to the database.  Note that script will only run if the import script passes all tests.
+

UPDATE.md

@@ -1,38 +1,116 @@
-# Keep the vector tiles updated
+# Keeping the Vector Tiles Updated
 
 Once you have imported OpenMapTiles you can also keep it up to date by importing the latest OSM changes and
 regenerating the tables.
 
 ## Import
 
-You can either keep the database up to date based on the daily OSM change feed
+You can either keep the database up to date based on the daily (or minutely) OSM change feed
 or import specific change files.
 
+### Choosing the Download Source
+
+While GeoFabrik currently provides extracts of basically all countries, they provide only daily updates.
+If you need minutely updates you might want to try openstreetmap.fr, for example like this: `make download-osmfr area=africa/eritrea`, which configures minutely updates.
+
+### Preparations
+
+If you plan to keep data updated automatically, before importing any data, make sure to set
+
+```
+DIFF_MODE=true
+```
+
+in the `.env`
+
+Now download fresh data:
+
+```
+make download area=your-area-of-choice
+```
+
 ### Keep Database Updated
 
-You can use the new imposm3 feature to keep the database updated (thanks to the [work by @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)). This will automatically download
-the OSM change feed and import it into the database.
-After each run you should also have a list of tiles that have updated.
+You can use imposm3 to keep the database updated (thanks to the [work by @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)).
+This will repeatedly download the OSM change feed and import it into the database.
+In order to be able to update the database, the initial download and import of the OSM data must be done when `DIFF_MODE=true` is set in the `.env` file.
+In this mode the initial download also sets the update source and the update intervals.
+
+To start the update process please use
+```
+make start-update-osm
+```
+
+To stop the update process please use
+```
+make stop-update-osm
+```
+
+After each update activation, **imposm3** will store lists of updated tiles in text format in subfolders of the `diffdir`,
+named for the date(s) on which the import took place (`YYYYMMDD`).
+
+See [Generate Changed Tiles](#generate-changed-tiles) below on how this file can be used.
+
+#### Note
+When the update process is actively updating the DB it is impossible to successfully generate tiles,
+as there will be conflicts and deadlocks related to the DB access.
+
+Unfortunately, there is no known way to execute an external command in-between rounds of the `update-osm` process.
+
+#### Troubleshooting
+
+The log file for osm update can be viewed using
 
 ```
-docker-compose run update-osm
+docker-compose logs --tail 100 --follow update-osm
+```
+
+Use `Ctrl-C` to stop following the log.
+
+The output will be similar to this:
+
+```
+[info] Importing #4889572 including changes till ....... +0000 UTC (2h10m10s behind)
+```
+
+It might take some time to catch up with the latest changes, but the "time behind" should decrease until it is a few minutes.
+If it doesn't, you need to download a new extract or check that there are enough system resources to keep-up with the changes.
+
+Finally you will get an output like this - this indicates, that some 6 objects were changed:
+
+```
+[progress]     3s C:       0/s (0) N:       0/s (0) W:       0/s (6) R:      0/s (0)
+```
+
+The process will keep running foreverprint something like this - which just means that no changes were in the latest changeset:
+
+```
+[progress]     0s C:       0/s (0) N:       0/s (0) W:       0/s (0) R:      0/s (0)
 ```
 
 ### Import Change File
 
-Given you have a file `changes.osc.gz` in your import folder. Once you ran the import command you should also have a list of tiles that have updated.
+You may perform a one-time import of OSM changes from the `changes.osc.gz` file in your import folder using
 
 ```
-docker-compose run import-osm-diff
+make import-diff
 ```
 
+Similar to[Keep Database Updated](#keep_database_updated) above, **imposm3** will store the list of updated tiles in text file in subfolders of the `diffdir`,
+named for the date on which the import took place (`YYYYMMDD`).
+
+See [Generate Changed Tiles](#generate-changed-tiles) below.
+
+#### Note
+There is no `make` command for downloading OSM changes into `changes.osc.gz`.
+You may perform this task using [`osmupdate`](https://wiki.openstreetmap.org/wiki/Osmupdate),
+[pyosmium-get-changes](https://docs.osmcode.org/pyosmium/latest/tools_get_changes.html),
+or downloading the changefile directly from the replication server.
+
 ## Generate Changed Tiles
 
-After the import has finished **imposm3** will store a list of tiles in text format in the `diffdir`.
-Copy the as `tiles.txt` to the import folder.
-
-Now run the command to read the tilelist and write the vector tiles for it to a new MBTiles.
+To generate all changed tiles, based on the lists of all updated tiles, and update the existing MBtiles file, please use
 
 ```
-docker-compose run generate-changed-vectortiles
+make generate-changed-tiles
 ```

docker-compose-MINGW64.yml

@@ -0,0 +1,11 @@
+# This version must match the MAKE_DC_VERSION in docker-compose.yml
+version: "3"
+
+volumes:
+  cache:
+
+services:
+
+  openmaptiles-tools:
+    volumes:
+      - cache:/cache

docker-compose.yml

@@ -1,99 +1,76 @@
-version: "2"
+# This version must match the MAKE_DC_VERSION value below
+version: "3"
+
 volumes:
   pgdata:
-  cache:
-services:
+
+networks:
   postgres:
-    image: "openmaptiles/postgis:${TOOLS_VERSION}"
+    driver: bridge
+
+services:
+
+  postgres:
+    image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
+    # Use "command: postgres -c jit=off" for PostgreSQL 11+ because of slow large MVT query processing
+    # Use "shm_size: 512m" if you want to prevent a possible 'No space left on device' during 'make generate-tiles-pg'
     volumes:
       - pgdata:/var/lib/postgresql/data
     networks:
-      - postgres_conn
+      - postgres
     ports:
-      - "5432"
-    env_file: .env
-  import-natural-earth:
-    image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-water:
-    image: "openmaptiles/import-water:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-lakelines:
-    image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-osm:
-    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
+      - "${PGPORT:-5432}:${PGPORT:-5432}"
     env_file: .env
     environment:
-      DIFF_MODE: ${DIFF_MODE}
-    networks:
-      - postgres_conn
-    volumes:
-      - ./data:/import
-      - ./build:/mapping
-      - cache:/cache
-  import-osmborder:
-    image: "openmaptiles/import-osmborder:${TOOLS_VERSION}"
+      # postgress container uses old variable names
+      POSTGRES_DB: ${PGDATABASE:-openmaptiles}
+      POSTGRES_USER: ${PGUSER:-openmaptiles}
+      POSTGRES_PASSWORD: ${PGPASSWORD:-openmaptiles}
+      PGPORT: ${PGPORT:-5432}
+
+  import-data:
+    image: "openmaptiles/import-data:${TOOLS_VERSION}"
     env_file: .env
     networks:
-      - postgres_conn
-  import-osm-diff:
-    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
+      - postgres
+
+  openmaptiles-tools: &openmaptiles-tools
+    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
     env_file: .env
-    command: ./import_diff.sh
     environment:
+      # Must match the version of this file (first line)
+      # download-osm will use it when generating a composer file
+      MAKE_DC_VERSION: "3"
+      # Allow DIFF_MODE, MIN_ZOOM, and MAX_ZOOM to be overwritten from shell
       DIFF_MODE: ${DIFF_MODE}
+      MIN_ZOOM: ${MIN_ZOOM}
+      MAX_ZOOM: ${MAX_ZOOM}
+      #Provide BBOX from *.bbox file if exists, else from .env
+      BBOX: ${BBOX}
+      # Imposm configuration file describes how to load updates when enabled
+      IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE}
+      # Control import-sql processes
+      MAX_PARALLEL_PSQL: ${MAX_PARALLEL_PSQL}
+      PGDATABASE: ${PGDATABASE:-openmaptiles}
+      PGUSER: ${PGUSER:-openmaptiles}
+      PGPASSWORD: ${PGPASSWORD:-openmaptiles}
+      PGPORT: ${PGPORT:-5432}
+      MBTILES_FILE: ${MBTILES_FILE}
     networks:
-      - postgres_conn
+      - postgres
     volumes:
+      - .:/tileset
       - ./data:/import
+      - ./data:/export
+      - ./build/sql:/sql
       - ./build:/mapping
-      - cache:/cache
+      - ./cache:/cache
+      - ./style:/style
+
   update-osm:
-    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
-    env_file: .env
-    environment:
-      DIFF_MODE: ${DIFF_MODE}
-    command: ./import_update.sh
-    networks:
-      - postgres_conn
-    volumes:
-      - ./data:/import
-      - ./build:/mapping
-      - cache:/cache
-  import-sql:
-    # This target is obsolete, and was left for backwards compatibility
-    # Use openmaptiles-tools target instead
-    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
-    command: import-sql
-    env_file: .env
-    networks:
-      - postgres_conn
-    volumes:
-      - .:/tileset
-      - ./build:/sql
-  import-wikidata:
-    image: "openmaptiles/import-wikidata:${TOOLS_VERSION}"
-    env_file: .env
-    command: import-wikidata
-    networks:
-      - postgres_conn
-    volumes:
-      - ./wikidata:/import
-  openmaptiles-tools:
-    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-    volumes:
-      - .:/tileset
-      - ./build:/sql
+    <<: *openmaptiles-tools
+    command: import-update
+
   generate-changed-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
     command: ./export-list.sh
@@ -101,31 +78,66 @@ services:
       - ./data:/export
       - ./build/openmaptiles.tm2source:/tm2source
     networks:
-      - postgres_conn
+      - postgres
     env_file: .env
+    environment:
+      MBTILES_NAME: ${MBTILES_FILE}
+      # Control tilelive-copy threads
+      COPY_CONCURRENCY: ${COPY_CONCURRENCY}
+      PGDATABASE: ${PGDATABASE:-openmaptiles}
+      PGUSER: ${PGUSER:-openmaptiles}
+      PGPASSWORD: ${PGPASSWORD:-openmaptiles}
+      PGPORT: ${PGPORT:-5432}
+
   generate-vectortiles:
     image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
     volumes:
       - ./data:/export
       - ./build/openmaptiles.tm2source:/tm2source
     networks:
-      - postgres_conn
+      - postgres
     env_file: .env
     environment:
+      MBTILES_NAME: ${MBTILES_FILE}
       BBOX: ${BBOX}
       MIN_ZOOM: ${MIN_ZOOM}
       MAX_ZOOM: ${MAX_ZOOM}
+      # Control tilelive-copy threads
+      COPY_CONCURRENCY: ${COPY_CONCURRENCY}
+      #
+      PGDATABASE: ${PGDATABASE:-openmaptiles}
+      PGUSER: ${PGUSER:-openmaptiles}
+      PGPASSWORD: ${PGPASSWORD:-openmaptiles}
+      PGPORT: ${PGPORT:-5432}
+
   postserve:
     image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
-    command: postserve openmaptiles.yaml --verbose
+    command: "postserve ${TILESET_FILE} --verbose --serve=${OMT_HOST:-http://localhost}:${PPORT:-8090}"
     env_file: .env
+    environment:
+      TILESET_FILE: ${TILESET_FILE}
     networks:
-      - postgres_conn
+      - postgres
     ports:
-      - "8090:8090"
+      - "${PPORT:-8090}:${PPORT:-8090}"
     volumes:
       - .:/tileset
 
-networks:
-  postgres_conn:
-    driver: bridge
+  maputnik_editor:
+    image: "maputnik/editor"
+    ports:
+      - "8088:8888"
+
+  tileserver-gl:
+    image: "maptiler/tileserver-gl:latest"
+    command:
+      - --port
+      - "${TPORT:-8080}"
+      - --config
+      - "/style/config.json"
+    ports:
+      - "${TPORT:-8080}:${TPORT:-8080}"
+    volumes:
+      - ./data:/data
+      - ./style:/style
+      - ./build:/build

integrity.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# A script to run the "integrity" continuous integration script.
+
+area=monaco
+echo MIN_ZOOM=0 >> .env
+echo MAX_ZOOM=14 >> .env
+./quickstart.sh $area
+export TEST_MODE=yes
+make generate-devdoc
+area=europe/monaco
+echo DIFF_MODE=true >> .env
+
+# Cleanup
+rm -fr data build cache
+# Create data/$area.repl.json
+make download-geofabrik area=$area
+# Download 2+ month old data
+export old_date=$(date --date="$(date +%Y-%m-15) -2 month" +'%y%m01')
+echo Downloading $old_date extract of $area
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "wget -O data/$area.osm.pbf http://download.geofabrik.de/$area-$old_date.osm.pbf"
+# Initial import and tile generation
+./quickstart.sh $area
+sleep 2
+echo Downloading updates
+# Loop to recover from potential "ERROR 429: Too Many Requests"
+docker-compose run --rm --user=$(id -u):$(id -g) openmaptiles-tools sh -c "
+while ! osmupdate --keep-tempfiles --base-url=$(sed -n 's/ *\"replication_url\": //p' data/$area.repl.json) data/$area.osm.pbf data/changes.osc.gz ; do 
+  sleep 2;
+  echo Sleeping...;
+  sleep 630;
+done"
+echo Downloading updates completed
+echo Importing updates
+make import-diff
+echo Generating new tiles
+make generate-tiles-pg

layers/aerodrome_label/aerodrome_label.sql

@@ -0,0 +1,62 @@
+
+-- etldoc: layer_aerodrome_label[shape=record fillcolor=lightpink, style="rounded,filled", label="layer_aerodrome_label | <z8> z8 | <z9> z9 | <z10_> z10+" ] ;
+
+CREATE OR REPLACE FUNCTION layer_aerodrome_label(bbox geometry,
+                                                 zoom_level integer)
+    RETURNS TABLE
+            (
+                id       bigint,
+                geometry geometry,
+                name     text,
+                name_en  text,
+                name_de  text,
+                tags     hstore,
+                class    text,
+                iata     text,
+                icao     text,
+                ele      int,
+                ele_ft   int
+            )
+AS
+$$
+SELECT
+    -- etldoc: osm_aerodrome_label_point -> layer_aerodrome_label:z8
+    -- etldoc: osm_aerodrome_label_point -> layer_aerodrome_label:z9
+    ABS(osm_id) AS id, -- mvt feature IDs can't be negative
+    geometry,
+    name,
+    COALESCE(NULLIF(name_en, ''), name) AS name_en,
+    COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+    tags,
+    aerodrome_type AS class,
+    NULLIF(iata, '') AS iata,
+    NULLIF(icao, '') AS icao,
+    substring(ele FROM E'^(-?\\d+)(\\D|$)')::int AS ele,
+    round(substring(ele FROM E'^(-?\\d+)(\\D|$)')::int * 3.2808399)::int AS ele_ft
+FROM osm_aerodrome_label_point
+WHERE geometry && bbox
+  AND aerodrome_type = 'international'
+  AND iata <> ''
+  AND zoom_level BETWEEN 8 AND 9
+
+UNION ALL
+
+SELECT
+    -- etldoc: osm_aerodrome_label_point -> layer_aerodrome_label:z10_
+    ABS(osm_id) AS id, -- mvt feature IDs can't be negative
+    geometry,
+    name,
+    COALESCE(NULLIF(name_en, ''), name) AS name_en,
+    COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+    tags,
+    aerodrome_type AS class,
+    NULLIF(iata, '') AS iata,
+    NULLIF(icao, '') AS icao,
+    substring(ele FROM E'^(-?\\d+)(\\D|$)')::int AS ele,
+    round(substring(ele FROM E'^(-?\\d+)(\\D|$)')::int * 3.2808399)::int AS ele_ft
+FROM osm_aerodrome_label_point
+WHERE geometry && bbox
+  AND zoom_level >= 10;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/aerodrome_label/aerodrome_label.yaml

@@ -1,13 +1,13 @@
 layer:
-  id: "aerodrome_label"
+  id: aerodrome_label
   description: |
       [Aerodrome labels](http://wiki.openstreetmap.org/wiki/Tag:aeroway%3Daerodrome)
   buffer_size: 64
   srs: +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over
   fields:
-    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the aerodrome.
-    name_en: English name `name:en` if available, otherwise `name`.
-    name_de: German name `name:de` if available, otherwise `name` or `name:en`.
+    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the aerodrome. Language-specific values are in `name:xx`.
+    name_en: English name `name:en` if available, otherwise `name`. This is deprecated and will be removed in a future release in favor of `name:en`.
+    name_de: German name `name:de` if available, otherwise `name` or `name:en`. This is deprecated and will be removed in a future release in favor of `name:de`.
     class:
       description: |
           Distinguish between more and less important aerodromes.
@@ -15,25 +15,36 @@ layer:
           [`aerodrome`](http://wiki.openstreetmap.org/wiki/Proposed_features/Aerodrome)
           and `aerodrome:type` tags.
       values:
-      - international
-      - public
-      - regional
-      - military
-      - private
-      - other
+        international:
+          aerodrome: 'international'
+          aerodrome_type: 'international'
+        public:
+          aerodrome: 'public'
+          aerodrome_type: ['%public%', 'civil']
+        regional:
+          aerodrome: 'regional'
+          aerodrome_type: 'regional'
+        military:
+          aerodrome: 'military'
+          aerodrome_type: '%military%'
+          military: 'airfield'
+        private:
+          aerodrome: 'private'
+          aerodrome_type: 'private'
+        other:
     iata: 3-character code issued by the IATA.
     icao: 4-letter code issued by the ICAO.
     ele: Elevation (`ele`) in meters.
     ele_ft: Elevation (`ele`) in feets.
   datasource:
     geometry_field: geometry
-    key_field: osm_id
+    key_field: id
     key_field_as_attribute: no
     srid: 900913
-    query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, iata, icao, ele, ele_ft FROM layer_aerodrome_label (!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
+    query: (SELECT id, geometry, name, name_en, name_de, {name_languages}, class, iata, icao, ele, ele_ft FROM layer_aerodrome_label(!bbox!, z(!scale_denominator!))) AS t
 schema:
-  - ./update.sql
-  - ./layer.sql
+  - ./update_aerodrome_label_point.sql
+  - ./aerodrome_label.sql
 datasources:
   - type: imposm3
     mapping_file: ./mapping.yaml

layers/aerodrome_label/layer.sql

@@ -1,60 +0,0 @@
-
--- etldoc: layer_aerodrome_label[shape=record fillcolor=lightpink, style="rounded,filled", label="layer_aerodrome_label | <z10_> z10+" ] ;
-
-CREATE OR REPLACE FUNCTION layer_aerodrome_label(
-    bbox geometry,
-    zoom_level integer,
-    pixel_width numeric)
-  RETURNS TABLE(
-    osm_id bigint,
-    geometry geometry,
-    name text,
-    name_en text,
-    name_de text,
-    tags hstore,
-    class text,
-    iata text,
-    icao text,
-    ele int,
-    ele_ft int) AS
-$$
-  -- etldoc: osm_aerodrome_label_point -> layer_aerodrome_label:z10_
-  SELECT
-    osm_id,
-    geometry,
-    name,
-    COALESCE(NULLIF(name_en, ''), name) AS name_en,
-    COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-    tags,
-    CASE
-      WHEN aerodrome = 'international'
-          OR aerodrome_type = 'international'
-        THEN 'international'
-      WHEN
-          aerodrome = 'public'
-          OR aerodrome_type LIKE '%public%'
-          OR aerodrome_type = 'civil'
-        THEN 'public'
-      WHEN
-          aerodrome = 'regional'
-          OR aerodrome_type = 'regional'
-        THEN 'regional'
-      WHEN
-          aerodrome = 'military'
-          OR aerodrome_type LIKE '%military%'
-          OR military = 'airfield'
-        THEN 'military'
-      WHEN
-          aerodrome = 'private'
-          OR aerodrome_type = 'private'
-        THEN 'private'
-      ELSE 'other'
-    END AS class,
-    NULLIF(iata, '') AS iata,
-    NULLIF(icao, '') AS icao,
-    substring(ele from E'^(-?\\d+)(\\D|$)')::int AS ele,
-    round(substring(ele from E'^(-?\\d+)(\\D|$)')::int*3.2808399)::int AS ele_ft
-  FROM osm_aerodrome_label_point
-  WHERE geometry && bbox AND zoom_level >= 10;
-
-$$ LANGUAGE SQL IMMUTABLE;

layers/aerodrome_label/mapping.yaml

@@ -3,7 +3,7 @@ tables:
   # etldoc: imposm3 -> osm_aerodrome_label_point
   aerodrome_label_point:
     type: geometry
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/aerodrome_label/style.json

@@ -0,0 +1,69 @@
+{
+  "layers": [
+    {
+      "id": "airport-label-major",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "aerodrome_label",
+      "minzoom": 8,
+      "maxzoom": 17,
+      "layout": {
+        "icon-size": 1,
+        "text-font": [
+          "Noto Sans Italic"
+        ],
+        "text-size": {
+          "stops": [
+            [
+              8,
+              10
+            ],
+            [
+              14,
+              12
+            ]
+          ]
+        },
+        "icon-image": "aerodrome.12",
+        "text-field": {
+          "stops": [
+            [
+              8,
+              " "
+            ],
+            [
+              11,
+              "{name:latin}\n{name:nonlatin}"
+            ]
+          ]
+        },
+        "visibility": "visible",
+        "text-anchor": "top",
+        "text-offset": [
+          0,
+          0.6
+        ],
+        "text-padding": 2,
+        "text-optional": true,
+        "symbol-z-order": "auto",
+        "text-max-width": 9,
+        "icon-allow-overlap": false,
+        "text-allow-overlap": false
+      },
+      "paint": {
+        "text-color": "#5e3b9e",
+        "text-halo-blur": 0.5,
+        "text-halo-color": "rgba(255, 255, 255, 0.8)",
+        "text-halo-width": 1
+      },
+      "filter": [
+        "all",
+        [
+          "has",
+          "iata"
+        ]
+      ],
+      "order": 190
+    }
+  ]
+}

layers/aerodrome_label/update.sql

@@ -1,51 +0,0 @@
-DROP TRIGGER IF EXISTS trigger_flag ON osm_aerodrome_label_point;
-DROP TRIGGER IF EXISTS trigger_refresh ON aerodrome_label.updates;
-
--- etldoc: osm_aerodrome_label_point -> osm_aerodrome_label_point
-CREATE OR REPLACE FUNCTION update_aerodrome_label_point() RETURNS VOID AS $$
-BEGIN
-  UPDATE osm_aerodrome_label_point
-  SET geometry = ST_Centroid(geometry)
-  WHERE ST_GeometryType(geometry) <> 'ST_Point';
-
-  UPDATE osm_aerodrome_label_point
-  SET tags = update_tags(tags, geometry)
-  WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-SELECT update_aerodrome_label_point();
-
--- Handle updates
-
-CREATE SCHEMA IF NOT EXISTS aerodrome_label;
-
-CREATE TABLE IF NOT EXISTS aerodrome_label.updates(id serial primary key, t text, unique (t));
-CREATE OR REPLACE FUNCTION aerodrome_label.flag() RETURNS trigger AS $$
-BEGIN
-    INSERT INTO aerodrome_label.updates(t) VALUES ('y')  ON CONFLICT(t) DO NOTHING;
-    RETURN null;
-END;
-$$ language plpgsql;
-
-CREATE OR REPLACE FUNCTION aerodrome_label.refresh() RETURNS trigger AS
-  $BODY$
-  BEGIN
-    RAISE LOG 'Refresh aerodrome_label';
-    PERFORM update_aerodrome_label_point();
-    DELETE FROM aerodrome_label.updates;
-    RETURN null;
-  END;
-  $BODY$
-language plpgsql;
-
-CREATE TRIGGER trigger_flag
-    AFTER INSERT OR UPDATE OR DELETE ON osm_aerodrome_label_point
-    FOR EACH STATEMENT
-    EXECUTE PROCEDURE aerodrome_label.flag();
-
-CREATE CONSTRAINT TRIGGER trigger_refresh
-    AFTER INSERT ON aerodrome_label.updates
-    INITIALLY DEFERRED
-    FOR EACH ROW
-    EXECUTE PROCEDURE aerodrome_label.refresh();

layers/aerodrome_label/update_aerodrome_label_point.sql

@@ -0,0 +1,109 @@
+DROP TRIGGER IF EXISTS trigger_flag ON osm_aerodrome_label_point;
+DROP TRIGGER IF EXISTS trigger_store ON osm_aerodrome_label_point;
+DROP TRIGGER IF EXISTS trigger_refresh ON aerodrome_label.updates;
+
+-- Partial index for zoom 8/9 queries
+CREATE INDEX IF NOT EXISTS osm_aerodrome_label_point_type_partial_idx
+    ON osm_aerodrome_label_point USING gist (geometry)
+    WHERE aerodrome_type = 'international'
+      AND iata <> '';
+
+CREATE SCHEMA IF NOT EXISTS aerodrome_label;
+
+CREATE TABLE IF NOT EXISTS aerodrome_label.osm_ids
+(
+    osm_id bigint PRIMARY KEY
+);
+
+-- etldoc: osm_aerodrome_label_point -> osm_aerodrome_label_point
+CREATE OR REPLACE FUNCTION update_aerodrome_label_point(full_update boolean) RETURNS void AS
+$$
+    UPDATE osm_aerodrome_label_point
+    SET geometry = ST_Centroid(geometry)
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM aerodrome_label.osm_ids))
+        AND ST_GeometryType(geometry) <> 'ST_Point';
+
+    UPDATE osm_aerodrome_label_point
+    SET tags = update_tags(tags, geometry)
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM aerodrome_label.osm_ids))
+        AND COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL
+        AND tags != update_tags(tags, geometry);
+
+    UPDATE osm_aerodrome_label_point
+    SET aerodrome_type=
+       CASE
+	    %%FIELD_MAPPING: class %%
+	    ELSE 'other' END
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM aerodrome_label.osm_ids))
+    AND aerodrome_type !=
+       CASE
+	    %%FIELD_MAPPING: class %%
+	    ELSE 'other' END;
+$$ LANGUAGE SQL;
+
+SELECT update_aerodrome_label_point(true);
+
+-- Handle updates
+
+CREATE OR REPLACE FUNCTION aerodrome_label.store() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO aerodrome_label.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TABLE IF NOT EXISTS aerodrome_label.updates
+(
+    id serial PRIMARY KEY,
+    t text,
+    UNIQUE (t)
+);
+CREATE OR REPLACE FUNCTION aerodrome_label.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO aerodrome_label.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION aerodrome_label.refresh() RETURNS trigger AS
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+    RAISE LOG 'Refresh aerodrome_label';
+
+    -- Analyze tracking and source tables before performing update
+    ANALYZE aerodrome_label.osm_ids;
+    ANALYZE osm_aerodrome_label_point;
+
+    PERFORM update_aerodrome_label_point(false);
+    -- noinspection SqlWithoutWhere
+    DELETE FROM aerodrome_label.osm_ids;
+    -- noinspection SqlWithoutWhere
+    DELETE FROM aerodrome_label.updates;
+
+    RAISE LOG 'Refresh aerodrome_label done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_store
+    AFTER INSERT OR UPDATE
+    ON osm_aerodrome_label_point
+    FOR EACH ROW
+EXECUTE PROCEDURE aerodrome_label.store();
+
+CREATE TRIGGER trigger_flag
+    AFTER INSERT OR UPDATE
+    ON osm_aerodrome_label_point
+    FOR EACH STATEMENT
+EXECUTE PROCEDURE aerodrome_label.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+    AFTER INSERT
+    ON aerodrome_label.updates
+    INITIALLY DEFERRED
+    FOR EACH ROW
+EXECUTE PROCEDURE aerodrome_label.refresh();

layers/aeroway/aeroway.sql

@@ -0,0 +1,70 @@
+-- etldoc: layer_aeroway[shape=record fillcolor=lightpink, style="rounded,filled",
+-- etldoc:     label="layer_aeroway |<z10> z10|<z11> z11|<z12> z12|<z13> z13|<z14_> z14+" ];
+
+CREATE OR REPLACE FUNCTION layer_aeroway(bbox geometry, zoom_level int)
+    RETURNS TABLE
+            (
+                geometry geometry,
+                class    text,
+                ref      text
+            )
+AS
+$$
+SELECT geometry, aeroway AS class, ref
+FROM (
+         -- etldoc:  osm_aeroway_linestring_gen_z10 -> layer_aeroway:z10
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_linestring_gen_z10
+         WHERE zoom_level = 10
+         UNION ALL
+         -- etldoc:  osm_aeroway_linestring_gen_z11 -> layer_aeroway:z11
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_linestring_gen_z11
+         WHERE zoom_level = 11
+         UNION ALL
+         -- etldoc:  osm_aeroway_linestring_gen_z12 -> layer_aeroway:z12
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_linestring_gen_z12
+         WHERE zoom_level = 12
+         UNION ALL
+         -- etldoc:  osm_aeroway_linestring -> layer_aeroway:z13
+         -- etldoc:  osm_aeroway_linestring -> layer_aeroway:z14_
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_linestring
+         WHERE zoom_level >= 13
+         UNION ALL
+         -- etldoc:  osm_aeroway_polygon_gen_z10 -> layer_aeroway:z10
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_polygon_gen_z10
+         WHERE zoom_level = 10
+         UNION ALL
+         -- etldoc:  osm_aeroway_polygon_gen_z11 -> layer_aeroway:z11
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_polygon_gen_z11
+         WHERE zoom_level = 11
+         UNION ALL
+         -- etldoc:  osm_aeroway_polygon_gen_z12 -> layer_aeroway:z12
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_polygon_gen_z12
+         WHERE zoom_level = 12
+         UNION ALL
+         -- etldoc:  osm_aeroway_polygon_gen_z13 -> layer_aeroway:z13
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_polygon_gen_z13
+         WHERE zoom_level = 13
+         UNION ALL
+         -- etldoc:  osm_aeroway_polygon -> layer_aeroway:z14_
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_polygon
+         WHERE zoom_level >= 14
+         UNION ALL
+         
+         -- etldoc:  osm_aeroway_point -> layer_aeroway:z14_
+         SELECT geometry, aeroway, ref
+         FROM osm_aeroway_point
+         WHERE zoom_level >= 14
+     ) AS zoom_levels
+WHERE geometry && bbox;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/aeroway/aeroway.yaml

@@ -19,11 +19,12 @@ layer:
       - helipad
       - taxiway
       - apron
+      - gate
   datasource:
     geometry_field: geometry
     query: (SELECT geometry, ref, class FROM layer_aeroway(!bbox!, z(!scale_denominator!))) AS t
 schema:
-  - ./layer.sql
+  - ./aeroway.sql
 datasources:
   - type: imposm3
     mapping_file: ./mapping.yaml

layers/aeroway/layer.sql

@@ -1,43 +0,0 @@
--- etldoc: layer_aeroway[shape=record fillcolor=lightpink, style="rounded,filled",
--- etldoc:     label="layer_aeroway |<z10> z10|<z11> z11|<z12> z12|<z13> z13|<z14_> z14+" ];
-
-CREATE OR REPLACE FUNCTION layer_aeroway(bbox geometry, zoom_level int)
-RETURNS TABLE(geometry geometry, class text, ref text) AS $$
-    SELECT geometry, aeroway AS class, ref FROM (
-        -- etldoc:  osm_aeroway_linestring_gen3 -> layer_aeroway:z10
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_linestring_gen3 WHERE zoom_level = 10
-        UNION ALL
-        -- etldoc:  osm_aeroway_linestring_gen2 -> layer_aeroway:z11
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_linestring_gen2 WHERE zoom_level = 11
-        UNION ALL
-        -- etldoc:  osm_aeroway_linestring_gen1 -> layer_aeroway:z12
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_linestring_gen1 WHERE zoom_level = 12
-        UNION ALL
-        -- etldoc:  osm_aeroway_linestring -> layer_aeroway:z13
-        -- etldoc:  osm_aeroway_linestring -> layer_aeroway:z14_
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_linestring WHERE zoom_level >= 13
-        UNION ALL
-
-        -- etldoc:  osm_aeroway_polygon_gen3 -> layer_aeroway:z10
-        -- etldoc:  osm_aeroway_polygon_gen3 -> layer_aeroway:z11
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_polygon_gen3 WHERE zoom_level BETWEEN 10 AND 11
-        UNION ALL
-        -- etldoc:  osm_aeroway_polygon_gen2 -> layer_aeroway:z12
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_polygon_gen2 WHERE zoom_level = 12
-        UNION ALL
-        -- etldoc:  osm_aeroway_polygon_gen1 -> layer_aeroway:z13
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_polygon_gen1 WHERE zoom_level = 13
-        UNION ALL
-        -- etldoc:  osm_aeroway_polygon -> layer_aeroway:z14_
-        SELECT geometry, aeroway, ref
-        FROM osm_aeroway_polygon WHERE zoom_level >= 14
-    ) AS zoom_levels
-    WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE;

layers/aeroway/mapping.yaml

@@ -1,34 +1,40 @@
 generalized_tables:
-  # etldoc: imposm3 -> osm_aeroway_linestring_gen3
-  aeroway_linestring_gen3:
-    source: aeroway_linestring_gen2
+  # etldoc: osm_aeroway_linestring_gen_z11 -> osm_aeroway_linestring_gen_z10
+  aeroway_linestring_gen_z10:
+    source: aeroway_linestring_gen_z11
     tolerance: ZRES11
 
-  # etldoc: imposm3 -> osm_aeroway_linestring_gen2
-  aeroway_linestring_gen2:
-    source: aeroway_linestring_gen1
+  # etldoc: osm_aeroway_linestring_gen_z12 -> osm_aeroway_linestring_gen_z11
+  aeroway_linestring_gen_z11:
+    source: aeroway_linestring_gen_z12
     tolerance: ZRES12
 
-  # etldoc: imposm3 -> osm_aeroway_linestring_gen1
-  aeroway_linestring_gen1:
+  # etldoc: osm_aeroway_linestring -> osm_aeroway_linestring_gen_z12
+  aeroway_linestring_gen_z12:
     source: aeroway_linestring
     sql_filter: ST_IsValid(geometry)
     tolerance: ZRES13
 
-  # etldoc: imposm3 -> osm_aeroway_polygon_gen3
-  aeroway_polygon_gen3:
-    source: aeroway_polygon_gen2
+  # etldoc: osm_aeroway_polygon_gen_z11 -> osm_aeroway_polygon_gen_z10
+  aeroway_polygon_gen_z10:
+    source: aeroway_polygon_gen_z11
+    sql_filter: area>power(ZRES9,2)
+    tolerance: ZRES10
+
+  # etldoc: osm_aeroway_polygon_gen_z12 -> osm_aeroway_polygon_gen_z11
+  aeroway_polygon_gen_z11:
+    source: aeroway_polygon_gen_z12
     sql_filter: area>power(ZRES10,2)
     tolerance: ZRES11
 
-  # etldoc: imposm3 -> osm_aeroway_polygon_gen2
-  aeroway_polygon_gen2:
-    source: aeroway_polygon_gen1
+  # etldoc: osm_aeroway_polygon_gen_z13 -> osm_aeroway_polygon_gen_z12
+  aeroway_polygon_gen_z12:
+    source: aeroway_polygon_gen_z13
     sql_filter: area>power(ZRES11,2)
     tolerance: ZRES12
 
-  # etldoc: imposm3 -> osm_aeroway_polygon_gen1
-  aeroway_polygon_gen1:
+  # etldoc: osm_aeroway_polygon -> osm_aeroway_polygon_gen_z13
+  aeroway_polygon_gen_z13:
     source: aeroway_polygon
     sql_filter: area>power(ZRES12,2) AND ST_IsValid(geometry)
     tolerance: ZRES13
@@ -50,7 +56,7 @@ tables:
   # etldoc: imposm3 -> osm_aeroway_polygon
   aeroway_polygon:
     type: polygon
-    fields:
+    columns:
     - *ref
     - name: osm_id
       type: id
@@ -67,7 +73,7 @@ tables:
   # etldoc: imposm3 -> osm_aeroway_linestring
   aeroway_linestring:
     type: linestring
-    fields:
+    columns:
     - *ref
     - name: osm_id
       type: id
@@ -80,3 +86,19 @@ tables:
       aeroway:
       - runway
       - taxiway
+      
+  # etldoc: imposm3 -> osm_aeroway_point
+  aeroway_point:
+    type: point
+    columns:
+    - *ref
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - name: aeroway
+      key: aeroway
+      type: string
+    mapping:
+      aeroway:
+      - gate

layers/aeroway/style.json

@@ -0,0 +1,203 @@
+{
+  "layers": [
+    {
+      "id": "aeroway_fill",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "aeroway",
+      "minzoom": 11,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": {
+          "stops": [
+            [
+              6,
+              "rgba(223, 223, 228, 1)"
+            ],
+            [
+              12,
+              "rgba(232, 231, 223, 1)"
+            ]
+          ]
+        },
+        "fill-opacity": 1
+      },
+      "metadata": {},
+      "filter": [
+        "==",
+        "$type",
+        "Polygon"
+      ],
+      "order": 3
+    },
+    {
+      "id": "aeroway_runway",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "aeroway",
+      "minzoom": 11,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "rgba(178, 181, 209, 1)",
+        "line-width": {
+          "base": 1.2,
+          "stops": [
+            [
+              11,
+              3
+            ],
+            [
+              20,
+              48
+            ]
+          ]
+        },
+        "line-opacity": 1
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "==",
+          "$type",
+          "LineString"
+        ],
+        [
+          "==",
+          "class",
+          "runway"
+        ]
+      ],
+      "order": 22
+    },
+    {
+      "id": "aeroway_taxiway",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "aeroway",
+      "minzoom": 11,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "rgba(178, 181, 209, 1)",
+        "line-width": {
+          "base": 1.2,
+          "stops": [
+            [
+              11,
+              1
+            ],
+            [
+              20,
+              24
+            ]
+          ]
+        },
+        "line-opacity": 1
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "==",
+          "$type",
+          "LineString"
+        ],
+        [
+          "==",
+          "class",
+          "taxiway"
+        ]
+      ],
+      "order": 23
+    },
+    {
+      "id": "airport_label",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "aeroway",
+      "minzoom": 14,
+      "layout": {
+        "text-font": [
+          "Noto Sans Italic",
+          "Noto Sans Regular"
+        ],
+        "text-size": {
+          "stops": [
+            [
+              15,
+              9
+            ],
+            [
+              19,
+              15
+            ]
+          ]
+        },
+        "text-field": "{ref}",
+        "visibility": "visible",
+        "symbol-placement": "line"
+      },
+      "paint": {
+        "text-color": "#333333",
+        "text-halo-color": "rgba(255, 255, 255, 0.8)",
+        "text-halo-width": 1
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "runway",
+          "taxiway"
+        ]
+      ],
+      "order": 191
+    },
+    {
+      "id": "airport_gate",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "aeroway",
+      "minzoom": 16.5,
+      "layout": {
+        "text-font": [
+          "Noto Sans Regular"
+        ],
+        "text-size": {
+          "stops": [
+            [
+              17,
+              9
+            ],
+            [
+              19,
+              15
+            ]
+          ]
+        },
+        "text-field": "{ref}",
+        "visibility": "visible"
+      },
+      "paint": {
+        "text-color": "rgba(135, 135, 135, 1)",
+        "text-halo-color": "rgba(255, 255, 255, 1)",
+        "text-halo-width": 1
+      },
+      "filter": [
+        "all",
+        [
+          "==",
+          "class",
+          "gate"
+        ]
+      ],
+      "order": 192
+    }
+  ]
+}

layers/boundary/boundary.yaml

@@ -1,19 +1,35 @@
 layer:
   id: "boundary"
+  requires:
+    tables:
+      - osm_border_linestring
+      - ne_10m_admin_0_countries
+      - ne_10m_admin_0_boundary_lines_land
+      - ne_10m_admin_1_states_provinces_lines
+      - ne_50m_admin_0_boundary_lines_land
+      - ne_110m_admin_0_boundary_lines_land
   description: |
-      Contains administrative boundaries as linestrings.
+      Contains administrative boundaries as linestrings and aboriginal lands as polygons.
       Until z4 [Natural Earth data](http://www.naturalearthdata.com/downloads/) is used after which
       OSM boundaries ([`boundary=administrative`](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative))
       are present from z5 to z14 (also for maritime boundaries with `admin_level <= 2` at z4).
       OSM data contains several [`admin_level`](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative#admin_level)
       but for most styles it makes sense to just style `admin_level=2` and `admin_level=4`.
   fields:
+    class:
+      description: |
+        Use the **class** to differentiate between different kinds of boundaries.  The class for `boundary=aboriginal_lands` is `aboriginal_lands`.
+    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value (area features only).
     admin_level: |
         OSM [admin_level](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative#admin_level)
         indicating the level of importance of this boundary.
         The `admin_level` corresponds to the lowest `admin_level`
         the line participates in.
         At low zoom levels the Natural Earth boundaries are mapped to the equivalent admin levels.
+    adm0_l: |
+        State name on the left of the border. For country boundaries only (`admin_level = 2`).
+    adm0_r: |
+        State name on the right of the border. For country boundaries only (`admin_level = 2`).
     disputed:
       description: |
           Mark with `1` if the border is disputed.
@@ -46,8 +62,10 @@ layer:
   buffer_size: 4
   datasource:
     geometry_field: geometry
-    query: (SELECT geometry, admin_level, disputed, disputed_name, claimed_by, maritime FROM layer_boundary(!bbox!, z(!scale_denominator!))) AS t
+    query: (SELECT geometry, admin_level, adm0_l, adm0_r, disputed, disputed_name, claimed_by, maritime, class, name, {name_languages} FROM layer_boundary(!bbox!, z(!scale_denominator!))) AS t
 schema:
+  - ./update_boundary_polygon.sql
+  - ./boundary_name.sql
   - ./boundary.sql
 datasources:
   - type: imposm3

layers/boundary/boundary_name.sql

@@ -0,0 +1,105 @@
+DROP TABLE IF EXISTS osm_border_linestring_adm CASCADE;
+
+-- etldoc: osm_border_linestring -> osm_border_linestring_adm
+-- etldoc: osm_border_disp_linestring -> osm_border_linestring_adm
+-- etldoc: ne_10m_admin_0_countries -> osm_border_linestring_adm
+CREATE TABLE IF NOT EXISTS osm_border_linestring_adm AS ( 
+  WITH 
+    -- Prepare lines from osm to be merged
+	multiline AS (
+        SELECT osm_id,
+               ST_Node(ST_Collect(geometry)) AS geometry,
+               BOOL_OR(maritime) AS maritime,
+               FALSE AS disputed
+    	FROM osm_border_linestring
+    	WHERE admin_level = 2 AND ST_Dimension(geometry) = 1
+		    AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring)
+              GROUP BY osm_id
+		),
+
+	mergedline AS (
+		SELECT osm_id,
+      		     (ST_Dump(ST_LineMerge(geometry))).geom AS geometry,
+			maritime,
+			disputed
+  		FROM multiline
+		),
+    -- Create polygons from all boundaries to preserve real shape of country
+	polyg AS (
+    	SELECT (ST_Dump(
+        		 ST_Polygonize(geometry))).geom AS geometry  
+    	FROM (
+			SELECT (ST_Dump(
+      				ST_LineMerge(geometry))).geom AS geometry
+  			FROM (SELECT ST_Node(
+                          ST_Collect(geometry)) AS geometry
+    			FROM osm_border_linestring
+    			WHERE admin_level = 2 AND ST_Dimension(geometry) = 1
+                ) nodes
+			) linemerge
+  		), 
+
+    centroids AS (
+		SELECT polyg.geometry,
+			   ne.adm0_a3
+		FROM polyg,
+			 ne_10m_admin_0_countries AS ne
+		WHERE ST_Within(
+			   ST_PointOnSurface(polyg.geometry), ne.geometry)
+    	),
+
+	country_osm_polyg AS  (
+		SELECT country.adm0_a3,
+			   border.geometry
+  		FROM polyg border,
+			 centroids country
+  		WHERE ST_Within(country.geometry, border.geometry)
+	),
+
+	rights AS (
+        SELECT osm_id,
+			   adm0_r,
+			   geometry,
+			   maritime,
+			   disputed
+		FROM (
+			SELECT a.osm_id AS osm_id,
+                   b.adm0_a3 AS adm0_r,
+                   a.geometry,
+                   a.maritime,
+                   a.disputed
+			FROM mergedline AS a
+			LEFT JOIN country_osm_polyg AS b
+            -- Create short line on the right of the boundary (mergedline) and find state where line lies.
+			ON ST_Within(
+				ST_OffsetCurve(
+				(ST_LineSubString(a.geometry, 0.3,0.3004)), 70, 'quad_segs=4 join=mitre'), b.geometry)
+            ) line_rights
+		)
+
+  SELECT osm_id,
+		 adm0_l,
+		 adm0_r,
+		 geometry,
+		 maritime,
+		 2::integer AS admin_level,
+		 disputed
+  FROM (
+    SELECT r.osm_id AS osm_id,
+           b.adm0_a3 AS adm0_l,
+           r.adm0_r AS adm0_r,
+           r.geometry,
+           r.maritime,
+           r.disputed
+    FROM rights AS r
+    LEFT JOIN country_osm_polyg AS b
+      -- Create short line on the left of the boundary (mergedline) and find state where line lies.
+      ON ST_Within(
+        ST_OffsetCurve(
+          (ST_LineSubString(r.geometry, 0.4,0.4004)), -70, 'quad_segs=4 join=mitre'), b.geometry)
+    ) both_lines
+);
+
+CREATE INDEX IF NOT EXISTS osm_border_linestring_adm_geom_idx
+  ON osm_border_linestring_adm
+  USING GIST (geometry);

layers/boundary/mapping.yaml

@@ -1,95 +1,95 @@
 generalized_tables:
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen11
-  border_disp_linestring_gen11:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 9600
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen10
-  border_disp_linestring_gen10:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 4800
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen9
-  border_disp_linestring_gen9:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 2400
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen8
-  border_disp_linestring_gen8:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 1200
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen7
-  border_disp_linestring_gen7:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 600
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen6
-  border_disp_linestring_gen6:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 300
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen5
-  border_disp_linestring_gen5:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 160
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen4
-  border_disp_linestring_gen4:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 80
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen3
-  border_disp_linestring_gen3:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 40
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen2
-  border_disp_linestring_gen2:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 20
-
-  # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen1
-  border_disp_linestring_gen1:
-    source: border_disp_linestring
-    sql_filter: admin_level = 2
-    tolerance: 10
-
-  # etldoc: osm_border_disp_relation -> osm_border_disp_linestring
+  # etldoc: osm_border_linestring -> osm_border_disp_linestring
   border_disp_linestring:
-    source: border_disp_relation
-    sql_filter: ST_GeometryType(geometry) = 'ST_LineString'
+    source: border_linestring
+    sql_filter: ST_GeometryType(geometry) = 'ST_LineString' AND (disputed OR dispute OR border_status = 'disputed' OR disputed_by <> '') AND admin_level = 2
+
+  # etldoc: osm_boundary_polygon_gen_z5 -> osm_boundary_polygon_gen_z4
+  boundary_polygon_gen_z4:
+    source: boundary_polygon_gen_z5
+    sql_filter: area>power(ZRES3,2)
+    tolerance: ZRES4
+
+  # etldoc: osm_boundary_polygon_gen_z6 -> osm_boundary_polygon_gen_z5
+  boundary_polygon_gen_z5:
+    source: boundary_polygon_gen_z6
+    sql_filter: area>power(ZRES4,2)
+    tolerance: ZRES5
+
+  # etldoc: osm_boundary_polygon_gen_z7 -> osm_boundary_polygon_gen_z6
+  boundary_polygon_gen_z6:
+    source: boundary_polygon_gen_z7
+    sql_filter: area>power(ZRES5,2)
+    tolerance: ZRES6
+
+  # etldoc: osm_boundary_polygon_gen_z8 -> osm_boundary_polygon_gen_z7
+  boundary_polygon_gen_z7:
+    source: boundary_polygon_gen_z8
+    sql_filter: area>power(ZRES6,2)
+    tolerance: ZRES7
+
+  # etldoc: osm_boundary_polygon_gen_z9 -> osm_boundary_polygon_gen_z8
+  boundary_polygon_gen_z8:
+    source: boundary_polygon_gen_z9
+    sql_filter: area>power(ZRES7,2)
+    tolerance: ZRES8
+
+  # etldoc: osm_boundary_polygon_gen_z10 -> osm_boundary_polygon_gen_z9
+  boundary_polygon_gen_z9:
+    source: boundary_polygon_gen_z10
+    sql_filter: area>power(ZRES8,2)
+    tolerance: ZRES9
+
+  # etldoc: osm_boundary_polygon_gen_z11 -> osm_boundary_polygon_gen_z10
+  boundary_polygon_gen_z10:
+    source: boundary_polygon_gen_z11
+    sql_filter: area>power(ZRES9,2)
+    tolerance: ZRES10
+
+  # etldoc: osm_boundary_polygon_gen_z12 -> osm_boundary_polygon_gen_z11
+  boundary_polygon_gen_z11:
+    source: boundary_polygon_gen_z12
+    sql_filter: area>power(ZRES10,2)
+    tolerance: ZRES11
+
+  # etldoc: osm_boundary_polygon_gen_z13 -> osm_boundary_polygon_gen_z12
+  boundary_polygon_gen_z12:
+    source: boundary_polygon_gen_z13
+    sql_filter: area>power(ZRES11,2)
+    tolerance: ZRES12
+
+  # etldoc: osm_boundary_polygon -> osm_boundary_polygon_gen_z13
+  boundary_polygon_gen_z13:
+    source: boundary_polygon
+    sql_filter: area>power(ZRES12,2) AND ST_IsValid(geometry)
+    tolerance: ZRES13
 
 
 tables:
-  # etldoc: imposm3 -> osm_border_disp_relation
-  border_disp_relation:
+  # etldoc: imposm3 -> osm_border_linestring
+  border_linestring:
     type: relation_member
-    fields:
+    filters:
+      require:
+        admin_level: [__any__]
+        boundary: [administrative]
+    columns:
     - name: relation_id
       type: id
     - name: osm_id
       type: id
       from_member: true
+    - name: member
+      type: member_id
+    - name: type
+      type: member_type
     - name: geometry
       type: geometry
     - key: name
       name: name
       type: string
-    - key: boundary
-      name: boundary
-      type: string
+      # Used for disputed boundary, e.g. "Line of actual control"
+      from_member: true
     - key: admin_level
       name: admin_level
       type: integer
@@ -99,15 +99,62 @@ tables:
     - key: disputed_by
       name: disputed_by
       type: string
+      from_member: true
+    - key: dispute 
+      name: dispute 
+      type: bool
+      from_member: true
+    - key: disputed
+      name: disputed
+      type: bool
+      from_member: true
+    - key: border_status
+      name: border_status
+      type: string
+      from_member: true
     - key: maritime
       name: maritime
       type: bool
       from_member: true
+    - key: boundary_type
+      name: boundary_type
+      type: string
+      from_member: true
+    - key: natural
+      name: natural
+      type: string
+      from_member: true
+    relation_types: [boundary]
     mapping:
-      type: [boundary]
+      boundary:
+      - administrative
+      border_status:
+      - dispute
+      boundary_type:
+      - maritime
+
+  # etldoc: imposm3 -> osm_boundary_polygon
+  boundary_polygon:
+    type: polygon
     filters:
       require:
-        #admin_level: ['2']
-        admin_level: [__any__]
-        claimed_by: [__any__]
-
+        type: [boundary]
+        boundary: [aboriginal_lands]
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: validated_geometry
+    - name: name
+      key: name
+      type: string
+    - name: tags
+      type: hstore_tags
+    - name: boundary
+      key: boundary
+      type: string
+    - name: area
+      type: area
+    mapping:
+      boundary:
+      - aboriginal_lands

layers/boundary/style.json

@@ -0,0 +1,287 @@
+{
+  "layers": [
+    {
+      "id": "boundary_3",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "boundary",
+      "minzoom": 3,
+      "layout": {
+        "line-join": "round",
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#845283",
+        "line-width": {
+          "base": 1,
+          "stops": [
+            [
+              4,
+              0.4
+            ],
+            [
+              5,
+              0.7
+            ],
+            [
+              12,
+              1.6
+            ]
+          ]
+        },
+        "line-opacity": {
+          "stops": [
+            [
+              3,
+              0.5
+            ],
+            [
+              10,
+              1
+            ]
+          ]
+        },
+        "line-dasharray": [
+          5,
+          3
+        ]
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "admin_level",
+          3,
+          4
+        ],
+        [
+          "==",
+          "maritime",
+          0
+        ]
+      ],
+      "order": 146
+    },
+    {
+      "id": "boundary_2",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "boundary",
+      "minzoom": 0,
+      "layout": {
+        "line-cap": "round",
+        "line-join": "round",
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#a37da1",
+        "line-width": {
+          "base": 1,
+          "stops": [
+            [
+              3,
+              0.5
+            ],
+            [
+              5,
+              1.2
+            ],
+            [
+              12,
+              3
+            ]
+          ]
+        },
+        "line-opacity": 1
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "==",
+          "admin_level",
+          2
+        ],
+        [
+          "==",
+          "maritime",
+          0
+        ],
+        [
+          "==",
+          "disputed",
+          0
+        ]
+      ],
+      "order": 147
+    },
+    {
+      "id": "boundary_2_disputed",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "boundary",
+      "minzoom": 0,
+      "layout": {
+        "line-cap": "round",
+        "line-join": "round",
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#a37da1",
+        "line-width": {
+          "base": 1,
+          "stops": [
+            [
+              3,
+              0.3
+            ],
+            [
+              5,
+              1.2
+            ],
+            [
+              12,
+              3
+            ]
+          ]
+        },
+        "line-opacity": 1,
+        "line-dasharray": [
+          4,
+          3
+        ]
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "==",
+          "admin_level",
+          2
+        ],
+        [
+          "==",
+          "disputed",
+          1
+        ],
+        [
+          "==",
+          "maritime",
+          0
+        ]
+      ],
+      "order": 148
+    },
+    {
+      "id": "boundary_2_disputed_maritime",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "boundary",
+      "minzoom": 0,
+      "layout": {
+        "line-cap": "round",
+        "line-join": "round",
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "rgba(129, 125, 163, 1)",
+        "line-width": {
+          "base": 1,
+          "stops": [
+            [
+              3,
+              0.5
+            ],
+            [
+              5,
+              1.2
+            ],
+            [
+              12,
+              3
+            ]
+          ]
+        },
+        "line-opacity": 1,
+        "line-dasharray": [
+          4,
+          3
+        ]
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "==",
+          "admin_level",
+          2
+        ],
+        [
+          "==",
+          "disputed",
+          1
+        ],
+        [
+          "==",
+          "maritime",
+          1
+        ]
+      ],
+      "order": 149
+    },
+    {
+      "id": "boundary_2_maritime",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "boundary",
+      "minzoom": 4,
+      "layout": {
+        "line-cap": "round",
+        "line-join": "round",
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#a37da1",
+        "line-width": {
+          "base": 1,
+          "stops": [
+            [
+              3,
+              0.5
+            ],
+            [
+              5,
+              1.2
+            ],
+            [
+              12,
+              3
+            ]
+          ]
+        },
+        "line-opacity": 1
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "==",
+          "admin_level",
+          2
+        ],
+        [
+          "==",
+          "disputed",
+          0
+        ],
+        [
+          "==",
+          "maritime",
+          1
+        ]
+      ],
+      "order": 150
+    }
+  ]
+}

layers/boundary/update_boundary_polygon.sql

@@ -0,0 +1,221 @@
+ALTER TABLE osm_boundary_polygon
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z13
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z12
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z11
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z10
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z9
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z8
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z7
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z6
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_boundary_polygon_gen_z5
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z13;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z12;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z11;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z10;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z9;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z8;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z7;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z6;
+DROP TRIGGER IF EXISTS update_row ON osm_boundary_polygon_gen_z5;
+DROP TRIGGER IF EXISTS trigger_flag ON osm_boundary_polygon;
+DROP TRIGGER IF EXISTS trigger_refresh ON boundary_polygon.updates;
+
+-- etldoc:  osm_boundary_polygon ->  osm_boundary_polygon
+-- etldoc:  osm_boundary_polygon_gen_z13 ->  osm_boundary_polygon_gen_z13
+-- etldoc:  osm_boundary_polygon_gen_z12 ->  osm_boundary_polygon_gen_z12
+-- etldoc:  osm_boundary_polygon_gen_z11 ->  osm_boundary_polygon_gen_z11
+-- etldoc:  osm_boundary_polygon_gen_z10 ->  osm_boundary_polygon_gen_z10
+-- etldoc:  osm_boundary_polygon_gen_z9 ->  osm_boundary_polygon_gen_z9
+-- etldoc:  osm_boundary_polygon_gen_z8 ->  osm_boundary_polygon_gen_z8
+-- etldoc:  osm_boundary_polygon_gen_z7 ->  osm_boundary_polygon_gen_z7
+-- etldoc:  osm_boundary_polygon_gen_z6 ->  osm_boundary_polygon_gen_z6
+-- etldoc:  osm_boundary_polygon_gen_z5 ->  osm_boundary_polygon_gen_z5
+CREATE OR REPLACE FUNCTION update_osm_boundary_polygon() RETURNS void AS
+$$
+BEGIN
+    UPDATE osm_boundary_polygon
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z13
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z12
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z11
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z10
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z9
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z8
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z7
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z6
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    UPDATE osm_boundary_polygon_gen_z5
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+END;
+$$ LANGUAGE plpgsql;
+
+SELECT update_osm_boundary_polygon();
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_point_geom_idx ON osm_boundary_polygon USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z13_point_geom_idx ON osm_boundary_polygon_gen_z13 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z12_point_geom_idx ON osm_boundary_polygon_gen_z12 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z11_point_geom_idx ON osm_boundary_polygon_gen_z11 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z10_point_geom_idx ON osm_boundary_polygon_gen_z10 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z9_point_geom_idx ON osm_boundary_polygon_gen_z9 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z8_point_geom_idx ON osm_boundary_polygon_gen_z8 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z7_point_geom_idx ON osm_boundary_polygon_gen_z7 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z6_point_geom_idx ON osm_boundary_polygon_gen_z6 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_boundary_polygon_gen_z5_point_geom_idx ON osm_boundary_polygon_gen_z5 USING gist (geometry_point);
+
+CREATE SCHEMA IF NOT EXISTS boundary_polygon;
+
+CREATE TABLE IF NOT EXISTS boundary_polygon.updates
+(
+    id serial PRIMARY KEY,
+    t  text,
+    UNIQUE (t)
+);
+
+CREATE OR REPLACE FUNCTION boundary_polygon.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO boundary_polygon.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION boundary_polygon.refresh() RETURNS trigger AS
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+    RAISE LOG 'Refresh boundary_polygon';
+
+    -- Analyze tracking and source tables before performing update
+    ANALYZE osm_boundary_polygon_gen_z5;
+    REFRESH MATERIALIZED VIEW osm_boundary_polygon_gen_z5;
+
+    -- noinspection SqlWithoutWhere
+    DELETE FROM boundary_polygon.updates;
+
+    RAISE LOG 'Refresh boundary_polygon done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION update_osm_boundary_polygon_row()
+    RETURNS trigger
+AS
+$$
+BEGIN
+    NEW.tags = update_tags(NEW.tags, NEW.geometry);
+    NEW.geometry_point = ST_PointOnSurface(NEW.geometry);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z13
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z12
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z11
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z10
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z9
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z8
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z7
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z6
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_boundary_polygon_gen_z5
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_boundary_polygon_row();
+
+CREATE TRIGGER trigger_flag
+    AFTER INSERT OR UPDATE OR DELETE
+    ON osm_boundary_polygon
+    FOR EACH STATEMENT
+EXECUTE PROCEDURE boundary_polygon.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+    AFTER INSERT
+    ON boundary_polygon.updates
+    INITIALLY DEFERRED
+    FOR EACH ROW
+EXECUTE PROCEDURE boundary_polygon.refresh();

layers/building/building.sql

@@ -1,145 +1,120 @@
 -- etldoc: layer_building[shape=record fillcolor=lightpink, style="rounded,filled",
 -- etldoc:     label="layer_building | <z13> z13 | <z14_> z14+ " ] ;
 
-CREATE OR REPLACE FUNCTION as_numeric(text) RETURNS NUMERIC AS $$
- -- Inspired by http://stackoverflow.com/questions/16195986/isnumeric-with-postgresql/16206123#16206123
-DECLARE test NUMERIC;
-BEGIN
-     test = $1::NUMERIC;
-     RETURN test;
-EXCEPTION WHEN others THEN
-     RETURN -1;
-END;
-$$ STRICT
-LANGUAGE plpgsql IMMUTABLE;
+CREATE INDEX IF NOT EXISTS osm_building_relation_building_idx ON osm_building_relation (building) WHERE building = '' AND ST_GeometryType(geometry) = 'ST_Polygon';
+CREATE INDEX IF NOT EXISTS osm_building_relation_member_idx ON osm_building_relation (member) WHERE role = 'outline';
 
-CREATE INDEX IF NOT EXISTS osm_building_relation_building_idx ON osm_building_relation(building) WHERE ST_GeometryType(geometry) = 'ST_Polygon';
-CREATE INDEX IF NOT EXISTS osm_building_relation_member_idx ON osm_building_relation(member);
---CREATE INDEX IF NOT EXISTS osm_building_associatedstreet_role_idx ON osm_building_associatedstreet(role) WHERE ST_GeometryType(geometry) = 'ST_Polygon';
---CREATE INDEX IF NOT EXISTS osm_building_street_role_idx ON osm_building_street(role) WHERE ST_GeometryType(geometry) = 'ST_Polygon';
+CREATE OR REPLACE VIEW osm_all_buildings AS
+(
+SELECT
+    -- etldoc: osm_building_relation -> layer_building:z14_
+    -- Buildings built from relations
+    member AS osm_id,
+    geometry,
+    COALESCE(CleanNumeric(height), CleanNumeric(buildingheight)) AS height,
+    COALESCE(CleanNumeric(min_height), CleanNumeric(buildingmin_height)) AS min_height,
+    COALESCE(CleanNumeric(levels), CleanNumeric(buildinglevels)) AS levels,
+    COALESCE(CleanNumeric(min_level), CleanNumeric(buildingmin_level)) AS min_level,
+    nullif(material, '') AS material,
+    nullif(colour, '') AS colour,
+    FALSE AS hide_3d
+FROM osm_building_relation
+WHERE building = ''
+  AND ST_GeometryType(geometry) = 'ST_Polygon'
+UNION ALL
 
-CREATE OR REPLACE VIEW osm_all_buildings AS (
-         -- etldoc: osm_building_relation -> layer_building:z14_
-         -- Buildings built from relations
-         SELECT member AS osm_id,geometry,
-                  COALESCE(nullif(as_numeric(height),-1),nullif(as_numeric(buildingheight),-1)) as height,
-                  COALESCE(nullif(as_numeric(min_height),-1),nullif(as_numeric(buildingmin_height),-1)) as min_height,
-                  COALESCE(nullif(as_numeric(levels),-1),nullif(as_numeric(buildinglevels),-1)) as levels,
-                  COALESCE(nullif(as_numeric(min_level),-1),nullif(as_numeric(buildingmin_level),-1)) as min_level,
-                  nullif(material, '') AS material,
-                  nullif(colour, '') AS colour,
-                  FALSE as hide_3d
-         FROM
-         osm_building_relation WHERE building = '' AND ST_GeometryType(geometry) = 'ST_Polygon'
-         UNION ALL
-
-         -- etldoc: osm_building_associatedstreet -> layer_building:z14_
-         -- Buildings in associatedstreet relations
-         SELECT member AS osm_id,geometry,
-                  COALESCE(nullif(as_numeric(height),-1),nullif(as_numeric(buildingheight),-1)) as height,
-                  COALESCE(nullif(as_numeric(min_height),-1),nullif(as_numeric(buildingmin_height),-1)) as min_height,
-                  COALESCE(nullif(as_numeric(levels),-1),nullif(as_numeric(buildinglevels),-1)) as levels,
-                  COALESCE(nullif(as_numeric(min_level),-1),nullif(as_numeric(buildingmin_level),-1)) as min_level,
-                  nullif(material, '') AS material,
-                  nullif(colour, '') AS colour,
-                  FALSE as hide_3d
-         FROM
-         osm_building_associatedstreet WHERE role = 'house' AND ST_GeometryType(geometry) = 'ST_Polygon'
-         UNION ALL
-         -- etldoc: osm_building_street -> layer_building:z14_
-         -- Buildings in street relations
-         SELECT member AS osm_id,geometry,
-                  COALESCE(nullif(as_numeric(height),-1),nullif(as_numeric(buildingheight),-1)) as height,
-                  COALESCE(nullif(as_numeric(min_height),-1),nullif(as_numeric(buildingmin_height),-1)) as min_height,
-                  COALESCE(nullif(as_numeric(levels),-1),nullif(as_numeric(buildinglevels),-1)) as levels,
-                  COALESCE(nullif(as_numeric(min_level),-1),nullif(as_numeric(buildingmin_level),-1)) as min_level,
-                  nullif(material, '') AS material,
-                  nullif(colour, '') AS colour,
-                  FALSE as hide_3d
-         FROM
-         osm_building_street WHERE role = 'house' AND ST_GeometryType(geometry) = 'ST_Polygon'
-         UNION ALL
-
-         -- etldoc: osm_building_multipolygon -> layer_building:z14_
-         -- Buildings that are inner/outer
-         SELECT osm_id,geometry,
-                  COALESCE(nullif(as_numeric(height),-1),nullif(as_numeric(buildingheight),-1)) as height,
-                  COALESCE(nullif(as_numeric(min_height),-1),nullif(as_numeric(buildingmin_height),-1)) as min_height,
-                  COALESCE(nullif(as_numeric(levels),-1),nullif(as_numeric(buildinglevels),-1)) as levels,
-                  COALESCE(nullif(as_numeric(min_level),-1),nullif(as_numeric(buildingmin_level),-1)) as min_level,
-                  nullif(material, '') AS material,
-                  nullif(colour, '') AS colour,
-                  FALSE as hide_3d
-         FROM
-         osm_building_polygon obp WHERE EXISTS (SELECT 1 FROM osm_building_multipolygon obm WHERE obp.osm_id = obm.osm_id)
-         UNION ALL
-         -- etldoc: osm_building_polygon -> layer_building:z14_
-         -- Standalone buildings
-         SELECT obp.osm_id,obp.geometry,
-                  COALESCE(nullif(as_numeric(obp.height),-1),nullif(as_numeric(obp.buildingheight),-1)) as height,
-                  COALESCE(nullif(as_numeric(obp.min_height),-1),nullif(as_numeric(obp.buildingmin_height),-1)) as min_height,
-                  COALESCE(nullif(as_numeric(obp.levels),-1),nullif(as_numeric(obp.buildinglevels),-1)) as levels,
-                  COALESCE(nullif(as_numeric(obp.min_level),-1),nullif(as_numeric(obp.buildingmin_level),-1)) as min_level,
-                  nullif(obp.material, '') AS material,
-                  nullif(obp.colour, '') AS colour,
-                  CASE WHEN obr.role='outline' THEN TRUE ELSE FALSE END as hide_3d
-         FROM
-         osm_building_polygon obp
-           LEFT JOIN osm_building_relation obr ON (obr.member = obp.osm_id)
-         WHERE obp.osm_id >= 0
-);
+SELECT
+    -- etldoc: osm_building_polygon -> layer_building:z14_
+    -- Standalone buildings
+    obp.osm_id,
+    obp.geometry,
+    COALESCE(CleanNumeric(obp.height), CleanNumeric(obp.buildingheight)) AS height,
+    COALESCE(CleanNumeric(obp.min_height), CleanNumeric(obp.buildingmin_height)) AS min_height,
+    COALESCE(CleanNumeric(obp.levels), CleanNumeric(obp.buildinglevels)) AS levels,
+    COALESCE(CleanNumeric(obp.min_level), CleanNumeric(obp.buildingmin_level)) AS min_level,
+    nullif(obp.material, '') AS material,
+    nullif(obp.colour, '') AS colour,
+    obr.role IS NOT NULL AS hide_3d
+FROM osm_building_polygon obp
+         LEFT JOIN osm_building_relation obr ON
+        obp.osm_id >= 0 AND
+        obr.member = obp.osm_id AND
+        obr.role = 'outline'
+WHERE ST_GeometryType(obp.geometry) IN ('ST_Polygon', 'ST_MultiPolygon')
+    );
 
 CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int)
-RETURNS TABLE(geometry geometry, osm_id bigint, render_height int, render_min_height int, colour text, hide_3d boolean) AS $$
-    SELECT geometry, osm_id, render_height, render_min_height,
+    RETURNS TABLE
+            (
+                geometry          geometry,
+                osm_id            bigint,
+                render_height     int,
+                render_min_height int,
+                colour            text,
+                hide_3d           boolean
+            )
+AS
+$$
+SELECT geometry,
+       osm_id,
+       render_height,
+       render_min_height,
        COALESCE(colour, CASE material
            -- Ordered by count from taginfo
-           WHEN 'cement_block' THEN '#6a7880'
-           WHEN 'brick' THEN '#bd8161'
-           WHEN 'plaster' THEN '#dadbdb'
-           WHEN 'wood' THEN '#d48741'
-           WHEN 'concrete' THEN '#d3c2b0'
-           WHEN 'metal' THEN '#b7b1a6'
-           WHEN 'stone' THEN '#b4a995'
-           WHEN 'mud' THEN '#9d8b75'
-           WHEN 'steel' THEN '#b7b1a6' -- same as metal
-           WHEN 'glass' THEN '#5a81a0'
-           WHEN 'traditional' THEN '#bd8161' -- same as brick
-           WHEN 'masonry' THEN '#bd8161' -- same as brick
-           WHEN 'Brick' THEN '#bd8161' -- same as brick
-           WHEN 'tin' THEN '#b7b1a6' -- same as metal
-           WHEN 'timber_framing' THEN '#b3b0a9'
-           WHEN 'sandstone' THEN '#b4a995' -- same as stone
-           WHEN 'clay' THEN '#9d8b75' -- same as mud
-       END) AS colour,
-      CASE WHEN hide_3d THEN TRUE ELSE NULL::boolean END AS hide_3d
-    FROM (
-        -- etldoc: osm_building_polygon_gen1 -> layer_building:z13
-        SELECT
-            osm_id, geometry,
-            NULL::int AS render_height, NULL::int AS render_min_height,
-            NULL::text AS material, NULL::text AS colour,
-            FALSE AS hide_3d
-        FROM osm_building_polygon_gen1
-        WHERE zoom_level = 13 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_building_polygon -> layer_building:z14_
-        SELECT DISTINCT ON (osm_id)
-           osm_id, geometry,
-           ceil( COALESCE(height, levels*3.66,5))::int AS render_height,
-           floor(COALESCE(min_height, min_level*3.66,0))::int AS render_min_height,
-           material,
-           colour,
-           hide_3d
-        FROM osm_all_buildings
-        WHERE
-            (levels IS NULL OR levels < 1000) AND
-            (min_level IS NULL OR min_level < 1000) AND
-            (height IS NULL OR height < 3000) AND
-            (min_height IS NULL OR min_height < 3000) AND
-            zoom_level >= 14 AND geometry && bbox
-    ) AS zoom_levels
-    ORDER BY render_height ASC, ST_YMin(geometry) DESC;
-$$ LANGUAGE SQL IMMUTABLE;
+                            WHEN 'cement_block' THEN '#6a7880'
+                            WHEN 'brick' THEN '#bd8161'
+                            WHEN 'plaster' THEN '#dadbdb'
+                            WHEN 'wood' THEN '#d48741'
+                            WHEN 'concrete' THEN '#d3c2b0'
+                            WHEN 'metal' THEN '#b7b1a6'
+                            WHEN 'stone' THEN '#b4a995'
+                            WHEN 'mud' THEN '#9d8b75'
+                            WHEN 'steel' THEN '#b7b1a6' -- same as metal
+                            WHEN 'glass' THEN '#5a81a0'
+                            WHEN 'traditional' THEN '#bd8161' -- same as brick
+                            WHEN 'masonry' THEN '#bd8161' -- same as brick
+                            WHEN 'Brick' THEN '#bd8161' -- same as brick
+                            WHEN 'tin' THEN '#b7b1a6' -- same as metal
+                            WHEN 'timber_framing' THEN '#b3b0a9'
+                            WHEN 'sandstone' THEN '#b4a995' -- same as stone
+                            WHEN 'clay' THEN '#9d8b75' -- same as mud
+           END) AS colour,
+       CASE WHEN hide_3d THEN TRUE END AS hide_3d
+FROM (
+         SELECT
+             -- etldoc: osm_building_block_gen_z13 -> layer_building:z13
+             osm_id,
+             geometry,
+             NULL::int AS render_height,
+             NULL::int AS render_min_height,
+             NULL::text AS material,
+             NULL::text AS colour,
+             FALSE AS hide_3d
+         FROM osm_building_block_gen_z13
+         WHERE zoom_level = 13
+           AND geometry && bbox
+         UNION ALL
+         SELECT
+                                  -- etldoc: osm_building_polygon -> layer_building:z14_
+             DISTINCT ON (osm_id) osm_id,
+                                  geometry,
+                                  ceil(COALESCE(height, levels * 3.66, 5))::int AS render_height,
+                                  floor(COALESCE(min_height, min_level * 3.66, 0))::int AS render_min_height,
+                                  material,
+                                  colour,
+                                  hide_3d
+         FROM osm_all_buildings
+         WHERE (levels IS NULL OR levels < 1000)
+           AND (min_level IS NULL OR min_level < 1000)
+           AND (height IS NULL OR height < 3000)
+           AND (min_height IS NULL OR min_height < 3000)
+           AND zoom_level >= 14
+           AND geometry && bbox
+     ) AS zoom_levels
+ORDER BY render_height ASC, ST_YMin(geometry) DESC;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE
+                ;
 
 -- not handled: where a building outline covers building parts

layers/building/building.yaml

@@ -1,22 +1,27 @@
 layer:
   id: "building"
   description: |
-      All [OSM Buildings](http://wiki.openstreetmap.org/wiki/Buildings). All building tags are imported ([`building=*`](http://wiki.openstreetmap.org/wiki/Key:building)). The buildings are not yet ready for 3D rendering support and any help to improve
-      this is welcomed.
+      All [OSM Buildings](http://wiki.openstreetmap.org/wiki/Buildings). All building tags are imported ([`building=*`](http://wiki.openstreetmap.org/wiki/Key:building)).
+      Only buildings with tag location:underground are excluded.
   buffer_size: 4
   datasource:
-    query: (SELECT geometry, render_height, render_min_height, colour, hide_3d FROM layer_building(!bbox!, z(!scale_denominator!))) AS t
+    geometry_field: geometry
+    key_field: osm_id
+    key_field_as_attribute: no
+    srid: 900913
+    query: (SELECT osm_id, geometry, render_height, render_min_height, colour, hide_3d FROM layer_building(!bbox!, z(!scale_denominator!))) AS t
   fields:
     render_height: |
-        An approximated height from levels and height of the building or building:part after the method of Paul Norman in [OSM Clear](https://github.com/ClearTables/osm-clear). For future 3D rendering of buildings.
+        An approximated height from levels and height of the building or building:part.
     render_min_height: |
-        An approximated height from levels and height of the bottom of the building or building:part after the method of Paul Norman in [OSM Clear](https://github.com/ClearTables/osm-clear). For future 3D rendering of buildings.
+        An approximated height from minimum levels or minimum height of the bottom of the building or building:part.
     colour: |
         Colour
     hide_3d: |
         If True, building (part) should not be rendered in 3D. Currently, [building outlines](https://wiki.openstreetmap.org/wiki/Simple_3D_buildings) are marked as hide_3d.
 schema:
+  - ./update_building.sql
   - ./building.sql
 datasources:
   - type: imposm3
-    mapping_file: ./mapping.yaml
+    mapping_file: ./mapping.yaml

layers/building/mapping.yaml

@@ -1,14 +1,14 @@
-generalized_tables:
-  # etldoc: imposm3 -> osm_building_polygon_gen1
-  building_polygon_gen1:
-    source: building_polygon
-    sql_filter: area>power(ZRES12,2) AND ST_IsValid(geometry)
-    tolerance: ZRES14
+#generalized_tables:
+#  # etldoc: imposm3 -> osm_building_polygon_gen1
+#  building_polygon_gen1:
+#    source: building_polygon
+#    sql_filter: area>power(ZRES12,2) AND ST_IsValid(geometry)
+#    tolerance: ZRES14
 
 tables:
   # etldoc: imposm3 -> osm_building_polygon
   building_polygon:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -60,200 +60,19 @@ tables:
       aeroway:
       - terminal
       - hangar
+      location:
+      - underground
     filters:
       reject:
         building: ["no","none","No"]
         building:part: ["no","none","No"]
         man_made: ["bridge"]
+        location: ["underground"]
     type: polygon
 
-  # etldoc: imposm3 -> osm_building_street
-  building_street:
-    fields:
-    - name: osm_id
-      type: id
-    - name: geometry
-      type: validated_geometry
-    - name: area
-      type: area
-    - name: building
-      key: building
-      type: string
-      from_member: true
-    - name: material
-      key: building:material
-      type: string
-    - name: colour
-      key: building:colour
-      type: string
-    - name: buildingpart
-      key: building:part
-      type: string
-      from_member: true
-    - name: buildingheight
-      key: building:height
-      type: string
-      from_member: true
-    - name: height
-      key: height
-      type: string
-      from_member: true
-    - name: buildingmin_height
-      key: building:min_height
-      type: string
-      from_member: true
-    - name: min_height
-      key: min_height
-      type: string
-      from_member: true
-    - name: buildinglevels
-      key: building:levels
-      type: string
-      from_member: true
-    - name: levels
-      key: levels
-      type: string
-      from_member: true
-    - name: buildingmin_level
-      key: building:min_level
-      type: string
-      from_member: true
-    - name: min_level
-      key: min_level
-      type: string
-      from_member: true
-    - name: relbuildingheight
-      key: building:height
-      type: string
-    - name: relheight
-      key: height
-      type: string
-    - name: relbuildingmin_height
-      key: building:min_height
-      type: string
-    - name: relmin_height
-      key: min_height
-      type: string
-    - name: relbuildinglevels
-      key: building:levels
-      type: string
-    - name: rellevels
-      key: levels
-      type: string
-    - name: relbuildingmin_level
-      key: building:min_level
-      type: string
-    - name: relmin_level
-      key: min_level
-      type: string
-    - name: member
-      type: member_id
-    - name: index
-      type: member_index
-    - name: role
-      type: member_role
-      from_member: true
-    - name: type
-      type: member_type
-    mapping:
-      type: [street]
-    type: relation_member
-
-  # etldoc: imposm3 -> osm_building_associatedstreet
-  building_associatedstreet:
-    fields:
-    - name: osm_id
-      type: id
-    - name: geometry
-      type: validated_geometry
-    - name: area
-      type: area
-    - name: building
-      key: building
-      type: string
-      from_member: true
-    - name: material
-      key: building:material
-      type: string
-    - name: colour
-      key: building:colour
-      type: string
-    - name: buildingpart
-      key: building:part
-      type: string
-      from_member: true
-    - name: buildingheight
-      key: building:height
-      type: string
-      from_member: true
-    - name: height
-      key: height
-      type: string
-      from_member: true
-    - name: buildingmin_height
-      key: building:min_height
-      type: string
-      from_member: true
-    - name: min_height
-      key: min_height
-      type: string
-      from_member: true
-    - name: buildinglevels
-      key: building:levels
-      type: string
-      from_member: true
-    - name: levels
-      key: levels
-      type: string
-      from_member: true
-    - name: buildingmin_level
-      key: building:min_level
-      type: string
-      from_member: true
-    - name: min_level
-      key: min_level
-      type: string
-      from_member: true
-    - name: relbuildingheight
-      key: building:height
-      type: string
-    - name: relheight
-      key: height
-      type: string
-    - name: relbuildingmin_height
-      key: building:min_height
-      type: string
-    - name: relmin_height
-      key: min_height
-      type: string
-    - name: relbuildinglevels
-      key: building:levels
-      type: string
-    - name: rellevels
-      key: levels
-      type: string
-    - name: relbuildingmin_level
-      key: building:min_level
-      type: string
-    - name: relmin_level
-      key: min_level
-      type: string
-    - name: member
-      type: member_id
-    - name: index
-      type: member_index
-    - name: role
-      type: member_role
-      from_member: true
-    - name: type
-      type: member_type
-    mapping:
-      type: [associatedStreet]
-    type: relation_member
-
   # etldoc: imposm3 -> osm_building_relation
   building_relation:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -342,76 +161,3 @@ tables:
     mapping:
       type: [building]
     type: relation_member
-
-  # etldoc: imposm3 -> osm_building_multipolygon
-  building_multipolygon:
-    fields:
-    - name: osm_id
-      type: id
-    - name: geometry
-      type: validated_geometry
-    - name: area
-      type: area
-    - name: building
-      key: building
-      type: string
-      from_member: true
-    - name: material
-      key: building:material
-      type: string
-    - name: colour
-      key: building:colour
-      type: string
-    - name: buildingpart
-      key: building:part
-      type: string
-      from_member: true
-    - name: buildingheight
-      key: building:height
-      type: string
-      from_member: true
-    - name: height
-      key: height
-      type: string
-      from_member: true
-    - name: buildingmin_height
-      key: building:min_height
-      type: string
-      from_member: true
-    - name: min_height
-      key: min_height
-      type: string
-      from_member: true
-    - name: buildinglevels
-      key: building:levels
-      type: string
-      from_member: true
-    - name: levels
-      key: levels
-      type: string
-      from_member: true
-    - name: buildingmin_level
-      key: building:min_level
-      type: string
-      from_member: true
-    - name: min_level
-      key: min_level
-      type: string
-      from_member: true
-    - name: member
-      type: member_id
-    - name: index
-      type: member_index
-    - name: role
-      type: member_role
-      from_member: true
-    - name: type
-      type: member_type
-    mapping:
-      type: [multipolygon]
-    filters:
-      reject:
-        building: ["no","none","No"]
-        building:part: ["no","none","No"]
-        man_made: ["bridge"]
-    type: relation_member

layers/building/style.json

@@ -0,0 +1,44 @@
+{
+  "layers": [
+    {
+      "id": "building",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "building",
+      "minzoom": 12,
+      "maxzoom": 24,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": {
+          "stops": [
+            [
+              13,
+              "rgba(222, 213, 207, 1)"
+            ],
+            [
+              16,
+              "#d9d0c9"
+            ]
+          ]
+        },
+        "fill-outline-color": {
+          "base": 1,
+          "stops": [
+            [
+              13,
+              "#9A918A"
+            ],
+            [
+              16,
+              "rgba(166, 157, 150, 1)"
+            ]
+          ]
+        }
+      },
+      "metadata": {},
+      "order": 19
+    }
+  ]
+}

layers/building/update_building.sql

@@ -0,0 +1,185 @@
+DROP TRIGGER IF EXISTS trigger_refresh ON buildings.updates;
+DROP TRIGGER IF EXISTS trigger_flag ON osm_building_polygon;
+
+-- Creating aggregated building blocks with removed small polygons and small
+-- holes. Aggregated polygons are simplified by Visvalingam-Whyatt algorithm.
+-- Aggregating is made block by block using country_osm_grid polygon table.
+
+-- Function returning recordset for matview.
+-- Returning recordset of buildings aggregates by zres 14, with removed small
+-- holes and with removed small buildings/blocks.
+
+CREATE OR REPLACE FUNCTION osm_building_block_gen1()
+    RETURNS table
+            (
+                osm_id   bigint,
+                geometry geometry
+            )
+AS
+$$
+DECLARE
+    zres14 float := Zres(14);
+    zres12 float := Zres(12);
+    zres14vw float := Zres(14) * Zres(14);
+    polyg_world record;
+
+BEGIN
+    FOR polyg_world IN 
+        SELECT ST_Transform(country.geometry, 3857) AS geometry 
+        FROM country_osm_grid country
+        
+        LOOP
+            FOR osm_id, geometry IN
+                WITH dta AS ( -- CTE is used because of optimization
+                    SELECT o.osm_id,
+                            o.geometry,
+                            ST_ClusterDBSCAN(o.geometry, eps := zres14, minpoints := 1) OVER () cid
+                    FROM osm_building_polygon o
+                    WHERE ST_Intersects(o.geometry, polyg_world.geometry)
+                )
+                SELECT (array_agg(dta.osm_id))[1] AS osm_id,
+                    ST_Buffer(
+                        ST_Union(
+                            ST_Buffer(
+                                ST_SnapToGrid(dta.geometry, 0.000001)
+                                , zres14, 'join=mitre')
+                            )
+                        , -zres14, 'join=mitre') AS geometry
+                FROM dta
+                GROUP BY cid
+
+                LOOP
+                    -- removing holes smaller than
+                    IF ST_NumInteriorRings(geometry) > 0 THEN -- only from geometries wih holes
+                        geometry := (
+                            -- there are some multi-geometries in this layer
+                            SELECT ST_Collect(gn)
+                            FROM (
+                                    -- in some cases are "holes" NULL, because all holes are smaller than
+                                    SELECT COALESCE(
+                                                -- exterior ring
+                                                    ST_MakePolygon(ST_ExteriorRing(dmp.geom), holes),
+                                                    ST_MakePolygon(ST_ExteriorRing(dmp.geom))
+                                                ) gn
+
+                                    FROM ST_Dump(geometry) dmp, -- 1 dump polygons
+                                        LATERAL (
+                                            SELECT array_agg(ST_Boundary(rg.geom)) holes -- 2 create array
+                                            FROM ST_DumpRings(dmp.geom) rg -- 3 from rings
+                                            WHERE rg.path[1] > 0 -- 5 except inner ring
+                                                AND ST_Area(rg.geom) >= power(zres12, 2) -- 4 bigger than
+                                            ) holes
+                                ) new_geom
+                        );
+                    END IF;
+
+                    IF ST_Area(geometry) < power(zres12, 2) THEN
+                        CONTINUE;
+                    END IF;
+
+                    -- simplify
+                    geometry := ST_SimplifyVW(geometry, zres14vw);
+
+                    RETURN NEXT;
+                END LOOP;
+        END LOOP;
+END;
+$$ LANGUAGE plpgsql STABLE
+                    STRICT
+                    PARALLEL SAFE;
+
+
+DROP MATERIALIZED VIEW IF EXISTS osm_building_block_gen1_dup CASCADE;
+
+CREATE MATERIALIZED VIEW osm_building_block_gen1_dup AS
+SELECT *
+FROM osm_building_block_gen1();
+
+CREATE INDEX ON osm_building_block_gen1_dup USING gist (geometry);
+
+-- etldoc: osm_building_polygon -> osm_building_block_gen_z13
+DROP MATERIALIZED VIEW IF EXISTS osm_building_block_gen_z13;
+CREATE MATERIALIZED VIEW osm_building_block_gen_z13 AS
+(
+WITH 
+    counts AS (
+        SELECT count(osm_id) AS counts,
+		        osm_id
+	    FROM osm_building_block_gen1_dup
+	GROUP BY osm_id
+    ),
+
+    duplicates AS (
+        SELECT counts.osm_id
+	    FROM counts
+	    WHERE counts.counts > 1
+    )
+
+SELECT osm.osm_id,
+		ST_Union(
+            ST_MakeValid(osm.geometry)) AS geometry
+	FROM osm_building_block_gen1_dup osm,
+			duplicates
+	WHERE osm.osm_id = duplicates.osm_id
+	GROUP BY osm.osm_id
+	
+	UNION ALL
+
+	SELECT osm.osm_id, 
+			osm.geometry 
+	FROM osm_building_block_gen1_dup osm, 
+            counts 
+	WHERE counts.counts = 1 
+		AND osm.osm_id = counts.osm_id
+);
+
+CREATE INDEX ON osm_building_block_gen_z13 USING gist (geometry);
+CREATE UNIQUE INDEX ON osm_building_block_gen_z13 USING btree (osm_id);
+
+-- Handle updates
+
+CREATE SCHEMA IF NOT EXISTS buildings;
+
+CREATE TABLE IF NOT EXISTS buildings.updates
+(
+    id serial PRIMARY KEY,
+    t text,
+    UNIQUE (t)
+);
+
+CREATE OR REPLACE FUNCTION buildings.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO buildings.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION buildings.refresh() RETURNS trigger AS
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+    RAISE LOG 'Refresh buildings block';
+    REFRESH MATERIALIZED VIEW osm_building_block_gen1_dup;
+    REFRESH MATERIALIZED VIEW osm_building_block_gen_z13;
+    -- noinspection SqlWithoutWhere
+    DELETE FROM buildings.updates;
+
+    RAISE LOG 'Update buildings block done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_flag
+    AFTER INSERT OR UPDATE OR DELETE
+    ON osm_building_polygon
+    FOR EACH STATEMENT
+EXECUTE PROCEDURE buildings.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+    AFTER INSERT
+    ON buildings.updates
+    INITIALLY DEFERRED
+    FOR EACH ROW
+EXECUTE PROCEDURE buildings.refresh();

layers/housenumber/housenumber.sql

@@ -0,0 +1,33 @@
+-- etldoc: layer_housenumber[shape=record fillcolor=lightpink, style="rounded,filled",
+-- etldoc:     label="layer_housenumber | <z14_> z14+" ] ;
+
+CREATE OR REPLACE FUNCTION layer_housenumber(bbox geometry, zoom_level integer)
+    RETURNS TABLE
+            (
+                osm_id      bigint,
+                geometry    geometry,
+                housenumber text
+            )
+AS
+$$
+SELECT
+    -- etldoc: osm_housenumber_point -> layer_housenumber:z14_
+    osm_id,
+    geometry,
+    display_housenumber(housenumber)
+FROM (
+    SELECT
+        osm_id,
+        geometry,
+        housenumber,
+        row_number() OVER(PARTITION BY concat(street, block_number, housenumber) ORDER BY has_name ASC) as rn
+    FROM osm_housenumber_point
+    WHERE 1=1
+        AND zoom_level >= 14
+        AND geometry && bbox
+) t
+WHERE rn = 1;
+
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/housenumber/housenumber.yaml

@@ -3,17 +3,20 @@ layer:
   description: |
       Everything in OpenStreetMap which contains a `addr:housenumber` tag useful for labelling housenumbers on a map.
       This adds significant size to *z14*. For buildings the centroid of the building is used as housenumber.
+      Duplicates within a tile are dropped if they have the same street/block_number (records without name tag are prioritized for preservation).
   buffer_size: 8
   srs: +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over
   fields:
-    housenumber: Value of the [`addr:housenumber`](http://wiki.openstreetmap.org/wiki/Key:addr) tag.
+    housenumber: Value of the [`addr:housenumber`](http://wiki.openstreetmap.org/wiki/Key:addr) tag. 
+      If there are multiple values separated by semi-colons, the first and last value separated by a dash.
   datasource:
     geometry_field: geometry
     srid: 900913
     query: (SELECT geometry, housenumber FROM layer_housenumber(!bbox!, z(!scale_denominator!))) AS t
 schema:
+  - ./housenumber_display.sql
   - ./housenumber_centroid.sql
-  - ./layer.sql
+  - ./housenumber.sql
 datasources:
   - type: imposm3
     mapping_file: ./mapping.yaml

layers/housenumber/housenumber_centroid.sql

@@ -1,51 +1,102 @@
 DROP TRIGGER IF EXISTS trigger_flag ON osm_housenumber_point;
+DROP TRIGGER IF EXISTS trigger_store ON osm_housenumber_point;
 DROP TRIGGER IF EXISTS trigger_refresh ON housenumber.updates;
 
--- etldoc: osm_housenumber_point -> osm_housenumber_point
-CREATE OR REPLACE FUNCTION convert_housenumber_point() RETURNS VOID AS $$
-BEGIN
-  UPDATE osm_housenumber_point
-  SET geometry =
-           CASE WHEN ST_NPoints(ST_ConvexHull(geometry))=ST_NPoints(geometry)
-           THEN ST_Centroid(geometry)
-           ELSE ST_PointOnSurface(geometry)
-    END
-  WHERE ST_GeometryType(geometry) <> 'ST_Point';
-END;
-$$ LANGUAGE plpgsql;
-
-SELECT convert_housenumber_point();
-
--- Handle updates
-
 CREATE SCHEMA IF NOT EXISTS housenumber;
 
-CREATE TABLE IF NOT EXISTS housenumber.updates(id serial primary key, t text, unique (t));
-CREATE OR REPLACE FUNCTION housenumber.flag() RETURNS trigger AS $$
+CREATE TABLE IF NOT EXISTS housenumber.osm_ids
+(
+    osm_id bigint PRIMARY KEY
+);
+
+-- etldoc: osm_housenumber_point -> osm_housenumber_point
+CREATE OR REPLACE FUNCTION convert_housenumber_point(full_update boolean) RETURNS void AS
+$$
+    UPDATE osm_housenumber_point
+    SET geometry =
+            CASE
+                WHEN ST_NPoints(ST_ConvexHull(geometry)) = ST_NPoints(geometry)
+                    THEN ST_Centroid(geometry)
+                ELSE ST_PointOnSurface(geometry)
+                END
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM housenumber.osm_ids))
+        AND ST_GeometryType(geometry) <> 'ST_Point'
+        AND ST_IsValid(geometry);
+
+    -- we don't need exact name just to know if it's present
+    UPDATE osm_housenumber_point
+    SET has_name = 
+            CASE
+                WHEN has_name = '' THEN '0'
+                ELSE '1'
+            END
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM housenumber.osm_ids));
+
+$$ LANGUAGE SQL;
+
+SELECT convert_housenumber_point(true);
+
+-- Handle updates
+
+CREATE OR REPLACE FUNCTION housenumber.store() RETURNS trigger AS
+$$
 BEGIN
-    INSERT INTO housenumber.updates(t) VALUES ('y')  ON CONFLICT(t) DO NOTHING;
-    RETURN null;
+    INSERT INTO housenumber.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
+    RETURN NULL;
 END;
-$$ language plpgsql;
+$$ LANGUAGE plpgsql;
+
+CREATE TABLE IF NOT EXISTS housenumber.updates
+(
+    id serial PRIMARY KEY,
+    t text,
+    UNIQUE (t)
+);
+CREATE OR REPLACE FUNCTION housenumber.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO housenumber.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION housenumber.refresh() RETURNS trigger AS
-  $BODY$
-  BEGIN
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
     RAISE LOG 'Refresh housenumber';
-    PERFORM convert_housenumber_point();
+
+    -- Analyze tracking and source tables before performing update
+    ANALYZE housenumber.osm_ids;
+    ANALYZE osm_housenumber_point;
+
+    PERFORM convert_housenumber_point(false);
+    -- noinspection SqlWithoutWhere
+    DELETE FROM housenumber.osm_ids;
+    -- noinspection SqlWithoutWhere
     DELETE FROM housenumber.updates;
-    RETURN null;
-  END;
-  $BODY$
-language plpgsql;
+
+    RAISE LOG 'Refresh housenumber done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_store
+    AFTER INSERT OR UPDATE
+    ON osm_housenumber_point
+    FOR EACH ROW
+EXECUTE PROCEDURE housenumber.store();
 
 CREATE TRIGGER trigger_flag
-    AFTER INSERT OR UPDATE OR DELETE ON osm_housenumber_point
+    AFTER INSERT OR UPDATE
+    ON osm_housenumber_point
     FOR EACH STATEMENT
-    EXECUTE PROCEDURE housenumber.flag();
+EXECUTE PROCEDURE housenumber.flag();
 
 CREATE CONSTRAINT TRIGGER trigger_refresh
-    AFTER INSERT ON housenumber.updates
+    AFTER INSERT
+    ON housenumber.updates
     INITIALLY DEFERRED
     FOR EACH ROW
-    EXECUTE PROCEDURE housenumber.refresh();
+EXECUTE PROCEDURE housenumber.refresh();

layers/housenumber/housenumber_display.sql

@@ -0,0 +1,20 @@
+CREATE OR REPLACE FUNCTION display_housenumber_nonnumeric(raw_housenumber text)
+RETURNS text AS $$
+  -- Find the position of the semicolon in the input string
+  -- and extract the first and last value
+  SELECT substring(raw_housenumber from 1 for position(';' in raw_housenumber) - 1)
+         || '–'
+         || substring(raw_housenumber from position(';' in raw_housenumber) + 1);
+$$ LANGUAGE SQL IMMUTABLE;
+
+
+CREATE OR REPLACE FUNCTION display_housenumber(raw_housenumber text)
+RETURNS text AS $$
+  SELECT CASE
+    WHEN raw_housenumber !~ ';' THEN raw_housenumber
+    WHEN raw_housenumber ~ '[^0-9;]' THEN display_housenumber_nonnumeric(raw_housenumber)
+    ELSE
+      (SELECT min(value)::text || '–' || max(value)::text
+        FROM unnest(array_remove(string_to_array(raw_housenumber, ';'), '')::bigint[]) AS value)
+  END
+$$ LANGUAGE SQL IMMUTABLE;

layers/housenumber/layer.sql

@@ -1,10 +0,0 @@
-
--- etldoc: layer_housenumber[shape=record fillcolor=lightpink, style="rounded,filled",
--- etldoc:     label="layer_housenumber | <z14_> z14+" ] ;
-
-CREATE OR REPLACE FUNCTION layer_housenumber(bbox geometry, zoom_level integer)
-RETURNS TABLE(osm_id bigint, geometry geometry, housenumber text) AS $$
-   -- etldoc: osm_housenumber_point -> layer_housenumber:z14_
-    SELECT osm_id, geometry, housenumber FROM osm_housenumber_point
-    WHERE zoom_level >= 14 AND geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE;

layers/housenumber/mapping.yaml

@@ -4,7 +4,7 @@ tables:
   # etldoc: imposm3 -> osm_housenumber_point
   housenumber_point:
     type: geometry
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -12,6 +12,15 @@ tables:
     - name: housenumber
       key: addr:housenumber
       type: string
+    - name: street
+      key: addr:street
+      type: string
+    - name: block_number
+      key: addr:block_number
+      type: string
+    - name: has_name
+      key: name
+      type: string
     type_mappings:
       points:
         addr:housenumber:

layers/housenumber/style.json

@@ -0,0 +1,40 @@
+{
+  "layers": [
+    {
+      "id": "housenumber",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "housenumber",
+      "minzoom": 17,
+      "layout": {
+        "text-font": [
+          "Noto Sans Regular"
+        ],
+        "text-size": {
+          "stops": [
+            [
+              17,
+              9
+            ],
+            [
+              22,
+              11
+            ]
+          ]
+        },
+        "text-field": "{housenumber}",
+        "text-padding": 3,
+        "text-line-height": -0.15,
+        "symbol-avoid-edges": false,
+        "text-allow-overlap": false,
+        "text-ignore-placement": false
+      },
+      "paint": {
+        "text-color": "rgba(102, 102, 102, 1)",
+        "text-halo-color": "rgba(255,255,255,0.8)",
+        "text-halo-width": 1
+      },
+      "order": 154
+    }
+  ]
+}

layers/landcover/generalized.sql

@@ -0,0 +1,287 @@
+DROP TABLE IF EXISTS osm_landcover_gen_z7;
+DROP TABLE IF EXISTS osm_landcover_gen_z8;
+DROP TABLE IF EXISTS osm_landcover_gen_z9;
+DROP TABLE IF EXISTS osm_landcover_gen_z10;
+DROP TABLE IF EXISTS osm_landcover_gen_z11;
+DROP TABLE IF EXISTS osm_landcover_gen_z12;
+DROP TABLE IF EXISTS osm_landcover_gen_z13;
+DROP TABLE IF EXISTS simplify_vw_z7 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z8 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z9 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z10 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z11 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z12 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z13 CASCADE;
+
+-- etldoc: osm_landcover_polygon ->  simplify_vw_z13
+CREATE TABLE simplify_vw_z13 AS
+(
+    SELECT subclass,
+           ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(13),2)),
+             0.001)) AS geometry
+    FROM osm_landcover_polygon
+    WHERE ST_Area(geometry) > power(zres(12),2)
+);
+CREATE INDEX ON simplify_vw_z13 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z13 ->  osm_landcover_gen_z13
+CREATE TABLE osm_landcover_gen_z13 AS
+(
+    SELECT subclass, ST_MakeValid((ST_dump(ST_Union(geometry))).geom) AS geometry
+    FROM (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) over () AS cid, geometry
+        FROM simplify_vw_z13
+        WHERE ST_NPoints(geometry) < 300
+          AND subclass IN ('wood', 'forest')) union_geom300
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           geometry
+    FROM simplify_vw_z13
+    WHERE (ST_NPoints(geometry) >= 300 AND subclass IN ('wood', 'forest'))
+       OR (subclass NOT IN ('wood', 'forest'))
+    );
+
+CREATE INDEX ON osm_landcover_gen_z13 USING GIST (geometry);
+
+
+-- etldoc: simplify_vw_z13 ->  simplify_vw_z12
+CREATE TABLE simplify_vw_z12 AS
+(
+    SELECT subclass,
+           ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(12),2)),
+             0.001)) AS geometry
+    FROM simplify_vw_z13
+    WHERE ST_Area(geometry) > power(zres(11),2)
+);
+CREATE INDEX ON simplify_vw_z12 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z12 ->  osm_landcover_gen_z12
+CREATE TABLE osm_landcover_gen_z12 AS
+(
+    SELECT subclass, ST_MakeValid((ST_dump(ST_Union(geometry))).geom) AS geometry
+    FROM (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) over () AS cid, geometry
+        FROM simplify_vw_z12
+        WHERE ST_NPoints(geometry) < 300
+          AND subclass IN ('wood', 'forest')) union_geom300
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           geometry
+    FROM simplify_vw_z12
+    WHERE (ST_NPoints(geometry) >= 300  AND subclass IN ('wood', 'forest'))
+       OR (subclass NOT IN ('wood', 'forest'))
+    );
+
+CREATE INDEX ON osm_landcover_gen_z12 USING GIST (geometry);
+
+
+-- etldoc: simplify_vw_z12 ->  simplify_vw_z11
+CREATE TABLE simplify_vw_z11 AS
+(
+    SELECT subclass,
+            ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(11),2)),
+             0.001)) AS geometry
+    FROM simplify_vw_z12
+    WHERE ST_Area(geometry) > power(zres(10),2)
+);
+CREATE INDEX ON simplify_vw_z11 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z11 ->  osm_landcover_gen_z11
+CREATE TABLE osm_landcover_gen_z11 AS
+(
+    SELECT subclass, ST_MakeValid((ST_dump(ST_Union(geometry))).geom) AS geometry
+    FROM (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) over () AS cid, geometry
+        FROM simplify_vw_z11
+        WHERE ST_NPoints(geometry) < 300
+          AND subclass IN ('wood', 'forest')) union_geom300
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           geometry
+    FROM simplify_vw_z11
+    WHERE (ST_NPoints(geometry) >= 300 AND subclass IN ('wood', 'forest'))
+       OR (subclass NOT IN ('wood', 'forest'))
+    );
+
+CREATE INDEX ON osm_landcover_gen_z11 USING GIST (geometry);
+
+
+-- etldoc: simplify_vw_z11 ->  simplify_vw_z10
+CREATE TABLE simplify_vw_z10 AS
+(
+    SELECT subclass,
+           ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(10),2)),
+             0.001)) AS geometry
+    FROM simplify_vw_z11
+    WHERE ST_Area(geometry) > power(zres(9),2)
+);
+CREATE INDEX ON simplify_vw_z10 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z10 ->  osm_landcover_gen_z10
+CREATE TABLE osm_landcover_gen_z10 AS
+(
+    SELECT subclass, ST_MakeValid((ST_dump(ST_Union(geometry))).geom) AS geometry
+    FROM (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) over () AS cid, geometry
+        FROM simplify_vw_z10
+        WHERE ST_NPoints(geometry) < 300
+          AND subclass IN ('wood', 'forest')) union_geom300
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           geometry
+    FROM simplify_vw_z10
+    WHERE (ST_NPoints(geometry) >= 300 AND subclass IN ('wood', 'forest'))
+       OR (subclass NOT IN ('wood', 'forest'))
+    );
+
+CREATE INDEX ON osm_landcover_gen_z10 USING GIST (geometry);
+
+
+-- etldoc: simplify_vw_z10 ->  simplify_vw_z9
+CREATE TABLE simplify_vw_z9 AS
+(
+    SELECT subclass,
+           ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(9),2)),
+             0.001)) AS geometry
+    FROM simplify_vw_z10
+    WHERE ST_Area(geometry) > power(zres(8),2)
+);
+CREATE INDEX ON simplify_vw_z9 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z9 ->  osm_landcover_gen_z9
+CREATE TABLE osm_landcover_gen_z9 AS
+(
+    SELECT subclass, ST_MakeValid((ST_dump(ST_Union(geometry))).geom) AS geometry
+    FROM (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) over () AS cid, geometry
+        FROM simplify_vw_z9
+        WHERE ST_NPoints(geometry) < 300
+          AND subclass IN ('wood', 'forest')) union_geom300
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           ST_MakeValid(
+            (ST_Dump(
+             ST_Union(geometry))).geom) AS geometry
+    FROM (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) over () AS cid, geometry
+        FROM simplify_vw_z9
+        WHERE ST_NPoints(geometry) >= 300
+          AND subclass IN ('wood', 'forest')) union_geom_rest
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           geometry
+    FROM simplify_vw_z9
+    WHERE subclass NOT IN ('wood', 'forest')
+    );
+
+CREATE INDEX ON osm_landcover_gen_z9 USING GIST (geometry);
+
+
+-- etldoc: simplify_vw_z9 ->  simplify_vw_z8
+CREATE TABLE simplify_vw_z8 AS
+(
+    SELECT subclass,
+           ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(8),2)),
+             0.001)) AS geometry
+    FROM simplify_vw_z9
+    WHERE ST_Area(geometry) > power(zres(7),2)
+    );
+CREATE INDEX ON simplify_vw_z8 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z8 ->  osm_landcover_gen_z8
+CREATE TABLE osm_landcover_gen_z8 AS
+(
+SELECT subclass,
+       ST_MakeValid(
+        (ST_Dump(
+         ST_Union(geometry))).geom) AS geometry
+    FROM
+        (
+        SELECT subclass,
+               ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) OVER () AS cid,
+               geometry
+        FROM simplify_vw_z8
+        WHERE subclass IN ('wood', 'forest')
+        ) union_geom
+    GROUP BY subclass,
+             cid
+    UNION ALL
+    SELECT subclass,
+           geometry
+    FROM simplify_vw_z8
+    WHERE subclass NOT IN ('wood', 'forest')
+    );
+
+CREATE INDEX ON osm_landcover_gen_z8 USING GIST (geometry);
+
+
+-- etldoc: simplify_vw_z8 ->  simplify_vw_z7
+CREATE TABLE simplify_vw_z7 AS
+(
+    SELECT subclass,
+           ST_MakeValid(
+            ST_SnapToGrid(
+             ST_SimplifyVW(geometry, power(zres(7),2)),
+             0.001)) AS geometry
+    FROM simplify_vw_z8
+    WHERE ST_Area(geometry) > power(zres(6),2)
+);
+CREATE INDEX ON simplify_vw_z7 USING GIST (geometry);
+
+-- etldoc: simplify_vw_z7 ->  osm_landcover_gen_z7
+CREATE TABLE osm_landcover_gen_z7 AS
+(
+SELECT subclass,
+       ST_MakeValid(
+        (ST_Dump(
+         ST_Union(geometry))).geom) AS geometry
+    FROM
+        (
+        SELECT  subclass,
+                ST_ClusterDBSCAN(geometry, eps := 0, minpoints := 1) OVER () AS cid,
+                geometry
+        FROM simplify_vw_z7
+        ) union_geom
+GROUP BY subclass,
+         cid
+    );
+
+CREATE INDEX ON osm_landcover_gen_z7 USING GIST (geometry);
+
+DROP TABLE IF EXISTS simplify_vw_z7 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z8 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z9 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z10 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z11 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z12 CASCADE;
+DROP TABLE IF EXISTS simplify_vw_z13 CASCADE;

layers/landcover/landcover.sql

@@ -9,132 +9,365 @@
 --);
 --CREATE INDEX IF NOT EXISTS landcover_grouped_gen2_geometry_idx ON landcover_grouped_gen2 USING gist(geometry);
 
-CREATE OR REPLACE FUNCTION landcover_class(subclass VARCHAR) RETURNS TEXT AS $$
-    SELECT CASE
-        WHEN subclass IN ('farmland', 'farm', 'orchard', 'vineyard', 'plant_nursery') THEN 'farmland'
-        WHEN subclass IN ('glacier', 'ice_shelf') THEN 'ice'
-        WHEN subclass IN ('wood', 'forest') THEN 'wood'
-        WHEN subclass IN ('bare_rock', 'scree') THEN 'rock'
-        WHEN subclass IN ('fell', 'grassland', 'heath', 'scrub', 'tundra', 'grass', 'meadow', 'allotments',
-                          'park', 'village_green', 'recreation_ground', 'garden', 'golf_course') THEN 'grass'
-        WHEN subclass IN ('wetland', 'bog', 'swamp', 'wet_meadow', 'marsh', 'reedbed',
-                          'saltern', 'tidalflat', 'saltmarsh', 'mangrove') THEN 'wetland'
-        WHEN subclass IN ('beach', 'sand', 'dune') THEN 'sand'
-        ELSE NULL
-    END;
-$$ LANGUAGE SQL IMMUTABLE;
+CREATE OR REPLACE FUNCTION landcover_class(subclass varchar) RETURNS text AS
+$$
+SELECT CASE
+           %%FIELD_MAPPING: class %%
+           END;
+$$ LANGUAGE SQL IMMUTABLE
+                -- STRICT
+                PARALLEL SAFE;
 
--- etldoc: ne_110m_glaciated_areas ->  landcover_z0
-CREATE OR REPLACE VIEW landcover_z0 AS (
-    SELECT NULL::bigint AS osm_id, geometry, 'glacier'::text AS subclass FROM ne_110m_glaciated_areas
-);
+-- ne_50m_antarctic_ice_shelves_polys
+-- etldoc: ne_50m_antarctic_ice_shelves_polys ->  ne_50m_antarctic_ice_shelves_polys_gen_z4
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_antarctic_ice_shelves_polys_gen_z4 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_antarctic_ice_shelves_polys_gen_z4 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(6)) as geometry,
+    'ice_shelf'::text AS subclass
+FROM ne_50m_antarctic_ice_shelves_polys
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_antarctic_ice_shelves_polys_gen_z4_idx ON ne_50m_antarctic_ice_shelves_polys_gen_z4 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z2 AS (
-    -- etldoc: ne_50m_glaciated_areas ->  landcover_z2
-    SELECT NULL::bigint AS osm_id, geometry, 'glacier'::text AS subclass FROM ne_50m_glaciated_areas
-    UNION ALL
-    -- etldoc: ne_50m_antarctic_ice_shelves_polys ->  landcover_z2
-    SELECT NULL::bigint AS osm_id, geometry, 'ice_shelf'::text AS subclass FROM ne_50m_antarctic_ice_shelves_polys
-);
+-- ne_110m_glaciated_areas
+-- etldoc: ne_110m_glaciated_areas ->  ne_110m_glaciated_areas_gen_z1
+DROP MATERIALIZED VIEW IF EXISTS ne_110m_glaciated_areas_gen_z1 CASCADE;
+CREATE MATERIALIZED VIEW ne_110m_glaciated_areas_gen_z1 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(3)) as geometry,
+    'glacier'::text AS subclass
+FROM ne_110m_glaciated_areas
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_110m_glaciated_areas_gen_z1_idx ON ne_110m_glaciated_areas_gen_z1 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z5 AS (
-    -- etldoc: ne_10m_glaciated_areas ->  landcover_z5
-    SELECT NULL::bigint AS osm_id, geometry, 'glacier'::text AS subclass FROM ne_10m_glaciated_areas
-    UNION ALL
-    -- etldoc: ne_10m_antarctic_ice_shelves_polys ->  landcover_z5
-    SELECT NULL::bigint AS osm_id, geometry, 'ice_shelf'::text AS subclass FROM ne_10m_antarctic_ice_shelves_polys
-);
+-- etldoc: ne_110m_glaciated_areas_gen_z1 ->  ne_110m_glaciated_areas_gen_z0
+DROP MATERIALIZED VIEW IF EXISTS ne_110m_glaciated_areas_gen_z0 CASCADE;
+CREATE MATERIALIZED VIEW ne_110m_glaciated_areas_gen_z0 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(2)) as geometry,
+    subclass
+FROM ne_110m_glaciated_areas_gen_z1
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_110m_glaciated_areas_gen_z0_idx ON ne_110m_glaciated_areas_gen_z0 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z7 AS (
-    -- etldoc: osm_landcover_polygon_gen7 ->  landcover_z7
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen7
-);
+-- etldoc: ne_50m_antarctic_ice_shelves_polys_gen_z4 ->  ne_50m_antarctic_ice_shelves_polys_gen_z3
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_antarctic_ice_shelves_polys_gen_z3 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_antarctic_ice_shelves_polys_gen_z3 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(5)) as geometry,
+    subclass
+FROM ne_50m_antarctic_ice_shelves_polys_gen_z4
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_antarctic_ice_shelves_polys_gen_z3_idx ON ne_50m_antarctic_ice_shelves_polys_gen_z3 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z8 AS (
-    -- etldoc: osm_landcover_polygon_gen6 ->  landcover_z8
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen6
-);
+-- etldoc: ne_50m_antarctic_ice_shelves_polys_gen_z3 ->  ne_50m_antarctic_ice_shelves_polys_gen_z2
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_antarctic_ice_shelves_polys_gen_z2 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_antarctic_ice_shelves_polys_gen_z2 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(4)) as geometry,
+    subclass
+FROM ne_50m_antarctic_ice_shelves_polys_gen_z3
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_antarctic_ice_shelves_polys_gen_z2_idx ON ne_50m_antarctic_ice_shelves_polys_gen_z2 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z9 AS (
-    -- etldoc: osm_landcover_polygon_gen5 ->  landcover_z9
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen5
-);
+-- ne_50m_glaciated_areas
+-- etldoc: ne_50m_glaciated_areas ->  ne_50m_glaciated_areas_gen_z4
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_glaciated_areas_gen_z4 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_glaciated_areas_gen_z4 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(6)) as geometry,
+    'glacier'::text AS subclass
+FROM ne_50m_glaciated_areas
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_glaciated_areas_gen_z4_idx ON ne_50m_glaciated_areas_gen_z4 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z10 AS (
-    -- etldoc: osm_landcover_polygon_gen4 ->  landcover_z10
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen4
-);
+-- etldoc: ne_50m_glaciated_areas_gen_z4 ->  ne_50m_glaciated_areas_gen_z3
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_glaciated_areas_gen_z3 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_glaciated_areas_gen_z3 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(5)) as geometry,
+    subclass
+FROM ne_50m_glaciated_areas_gen_z4
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_glaciated_areas_gen_z3_idx ON ne_50m_glaciated_areas_gen_z3 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z11 AS (
-    -- etldoc: osm_landcover_polygon_gen3 ->  landcover_z11
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen3
-);
+-- etldoc: ne_50m_glaciated_areas_gen_z3 ->  ne_50m_glaciated_areas_gen_z2
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_glaciated_areas_gen_z2 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_glaciated_areas_gen_z2 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(4)) as geometry,
+    subclass
+FROM ne_50m_glaciated_areas_gen_z3
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_glaciated_areas_gen_z2_idx ON ne_50m_glaciated_areas_gen_z2 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z12 AS (
-    -- etldoc: osm_landcover_polygon_gen2 ->  landcover_z12
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen2
-);
+-- ne_10m_glaciated_areas
+-- etldoc: ne_10m_glaciated_areas ->  ne_10m_glaciated_areas_gen_z6
+DROP MATERIALIZED VIEW IF EXISTS ne_10m_glaciated_areas_gen_z6 CASCADE;
+CREATE MATERIALIZED VIEW ne_10m_glaciated_areas_gen_z6 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(8)) as geometry,
+    'glacier'::text AS subclass
+FROM ne_10m_glaciated_areas
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_10m_glaciated_areas_gen_z6_idx ON ne_10m_glaciated_areas_gen_z6 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z13 AS (
-    -- etldoc: osm_landcover_polygon_gen1 ->  landcover_z13
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon_gen1
-);
+-- etldoc: ne_10m_glaciated_areas_gen_z6 ->  ne_10m_glaciated_areas_gen_z5
+DROP MATERIALIZED VIEW IF EXISTS ne_10m_glaciated_areas_gen_z5 CASCADE;
+CREATE MATERIALIZED VIEW ne_10m_glaciated_areas_gen_z5 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(7)) as geometry,
+    subclass
+FROM ne_10m_glaciated_areas_gen_z6
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_10m_glaciated_areas_gen_z5_idx ON ne_10m_glaciated_areas_gen_z5 USING gist (geometry);
 
-CREATE OR REPLACE VIEW landcover_z14 AS (
-    -- etldoc: osm_landcover_polygon ->  landcover_z14
-    SELECT osm_id, geometry, subclass FROM osm_landcover_polygon
-);
+-- ne_10m_antarctic_ice_shelves_polys
+-- etldoc: ne_10m_antarctic_ice_shelves_polys ->  ne_10m_antarctic_ice_shelves_polys_gen_z6
+DROP MATERIALIZED VIEW IF EXISTS ne_10m_antarctic_ice_shelves_polys_gen_z6 CASCADE;
+CREATE MATERIALIZED VIEW ne_10m_antarctic_ice_shelves_polys_gen_z6 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(8)) as geometry,
+    'ice_shelf'::text AS subclass
+FROM ne_10m_antarctic_ice_shelves_polys
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_10m_antarctic_ice_shelves_polys_gen_z6_idx ON ne_10m_antarctic_ice_shelves_polys_gen_z6 USING gist (geometry);
 
--- etldoc: layer_landcover[shape=record fillcolor=lightpink, style="rounded, filled", label="layer_landcover | <z0_1> z0-z1 | <z2_4> z2-z4 | <z5_6> z5-z6 |<z7> z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14+" ] ;
+-- etldoc: ne_10m_antarctic_ice_shelves_polys_gen_z6 ->  ne_10m_antarctic_ice_shelves_polys_gen_z5
+DROP MATERIALIZED VIEW IF EXISTS ne_10m_antarctic_ice_shelves_polys_gen_z5 CASCADE;
+CREATE MATERIALIZED VIEW ne_10m_antarctic_ice_shelves_polys_gen_z5 AS
+(
+SELECT
+    ST_Simplify(geometry, ZRes(7)) as geometry,
+    subclass
+FROM ne_10m_antarctic_ice_shelves_polys_gen_z6
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_10m_antarctic_ice_shelves_polys_gen_z5_idx ON ne_10m_antarctic_ice_shelves_polys_gen_z5 USING gist (geometry);
+
+-- etldoc: ne_110m_glaciated_areas_gen_z0 ->  landcover_z0
+CREATE OR REPLACE VIEW landcover_z0 AS
+(
+SELECT
+    geometry,
+    subclass
+FROM ne_110m_glaciated_areas_gen_z0
+    );
+
+-- etldoc: ne_110m_glaciated_areas_gen_z1 ->  landcover_z1
+CREATE OR REPLACE VIEW landcover_z1 AS
+(
+SELECT
+    geometry,
+    subclass
+FROM ne_110m_glaciated_areas_gen_z1
+    );
+
+CREATE OR REPLACE VIEW landcover_z2 AS
+(
+-- etldoc: ne_50m_glaciated_areas_gen_z2 ->  landcover_z2
+SELECT
+    geometry,
+    subclass
+FROM ne_50m_glaciated_areas_gen_z2
+UNION ALL
+-- etldoc: ne_50m_antarctic_ice_shelves_polys_gen_z2 ->  landcover_z2
+SELECT
+    geometry,
+    subclass
+FROM ne_50m_antarctic_ice_shelves_polys_gen_z2
+    );
+
+CREATE OR REPLACE VIEW landcover_z3 AS
+(
+-- etldoc: ne_50m_glaciated_areas_gen_z3 ->  landcover_z3
+SELECT
+    geometry,
+    subclass
+FROM ne_50m_glaciated_areas_gen_z3
+UNION ALL
+-- etldoc: ne_50m_antarctic_ice_shelves_polys_gen_z3 ->  landcover_z3
+SELECT
+    geometry,
+    subclass
+FROM ne_50m_antarctic_ice_shelves_polys_gen_z3
+    );
+
+CREATE OR REPLACE VIEW landcover_z4 AS
+(
+-- etldoc: ne_50m_glaciated_areas_gen_z4 ->  landcover_z4
+SELECT
+    geometry,
+    subclass
+FROM ne_50m_glaciated_areas_gen_z4
+UNION ALL
+-- etldoc: ne_50m_antarctic_ice_shelves_polys_gen_z4 ->  landcover_z4
+SELECT
+    geometry,
+    subclass
+FROM ne_50m_antarctic_ice_shelves_polys_gen_z4
+    );
+
+CREATE OR REPLACE VIEW landcover_z5 AS
+(
+-- etldoc: ne_10m_glaciated_areas_gen_z5 ->  landcover_z5
+SELECT
+    geometry,
+    subclass
+FROM ne_10m_glaciated_areas_gen_z5
+UNION ALL
+-- etldoc: ne_10m_antarctic_ice_shelves_polys_gen_z5 ->  landcover_z5
+SELECT
+    geometry,
+    subclass
+FROM ne_10m_antarctic_ice_shelves_polys_gen_z5
+    );
+
+CREATE OR REPLACE VIEW landcover_z6 AS
+(
+-- etldoc: ne_10m_glaciated_areas_gen_z6 ->  landcover_z6
+SELECT
+    geometry,
+    subclass
+FROM ne_10m_glaciated_areas_gen_z6
+UNION ALL
+-- etldoc: ne_10m_antarctic_ice_shelves_polys_gen_z6 ->  landcover_z6
+SELECT
+    geometry,
+    subclass
+FROM ne_10m_antarctic_ice_shelves_polys_gen_z6
+    );
+
+-- etldoc: layer_landcover[shape=record fillcolor=lightpink, style="rounded, filled", label="layer_landcover | <z0> z0 | <z1> z1 | <z2> z2 | <z3> z3 | <z4> z4 | <z5> z5 | <z6> z6 |<z7> z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_landcover(bbox geometry, zoom_level int)
-RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text) AS $$
-    SELECT osm_id, geometry,
-        landcover_class(subclass) AS class,
-        subclass
-        FROM (
-        -- etldoc:  landcover_z0 -> layer_landcover:z0_1
-        SELECT * FROM landcover_z0
-        WHERE zoom_level BETWEEN 0 AND 1 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z2 -> layer_landcover:z2_4
-        SELECT * FROM landcover_z2
-        WHERE zoom_level BETWEEN 2 AND 4 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z5 -> layer_landcover:z5_6
-        SELECT * FROM landcover_z5
-        WHERE zoom_level BETWEEN 5 AND 6 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z7 -> layer_landcover:z7
-        SELECT *
-        FROM landcover_z7 WHERE zoom_level = 7 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z8 -> layer_landcover:z8
-        SELECT *
-        FROM landcover_z8 WHERE zoom_level = 8 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z9 -> layer_landcover:z9
-        SELECT *
-        FROM landcover_z9 WHERE zoom_level = 9 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z10 -> layer_landcover:z10
-        SELECT *
-        FROM landcover_z10 WHERE zoom_level = 10 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z11 -> layer_landcover:z11
-        SELECT *
-        FROM landcover_z11 WHERE zoom_level = 11 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z12 -> layer_landcover:z12
-        SELECT *
-        FROM landcover_z12 WHERE zoom_level = 12 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z13 -> layer_landcover:z13
-        SELECT *
-        FROM landcover_z13 WHERE zoom_level = 13 AND geometry && bbox
-        UNION ALL
-        -- etldoc:  landcover_z14 -> layer_landcover:z14_
-        SELECT *
-        FROM landcover_z14 WHERE zoom_level >= 14 AND geometry && bbox
-    ) AS zoom_levels;
-$$ LANGUAGE SQL IMMUTABLE;
+    RETURNS TABLE
+            (
+                geometry geometry,
+                class    text,
+                subclass text
+            )
+AS
+$$
+SELECT geometry,
+       landcover_class(subclass) AS class,
+       subclass
+FROM (
+         -- etldoc:  landcover_z0 -> layer_landcover:z0
+         SELECT geometry, 
+                subclass
+         FROM landcover_z0
+         WHERE zoom_level = 0
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  landcover_z1 -> layer_landcover:z1
+         SELECT geometry,
+                subclass
+         FROM landcover_z1
+         WHERE zoom_level = 1
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  landcover_z2 -> layer_landcover:z2
+         SELECT geometry, 
+                subclass
+         FROM landcover_z2
+         WHERE zoom_level = 2
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  landcover_z3 -> layer_landcover:z3
+         SELECT geometry,
+                subclass
+         FROM landcover_z3
+         WHERE zoom_level = 3
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  landcover_z4 -> layer_landcover:z4
+         SELECT geometry,
+                subclass
+         FROM landcover_z4
+         WHERE zoom_level = 4
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  landcover_z5 -> layer_landcover:z5
+         SELECT geometry, 
+                subclass
+         FROM landcover_z5
+         WHERE zoom_level = 5
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  landcover_z6 -> layer_landcover:z6
+         SELECT geometry,
+                subclass
+         FROM landcover_z6
+         WHERE zoom_level = 6
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z7 -> layer_landcover:z7
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z7
+         WHERE zoom_level = 7
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z8 -> layer_landcover:z8
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z8
+         WHERE zoom_level = 8
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z9 -> layer_landcover:z9
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z9
+         WHERE zoom_level = 9
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z10 -> layer_landcover:z10
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z10
+         WHERE zoom_level = 10
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z11 -> layer_landcover:z11
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z11
+         WHERE zoom_level = 11
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z12 -> layer_landcover:z12
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z12
+         WHERE zoom_level = 12
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_gen_z13 -> layer_landcover:z13
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_gen_z13
+         WHERE zoom_level = 13
+           AND geometry && bbox
+         UNION ALL
+         -- etldoc:  osm_landcover_polygon -> layer_landcover:z14_
+         SELECT geometry, 
+                subclass
+         FROM osm_landcover_polygon
+         WHERE zoom_level >= 14
+           AND geometry && bbox
+     ) AS zoom_levels;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/landcover/landcover.yaml

@@ -1,5 +1,12 @@
 layer:
   id: "landcover"
+  requires:
+    tables:
+      - ne_10m_antarctic_ice_shelves_polys
+      - ne_10m_glaciated_areas
+      - ne_50m_antarctic_ice_shelves_polys
+      - ne_50m_glaciated_areas
+      - ne_110m_glaciated_areas
   description: |
       Landcover is used to describe the physical material at the surface of the earth. At lower zoom levels this is
       from Natural Earth data for glaciers and ice shelves and at higher zoom levels the landcover is [implied by OSM tags](http://wiki.openstreetmap.org/wiki/Landcover). The most common use case for this layer
@@ -10,12 +17,20 @@ layer:
       description: |
           Use the **class** to assign natural colors for **landcover**.
       values:
-      - farmland
-      - ice
-      - wood
-      - grass
-      - wetland
-      - sand
+        farmland:
+          subclass: ['farmland', 'farm', 'orchard', 'vineyard', 'plant_nursery']
+        ice:
+          subclass: ['glacier', 'ice_shelf']
+        wood:
+          subclass: ['wood', 'forest']
+        rock:
+          subclass: ['bare_rock', 'scree']
+        grass:
+          subclass: ['fell', 'grassland', 'heath', 'scrub', 'shrubbery', 'tundra', 'grass', 'meadow', 'allotments', 'park', 'village_green', 'recreation_ground', 'garden', 'golf_course']
+        wetland:
+          subclass: ['wetland', 'bog', 'swamp', 'wet_meadow', 'marsh', 'reedbed', 'saltern', 'tidalflat', 'saltmarsh', 'mangrove']
+        sand:
+          subclass: ['beach', 'sand', 'dune']
     subclass:
       description: |
           Use **subclass** to do more precise styling.
@@ -31,6 +46,7 @@ layer:
       - bog
       - dune
       - scrub
+      - shrubbery
       - farm
       - farmland
       - fell
@@ -65,6 +81,7 @@ layer:
     geometry_field: geometry
     query: (SELECT geometry, class, subclass FROM layer_landcover(!bbox!, z(!scale_denominator!))) AS t
 schema:
+  - ./generalized.sql
   - ./landcover.sql
 datasources:
   - type: imposm3

layers/landcover/mapping.yaml

@@ -1,52 +1,7 @@
-
-generalized_tables:
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen7
-  landcover_polygon_gen7:
-    source: landcover_polygon_gen6
-    sql_filter: area>power(ZRES5,2)
-    tolerance: ZRES7
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen6
-  landcover_polygon_gen6:
-    source: landcover_polygon_gen5
-    sql_filter: area>power(ZRES6,2)
-    tolerance: ZRES8
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen5
-  landcover_polygon_gen5:
-    source: landcover_polygon_gen4
-    sql_filter: area>power(ZRES7,2)
-    tolerance: ZRES9
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen4
-  landcover_polygon_gen4:
-    source: landcover_polygon_gen3
-    sql_filter: area>power(ZRES8,2)
-    tolerance: ZRES10
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen3
-  landcover_polygon_gen3:
-    source: landcover_polygon_gen2
-    sql_filter: area>power(ZRES8,2)
-    tolerance: ZRES11
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen2
-  landcover_polygon_gen2:
-    source: landcover_polygon_gen1
-    sql_filter: area>power(ZRES9,2)
-    tolerance: ZRES12
-
-  # etldoc: imposm3 -> osm_landcover_polygon_gen1
-  landcover_polygon_gen1:
-    source: landcover_polygon
-    sql_filter: area>power(ZRES10,2) AND ST_IsValid(geometry)
-    tolerance: ZRES13
-
 tables:
   # etldoc: imposm3 -> osm_landcover_polygon
   landcover_polygon:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -71,8 +26,6 @@ tables:
       - forest
       - village_green
       - recreation_ground
-      # There are 600 parks tagged with landuse=park instead of leisure=park
-      - park
       natural:
       - wood
       - wetland
@@ -80,6 +33,7 @@ tables:
       - grassland
       - heath
       - scrub
+      - shrubbery
       - tundra
       - glacier
       - bare_rock

layers/landcover/style.json

@@ -0,0 +1,476 @@
+{
+  "layers": [
+    {
+      "id": "landcover_classes",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "maxzoom": 13,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "farmland",
+          "#eef0d5",
+          "wood",
+          "#add19e",
+          "rock",
+          "#eee5dc",
+          "grass",
+          "#cdebb0",
+          "sand",
+          "#f5e9c6",
+          "wetland",
+          "#add19e",
+          "#000"
+        ],
+        "fill-opacity": {
+          "stops": [
+            [
+              7,
+              0.5
+            ],
+            [
+              10,
+              1
+            ]
+          ]
+        },
+        "fill-antialias": false
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "farmland",
+          "wood",
+          "rock",
+          "grass",
+          "wetland",
+          "sand"
+        ]
+      ],
+      "order": 4
+    },
+    {
+      "id": "landcover_class_outline",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#c7c9ae",
+        "line-width": 0.5
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "farmland"
+        ]
+      ],
+      "order": 5
+    },
+    {
+      "id": "landcover_park",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "minzoom": 13,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": "#c8facc",
+        "fill-antialias": true
+      },
+      "filter": [
+        "all",
+        [
+          "==",
+          "subclass",
+          "park"
+        ]
+      ],
+      "order": 6
+    },
+    {
+      "id": "landcover_subclasses",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "minzoom": 13,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": [
+          "match",
+          [
+            "get",
+            "subclass"
+          ],
+          "allotments",
+          "#c9e1bf",
+          "bare_rock",
+          "#eee5dc",
+          "beach",
+          "#fff1ba",
+          "bog",
+          "#d6d99f",
+          "dune",
+          "#f5e9c6",
+          "scrub",
+          "#c8d7ab",
+          "farm",
+          "#f5dcba",
+          "farmland",
+          "#eef0d5",
+          "forest",
+          "#add19e",
+          "grass",
+          "#cdebb0",
+          "grassland",
+          "#cdebb0",
+          "golf_course",
+          "#def6c0",
+          "heath",
+          "#d6d99f",
+          "mangrove",
+          "#c8d7ab",
+          "meadow",
+          "#cdebb0",
+          "orchard",
+          "#aedfa3",
+          "park",
+          "#c8facc",
+          "garden",
+          "#cdebb0",
+          "plant_nursery",
+          "#aedfa3",
+          "recreation_ground",
+          "#d5ffd9",
+          "reedbed",
+          "#cdebb0",
+          "saltmarsh",
+          "#cdebb0",
+          "sand",
+          "#f5e9c6",
+          "scree",
+          "#eee5dc",
+          "swamp",
+          "#add19e",
+          "tidalflat",
+          "#DED6CF",
+          "village_green",
+          "#cdebb0",
+          "vineyard",
+          "#aedfa3",
+          "wet_meadow",
+          "#cdebb0",
+          "wetland",
+          "#add19e",
+          "wood",
+          "#add19e",
+          "marsh",
+          "#ff0",
+          "#FFFFFF"
+        ],
+        "fill-antialias": true
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "subclass",
+          "allotments",
+          "bare_rock",
+          "beach",
+          "dune",
+          "scrub",
+          "farm",
+          "farmland",
+          "forest",
+          "garden",
+          "grass",
+          "grassland",
+          "golf_course",
+          "heath",
+          "meadow",
+          "orchard",
+          "plant_nursery",
+          "recreation_ground",
+          "reedbed",
+          "saltmarsh",
+          "sand",
+          "scree",
+          "swamp",
+          "tidalflat",
+          "tundra",
+          "village_green",
+          "vineyard",
+          "wet_meadow",
+          "wetland",
+          "wood"
+        ]
+      ],
+      "order": 7
+    },
+    {
+      "id": "landcover_subclass_patterns",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "minzoom": 13,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-opacity": [
+          "match",
+          [
+            "get",
+            "subclass"
+          ],
+          "beach",
+          0.4,
+          "forest",
+          0.4,
+          "bare_rock",
+          0.3,
+          "scrub",
+          0.6,
+          "garden",
+          0.6,
+          "scree",
+          0.3,
+          "wood",
+          0.4,
+          1
+        ],
+        "fill-pattern": [
+          "match",
+          [
+            "get",
+            "subclass"
+          ],
+          "allotments",
+          "allotments",
+          "bare_rock",
+          "rock_overlay",
+          "beach",
+          "beach",
+          "bog",
+          "wetland_bog",
+          "scrub",
+          "scrub",
+          "forest",
+          "leaftype_unknown",
+          "garden",
+          "plant_nursery",
+          "mangrove",
+          "wetland_mangrove",
+          "marsh",
+          "wetland_marsh",
+          "orchard",
+          "orchard",
+          "plant_nursery",
+          "plant_nursery",
+          "reedbed",
+          "wetland_reed",
+          "saltmarsh",
+          "wetland_marsh",
+          "scree",
+          "scree_overlay",
+          "swamp",
+          "wetland_swamp",
+          "vineyard",
+          "vineyard",
+          "wet_meadow",
+          "wetland_marsh",
+          "wetland",
+          "wetland",
+          "wood",
+          "leaftype_unknown",
+          ""
+        ]
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "subclass",
+          "allotments",
+          "bare_rock",
+          "beach",
+          "bog",
+          "dune",
+          "scrub",
+          "farm",
+          "farmland",
+          "forest",
+          "garden",
+          "grass",
+          "grassland",
+          "golf_course",
+          "heath",
+          "mangrove",
+          "marsh",
+          "meadow",
+          "orchard",
+          "park",
+          "plant_nursery",
+          "recreation_ground",
+          "reedbed",
+          "saltern",
+          "saltmarsh",
+          "sand",
+          "scree",
+          "swamp",
+          "village_green",
+          "vineyard",
+          "wet_meadow",
+          "wetland",
+          "wood"
+        ]
+      ],
+      "order": 8
+    },
+    {
+      "id": "landcover_subclass_outline",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "minzoom": 15,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": [
+          "match",
+          [
+            "get",
+            "subclass"
+          ],
+          "allotments",
+          "#B1C6A8",
+          "farm",
+          "#d1b48c",
+          "farmland",
+          "#c7c9ae",
+          "recreation_ground",
+          "#3c6640",
+          "#000"
+        ],
+        "line-width": [
+          "match",
+          [
+            "get",
+            "subclass"
+          ],
+          "recreation_ground",
+          0.3,
+          0.5
+        ],
+        "line-opacity": 1
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "subclass",
+          "allotments",
+          "farm",
+          "farmland",
+          "recreation_ground"
+        ]
+      ],
+      "order": 9
+    },
+    {
+      "id": "landcover_ice",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "minzoom": 5,
+      "paint": {
+        "fill-color": "#ddecec",
+        "fill-antialias": false
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "ice"
+        ]
+      ],
+      "order": 10
+    },
+    {
+      "id": "landcover_ice_outline",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "landcover",
+      "minzoom": 5,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#9cf",
+        "line-width": {
+          "stops": [
+            [
+              5,
+              1
+            ],
+            [
+              10,
+              1.5
+            ]
+          ]
+        },
+        "line-dasharray": {
+          "stops": [
+            [
+              5,
+              [
+                1,
+                0
+              ]
+            ],
+            [
+              10,
+              [
+                4,
+                2
+              ]
+            ]
+          ]
+        }
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "ice"
+        ]
+      ],
+      "order": 11
+    }
+  ]
+}

layers/landuse/class.sql

@@ -0,0 +1,10 @@
+-- Unify class names that represent the same type of feature
+CREATE OR REPLACE FUNCTION landuse_unify(class text) RETURNS text LANGUAGE plpgsql
+AS
+$$
+BEGIN
+  RETURN CASE
+    WHEN class='grave_yard' THEN 'cemetery'
+    ELSE class END;
+END;
+$$;

layers/landuse/landuse.sql

@@ -1,111 +1,379 @@
--- etldoc: ne_50m_urban_areas -> landuse_z4
-CREATE OR REPLACE VIEW landuse_z4 AS (
-    SELECT NULL::bigint AS osm_id, geometry, 'residential'::text AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS tourism, NULL::text AS place, NULL::text AS waterway, scalerank
-    FROM ne_50m_urban_areas
-    WHERE scalerank <= 2
+-- ne_50m_urban_areas
+-- etldoc: ne_50m_urban_areas ->  ne_50m_urban_areas_gen_z5
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_urban_areas_gen_z5 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_urban_areas_gen_z5 AS
+(
+SELECT
+       NULL::bigint AS osm_id,
+       ST_Simplify(geometry, ZRes(7)) as geometry,
+       'residential'::text AS landuse,
+       NULL::text AS amenity,
+       NULL::text AS leisure,
+       NULL::text AS tourism,
+       NULL::text AS place,
+       NULL::text AS waterway,
+       scalerank
+FROM ne_50m_urban_areas
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_urban_areas_gen_z5_idx ON ne_50m_urban_areas_gen_z5 USING gist (geometry);
+
+-- etldoc: ne_50m_urban_areas_gen_z5 ->  ne_50m_urban_areas_gen_z4
+DROP MATERIALIZED VIEW IF EXISTS ne_50m_urban_areas_gen_z4 CASCADE;
+CREATE MATERIALIZED VIEW ne_50m_urban_areas_gen_z4 AS
+(
+SELECT
+       osm_id,
+       ST_Simplify(geometry, ZRes(6)) as geometry,
+       landuse,
+       amenity,
+       leisure,
+       tourism,
+       place,
+       waterway
+FROM ne_50m_urban_areas_gen_z5
+WHERE scalerank <= 2
+    ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
+CREATE INDEX IF NOT EXISTS ne_50m_urban_areas_gen_z4_idx ON ne_50m_urban_areas_gen_z4 USING gist (geometry);
+
+-- etldoc: osm_landuse_polygon_gen_z6 ->  osm_landuse_polygon_gen_z6_union
+-- etldoc: osm_residential_gen_z6 ->  osm_landuse_polygon_gen_z6_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z6_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z6
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z6
 );
 
--- etldoc: ne_50m_urban_areas -> landuse_z5
-CREATE OR REPLACE VIEW landuse_z5 AS (
-    SELECT NULL::bigint AS osm_id, geometry, 'residential'::text AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS tourism, NULL::text AS place, NULL::text AS waterway, scalerank
-    FROM ne_50m_urban_areas
+-- etldoc: osm_landuse_polygon_gen_z7 ->  osm_landuse_polygon_gen_z7_union
+-- etldoc: osm_residential_gen_z7 ->  osm_landuse_polygon_gen_z7_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z7_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z7
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z7
 );
 
--- etldoc: osm_landuse_polygon_gen7 -> landuse_z6
-CREATE OR REPLACE VIEW landuse_z6 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen7
+-- etldoc: osm_landuse_polygon_gen_z8 ->  osm_landuse_polygon_gen_z8_union
+-- etldoc: osm_residential_gen_z8 ->  osm_landuse_polygon_gen_z8_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z8_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z8
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z8
 );
 
--- etldoc: osm_landuse_polygon_gen6 -> landuse_z8
-CREATE OR REPLACE VIEW landuse_z8 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen6
+-- etldoc: osm_landuse_polygon_gen_z9 ->  osm_landuse_polygon_gen_z9_union
+-- etldoc: osm_residential_gen_z9 ->  osm_landuse_polygon_gen_z9_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z9_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z9
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z9
 );
 
--- etldoc: osm_landuse_polygon_gen5 -> landuse_z9
-CREATE OR REPLACE VIEW landuse_z9 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen5
+-- etldoc: osm_landuse_polygon_gen_z10 ->  osm_landuse_polygon_gen_z10_union
+-- etldoc: osm_residential_gen_z10 ->  osm_landuse_polygon_gen_z10_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z10_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z10
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z10
 );
 
--- etldoc: osm_landuse_polygon_gen4 -> landuse_z10
-CREATE OR REPLACE VIEW landuse_z10 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen4
+-- etldoc: osm_landuse_polygon_gen_z11 ->  osm_landuse_polygon_gen_z11_union
+-- etldoc: osm_residential_gen_z11 ->  osm_landuse_polygon_gen_z11_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z11_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z11
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z11
 );
 
--- etldoc: osm_landuse_polygon_gen3 -> landuse_z11
-CREATE OR REPLACE VIEW landuse_z11 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen3
-);
-
--- etldoc: osm_landuse_polygon_gen2 -> landuse_z12
-CREATE OR REPLACE VIEW landuse_z12 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen2
-);
-
--- etldoc: osm_landuse_polygon_gen1 -> landuse_z13
-CREATE OR REPLACE VIEW landuse_z13 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon_gen1
-);
-
--- etldoc: osm_landuse_polygon -> landuse_z14
-CREATE OR REPLACE VIEW landuse_z14 AS (
-    SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank
-    FROM osm_landuse_polygon
+-- etldoc: osm_landuse_polygon_gen_z12 ->  osm_landuse_polygon_gen_z12_union
+-- etldoc: osm_residential_gen_z12 ->  osm_landuse_polygon_gen_z12_union
+CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z12_union AS
+(
+       SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z12
+		 WHERE landuse <> 'residential'
+       UNION ALL
+       SELECT NULL::bigint AS osm_id,
+              geometry,
+              'residential' AS landuse,
+              '' AS amenity,
+              '' AS leisure,
+              '' AS tourism,
+              '' AS place,
+              '' AS waterway
+         FROM osm_residential_gen_z12
 );
 
 -- etldoc: layer_landuse[shape=record fillcolor=lightpink, style="rounded,filled",
--- etldoc:     label="layer_landuse |<z4> z4|<z5>z5|<z6>z6|<z7>z7| <z8> z8 |<z9> z9 |<z10> z10 |<z11> z11|<z12> z12|<z13> z13|<z14> z14+" ] ;
+-- etldoc:     label="layer_landuse |<z4> z4|<z5> z5|<z6> z6|<z7> z7|<z8> z8|<z9> z9|<z10> z10|<z11> z11|<z12> z12|<z13> z13|<z14> z14+" ] ;
 
 CREATE OR REPLACE FUNCTION layer_landuse(bbox geometry, zoom_level int)
-RETURNS TABLE(osm_id bigint, geometry geometry, class text) AS $$
-    SELECT osm_id, geometry,
-        COALESCE(
-            NULLIF(landuse, ''),
-            NULLIF(amenity, ''),
-            NULLIF(leisure, ''),
-            NULLIF(tourism, ''),
-            NULLIF(place, ''),
-            NULLIF(waterway, '')
-        ) AS class
-        FROM (
-        -- etldoc: landuse_z4 -> layer_landuse:z4
-        SELECT * FROM landuse_z4
-        WHERE zoom_level = 4
-        UNION ALL
-        -- etldoc: landuse_z5 -> layer_landuse:z5
-        SELECT * FROM landuse_z5
-        WHERE zoom_level = 5
-        UNION ALL
-        -- etldoc: landuse_z6 -> layer_landuse:z6
-        -- etldoc: landuse_z6 -> layer_landuse:z7
-        SELECT * FROM landuse_z6 WHERE zoom_level BETWEEN 6 AND 7
-        UNION ALL
-        -- etldoc: landuse_z8 -> layer_landuse:z8
-        SELECT * FROM landuse_z8 WHERE zoom_level = 8
-        UNION ALL
-        -- etldoc: landuse_z9 -> layer_landuse:z9
-        SELECT * FROM landuse_z9 WHERE zoom_level = 9
-        UNION ALL
-        -- etldoc: landuse_z10 -> layer_landuse:z10
-        SELECT * FROM landuse_z10 WHERE zoom_level = 10
-        UNION ALL
-        -- etldoc: landuse_z11 -> layer_landuse:z11
-        SELECT * FROM landuse_z11 WHERE zoom_level = 11
-        UNION ALL
-        -- etldoc: landuse_z12 -> layer_landuse:z12
-        SELECT * FROM landuse_z12 WHERE zoom_level = 12
-        UNION ALL
-        -- etldoc: landuse_z13 -> layer_landuse:z13
-        SELECT * FROM landuse_z13 WHERE zoom_level = 13
-        UNION ALL
-        -- etldoc: landuse_z14 -> layer_landuse:z14
-        SELECT * FROM landuse_z14 WHERE zoom_level >= 14
-    ) AS zoom_levels
-    WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE;
+    RETURNS TABLE
+            (
+                osm_id   bigint,
+                geometry geometry,
+                class    text
+            )
+AS
+$$
+SELECT osm_id,
+       geometry,
+       landuse_unify(
+         COALESCE(
+               NULLIF(landuse, ''),
+               NULLIF(amenity, ''),
+               NULLIF(leisure, ''),
+               NULLIF(tourism, ''),
+               NULLIF(place, ''),
+               NULLIF(waterway, '')
+           )) AS class
+FROM (
+         -- etldoc: ne_50m_urban_areas_gen_z4 -> layer_landuse:z4
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM ne_50m_urban_areas_gen_z4
+         WHERE zoom_level = 4
+         UNION ALL
+         -- etldoc: ne_50m_urban_areas_gen_z5 -> layer_landuse:z5
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM ne_50m_urban_areas_gen_z5
+         WHERE zoom_level = 5
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z6_union -> layer_landuse:z6
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z6_union
+         WHERE zoom_level = 6
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z7_union -> layer_landuse:z7
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z7_union
+         WHERE zoom_level = 7
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z8_union -> layer_landuse:z8
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z8_union
+         WHERE zoom_level = 8
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z9_union -> layer_landuse:z9
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z9_union
+         WHERE zoom_level = 9
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z10_union -> layer_landuse:z10
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z10_union
+         WHERE zoom_level = 10
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z11_union -> layer_landuse:z11
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z11_union
+         WHERE zoom_level = 11
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z12_union -> layer_landuse:z12
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z12_union
+         WHERE zoom_level = 12
+         UNION ALL
+         -- etldoc: osm_landuse_polygon_gen_z13 -> layer_landuse:z13
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon_gen_z13
+         WHERE zoom_level = 13
+         UNION ALL
+         -- etldoc: osm_landuse_polygon -> layer_landuse:z14
+         SELECT osm_id,
+                geometry,
+                landuse,
+                amenity,
+                leisure,
+                tourism,
+                place,
+                waterway
+         FROM osm_landuse_polygon
+         WHERE zoom_level >= 14
+     ) AS zoom_levels
+WHERE geometry && bbox;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/landuse/landuse.yaml

@@ -1,5 +1,8 @@
 layer:
   id: "landuse"
+  requires:
+    tables:
+      - ne_50m_urban_areas
   description: |
       Landuse is used to describe use of land by humans. At lower zoom levels this is
       from Natural Earth data for residential (urban) areas and at higher zoom levels mostly OSM `landuse` tags.
@@ -22,6 +25,7 @@ layer:
       - residential
       - commercial
       - industrial
+      - garages
       - retail
       - bus_station
       - school
@@ -37,12 +41,16 @@ layer:
       - theme_park
       - zoo
       - suburb
+      - quarter
       - neighbourhood
       - dam
+      - quarry
   datasource:
     geometry_field: geometry
     query: (SELECT geometry, class FROM layer_landuse(!bbox!, z(!scale_denominator!))) AS t
 schema:
+  - ./class.sql
+  - ./prep_landuse.sql
   - ./landuse.sql
 datasources:
   - type: imposm3

layers/landuse/mapping.yaml

@@ -1,35 +1,41 @@
 generalized_tables:
-  # etldoc: imposm3 -> osm_landuse_polygon_gen7
-  landuse_polygon_gen7:
-    source: landuse_polygon_gen6
+  # etldoc: osm_landuse_polygon_gen_z7 -> osm_landuse_polygon_gen_z6
+  landuse_polygon_gen_z6:
+    source: landuse_polygon_gen_z7
+    tolerance: ZRES6
+    sql_filter: area>power(ZRES6,2)
+  # etldoc: osm_landuse_polygon_gen_z8 -> osm_landuse_polygon_gen_z7
+  landuse_polygon_gen_z7:
+    source: landuse_polygon_gen_z8
     tolerance: ZRES7
-  # etldoc: imposm3 -> osm_landuse_polygon_gen6
-  landuse_polygon_gen6:
-    source: landuse_polygon_gen5
-    sql_filter: area>power(ZRES6,2) AND (landuse='residential' OR place='suburb' OR place='neighbourhood')
+    sql_filter: area>power(ZRES6,2)
+  # etldoc: osm_landuse_polygon_gen_z9 -> osm_landuse_polygon_gen_z8
+  landuse_polygon_gen_z8:
+    source: landuse_polygon_gen_z9
+    sql_filter: area>power(ZRES6,2) AND (landuse='residential' OR place='suburb' OR place='quarter' OR place='neighbourhood')
     tolerance: ZRES8
-  # etldoc: imposm3 -> osm_landuse_polygon_gen5
-  landuse_polygon_gen5:
-    source: landuse_polygon_gen4
+  # etldoc: osm_landuse_polygon_gen_z10 -> osm_landuse_polygon_gen_z9
+  landuse_polygon_gen_z9:
+    source: landuse_polygon_gen_z10
     sql_filter: area>power(ZRES7,2)
     tolerance: ZRES9
-  # etldoc: imposm3 -> osm_landuse_polygon_gen4
-  landuse_polygon_gen4:
-    source: landuse_polygon_gen3
+  # etldoc: osm_landuse_polygon_gen_z11 -> osm_landuse_polygon_gen_z10
+  landuse_polygon_gen_z10:
+    source: landuse_polygon_gen_z11
     sql_filter: area>power(ZRES8,2)
     tolerance: ZRES10
-  # etldoc: imposm3 -> osm_landuse_polygon_gen3
-  landuse_polygon_gen3:
-    source: landuse_polygon_gen2
+  # etldoc: osm_landuse_polygon_gen_z12 -> osm_landuse_polygon_gen_z11
+  landuse_polygon_gen_z11:
+    source: landuse_polygon_gen_z12
     sql_filter: area>power(ZRES9,2)
     tolerance: ZRES11
-  # etldoc: imposm3 -> osm_landuse_polygon_gen2
-  landuse_polygon_gen2:
-    source: landuse_polygon_gen1
+  # etldoc: osm_landuse_polygon_gen_z13 -> osm_landuse_polygon_gen_z12
+  landuse_polygon_gen_z12:
+    source: landuse_polygon_gen_z13
     sql_filter: area>power(ZRES10,2)
     tolerance: ZRES12
-  # etldoc: imposm3 -> osm_landuse_polygon_gen1
-  landuse_polygon_gen1:
+  # etldoc: osm_landuse_polygon -> osm_landuse_polygon_gen_z13
+  landuse_polygon_gen_z13:
     source: landuse_polygon
     sql_filter: area>power(ZRES11,2) AND ST_IsValid(geometry)
     tolerance: ZRES13
@@ -38,7 +44,7 @@ tables:
   # etldoc: imposm3 -> osm_landuse_polygon
   landuse_polygon:
     type: polygon
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -68,6 +74,7 @@ tables:
       - railway
       - cemetery
       - military
+      - quarry
       # zoning
       - residential
       - commercial
@@ -82,6 +89,7 @@ tables:
       - college
       - library
       - hospital
+      - grave_yard
       leisure:
       - stadium
       - pitch
@@ -92,6 +100,7 @@ tables:
       - zoo
       place:
       - suburb
+      - quarter
       - neighbourhood
       waterway:
       - dam

layers/landuse/prep_landuse.sql

@@ -0,0 +1,176 @@
+DROP TABLE IF EXISTS cluster_zres14;
+CREATE TABLE cluster_zres14 AS 
+(
+WITH single_geom AS (
+        SELECT (ST_Dump(geometry)).geom AS geometry
+        FROM osm_landuse_polygon 
+        WHERE landuse='residential'
+    )
+    SELECT ST_ClusterDBSCAN(geometry, eps := zres(14), minpoints := 1) over () AS cid,
+           geometry
+    FROM single_geom
+);
+CREATE INDEX ON cluster_zres14 USING gist(geometry);
+
+
+DROP TABLE IF EXISTS cluster_zres14_union;
+CREATE TABLE cluster_zres14_union AS (
+SELECT ST_Buffer(
+            ST_Union(
+                ST_Buffer(
+                    ST_SnapToGrid(geometry, 0.01)
+                    , zres(14), 'join=mitre'
+                )
+            ),-zres(14), 'join=mitre'
+        ) AS geometry
+FROM cluster_zres14
+GROUP BY cid
+);
+CREATE INDEX ON cluster_zres14_union USING gist(geometry);
+
+
+DROP TABLE IF EXISTS cluster_zres12;
+CREATE TABLE cluster_zres12 AS 
+(
+WITH single_geom AS (
+    SELECT (ST_Dump(geometry)).geom AS geometry
+    FROM osm_landuse_polygon 
+    WHERE landuse='residential'
+    )
+    SELECT ST_ClusterDBSCAN(geometry, eps := zres(12), minpoints := 1) over () AS cid,
+           geometry
+    FROM single_geom
+);
+CREATE INDEX ON cluster_zres12 USING gist(geometry);
+
+
+DROP TABLE IF EXISTS cluster_zres12_union;
+CREATE TABLE cluster_zres12_union AS 
+(
+SELECT ST_Buffer(
+            ST_Union(
+                ST_Buffer(
+                    ST_SnapToGrid(geometry, 1)
+                        , zres(12), 'join=mitre'
+                    )
+                ), -zres(12), 'join=mitre'
+            ) AS geometry
+FROM cluster_zres12
+GROUP BY cid
+);
+CREATE INDEX ON cluster_zres12_union USING gist(geometry);
+
+
+DROP TABLE IF EXISTS cluster_zres9;
+CREATE TABLE cluster_zres9 AS 
+(
+WITH single_geom AS (
+        SELECT (ST_Dump(geometry)).geom AS geometry
+        FROM osm_landuse_polygon 
+        WHERE landuse='residential'
+    )
+    SELECT ST_ClusterDBSCAN(geometry, eps := zres(9), minpoints := 1) over () AS cid,
+           geometry
+    FROM single_geom
+);
+CREATE INDEX ON cluster_zres9 USING gist(geometry);
+
+
+DROP TABLE IF EXISTS cluster_zres9_union;
+CREATE TABLE cluster_zres9_union AS 
+(
+SELECT ST_Buffer(
+            ST_Union(
+                ST_Buffer(
+                    ST_SnapToGrid(geometry, 1)
+                        , zres(9), 'join=mitre'
+                    )
+                ), -zres(9), 'join=mitre'
+            ) AS geometry
+FROM cluster_zres9
+GROUP BY cid
+);
+CREATE INDEX ON cluster_zres9_union USING gist(geometry);
+
+-- For z6
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z6
+DROP TABLE IF EXISTS osm_residential_gen_z6 CASCADE;
+CREATE TABLE osm_residential_gen_z6 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(6), 2)) AS geometry
+FROM cluster_zres9_union
+WHERE ST_Area(geometry) > power(zres(6), 2)
+);
+CREATE INDEX ON osm_residential_gen_z6 USING gist(geometry);
+
+
+-- For z7
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z7
+DROP TABLE IF EXISTS osm_residential_gen_z7 CASCADE;
+CREATE TABLE osm_residential_gen_z7 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(7), 2)) AS geometry
+FROM cluster_zres12_union
+WHERE ST_Area(geometry) > power(zres(6), 2)
+);
+CREATE INDEX ON osm_residential_gen_z7 USING gist(geometry);
+
+
+-- For z8
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z8
+DROP TABLE IF EXISTS osm_residential_gen_z8 CASCADE;
+CREATE TABLE osm_residential_gen_z8 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(8), 2)) AS geometry
+FROM cluster_zres12_union
+WHERE ST_Area(geometry) > power(zres(7), 2)
+);
+CREATE INDEX ON osm_residential_gen_z8 USING gist(geometry);
+
+
+-- For z9
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z9
+DROP TABLE IF EXISTS osm_residential_gen_z9 CASCADE;
+CREATE TABLE osm_residential_gen_z9 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(9), 2)) AS geometry
+FROM cluster_zres12_union
+WHERE ST_Area(geometry) > power(zres(9), 2)
+);
+CREATE INDEX ON osm_residential_gen_z9 USING gist(geometry);
+
+
+-- For z10
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z10
+DROP TABLE IF EXISTS osm_residential_gen_z10 CASCADE;
+CREATE TABLE osm_residential_gen_z10 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(10), 2)) AS geometry
+FROM cluster_zres14_union
+WHERE ST_Area(geometry) > power(zres(10), 2)
+);
+CREATE INDEX ON osm_residential_gen_z10 USING gist(geometry);
+
+
+-- For z11
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z11
+DROP TABLE IF EXISTS osm_residential_gen_z11 CASCADE;
+CREATE TABLE osm_residential_gen_z11 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(11), 2)) AS geometry
+FROM cluster_zres14_union
+WHERE ST_Area(geometry) > power(zres(11), 2)
+);
+CREATE INDEX ON osm_residential_gen_z11 USING gist(geometry);
+
+
+-- For z12
+-- etldoc: osm_landuse_polygon ->  osm_residential_gen_z12
+DROP TABLE IF EXISTS osm_residential_gen_z12 CASCADE;
+CREATE TABLE osm_residential_gen_z12 AS
+(
+SELECT ST_SimplifyVW(geometry, power(zres(12), 2)) AS geometry
+FROM cluster_zres14_union
+WHERE ST_Area(geometry) > power(zres(12), 2)
+);
+CREATE INDEX ON osm_residential_gen_z12 USING gist(geometry);

layers/landuse/style.json

@@ -0,0 +1,369 @@
+{
+  "layers": [
+    {
+      "id": "landuse_classes",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landuse",
+      "minzoom": 7,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "railway",
+          "#ebdbe8",
+          "residential",
+          "#e0dfdf",
+          "cemetery",
+          "#aacbaf",
+          "military",
+          "#fceaea",
+          "commercial",
+          "#f2dad9",
+          "industrial",
+          "#ebdbe8",
+          "garages",
+          "#dfddce",
+          "retail",
+          "#ffd6d1",
+          "bus_station",
+          "#e9e7e2",
+          "school",
+          "#ffffe5",
+          "university",
+          "#ffffe5",
+          "kindergarten",
+          "#ffffe5",
+          "college",
+          "#ffffe5",
+          "hospital",
+          "#ffffe5",
+          "stadium",
+          "#d5ffd9",
+          "pitch",
+          "#aae0cb",
+          "playground",
+          "#d5ffd9",
+          "track",
+          "#aae0cb",
+          "dam",
+          "#adadad",
+          "#000"
+        ],
+        "fill-opacity": 1
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "railway",
+          "cemetery",
+          "military",
+          "residential",
+          "commercial",
+          "industrial",
+          "garages",
+          "retail",
+          "bus_station",
+          "school",
+          "university",
+          "kindergarten",
+          "college",
+          "hospital",
+          "stadium",
+          "pitch",
+          "playground",
+          "track",
+          "dam"
+        ],
+        [
+          "==",
+          "$type",
+          "Polygon"
+        ]
+      ],
+      "order": 1
+    },
+    {
+      "id": "landuse_residential",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landuse",
+      "minzoom": 6,
+      "maxzoom": 24,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": {
+          "stops": [
+            [
+              7,
+              "#d0d0d0"
+            ],
+            [
+              11,
+              "#dddddd"
+            ],
+            [
+              12,
+              "#e0dfdf"
+            ]
+          ]
+        }
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "residential",
+          "suburbs",
+          "neighbourhood"
+        ]
+      ],
+      "order": 2
+    },
+    {
+      "id": "landuse_class_pattern",
+      "type": "fill",
+      "source": "openmaptiles",
+      "source-layer": "landuse",
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "fill-color": "#000000",
+        "fill-opacity": 1,
+        "fill-pattern": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "military",
+          "military_red_hatch",
+          "cemetery",
+          "grave_yard_generic",
+          ""
+        ]
+      },
+      "metadata": {},
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "military",
+          "cemetery"
+        ]
+      ],
+      "order": 25
+    },
+    {
+      "id": "landuse_class_outline",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "landuse",
+      "minzoom": 13,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "railway",
+          "#c6b3c3",
+          "military",
+          "#ff5555",
+          "residential",
+          "#b9b9b9",
+          "commercial",
+          "#f2dad9",
+          "industrial",
+          "#c6b3c3",
+          "retail",
+          "#d99c95",
+          "school",
+          "#A6A68C",
+          "university",
+          "#A6A68C",
+          "kindergarten",
+          "#A6A68C",
+          "college",
+          "#A6A68C",
+          "hospital",
+          "#A6A68C",
+          "stadium",
+          "#7ca680",
+          "pitch",
+          "#7aaa97",
+          "playground",
+          "#3c6640",
+          "track",
+          "#7aaa96",
+          "theme_park",
+          "#660033",
+          "zoo",
+          "#660033",
+          "dam",
+          "#444444",
+          "#000"
+        ],
+        "line-width": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "railway",
+          0.7,
+          "military",
+          2,
+          "residential",
+          0.5,
+          "commercial",
+          0.5,
+          "industrial",
+          0.5,
+          "retail",
+          0.5,
+          "school",
+          0.3,
+          "university",
+          0.3,
+          "kindergarten",
+          0.3,
+          "college",
+          0.3,
+          "hospital",
+          0.3,
+          "stadium",
+          0.3,
+          "pitch",
+          0.5,
+          "playground",
+          0.3,
+          "track",
+          0.5,
+          "theme_park",
+          1,
+          "zoo",
+          1,
+          "dam",
+          2,
+          1
+        ],
+        "line-offset": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "military",
+          1,
+          0
+        ],
+        "line-opacity": [
+          "match",
+          [
+            "get",
+            "class"
+          ],
+          "military",
+          0.24,
+          1
+        ]
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "railway",
+          "military",
+          "residential",
+          "commercial",
+          "industrial",
+          "retail",
+          "school",
+          "university",
+          "kindergarten",
+          "college",
+          "hospital",
+          "stadium",
+          "pitch",
+          "playground",
+          "track",
+          "theme_park",
+          "zoo",
+          "dam"
+        ]
+      ],
+      "order": 26
+    },
+    {
+      "id": "landuse_class_themepark",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "landuse",
+      "minzoom": 13,
+      "layout": {
+        "line-cap": "square",
+        "line-join": "round",
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "#660033",
+        "line-width": {
+          "stops": [
+            [
+              9,
+              3.5
+            ],
+            [
+              14,
+              5.5
+            ]
+          ]
+        },
+        "line-offset": 2,
+        "line-opacity": {
+          "stops": [
+            [
+              9,
+              0.1
+            ],
+            [
+              12,
+              0.3
+            ]
+          ]
+        }
+      },
+      "filter": [
+        "all",
+        [
+          "in",
+          "class",
+          "theme_park",
+          "zoo"
+        ]
+      ],
+      "order": 27
+    }
+  ]
+}

layers/mountain_peak/layer.sql

@@ -1,55 +0,0 @@
-
--- etldoc: layer_mountain_peak[shape=record fillcolor=lightpink,
--- etldoc:     style="rounded,filled", label="layer_mountain_peak | <z7_> z7+" ] ;
-
-CREATE OR REPLACE FUNCTION layer_mountain_peak(
-    bbox geometry, 
-    zoom_level integer, 
-    pixel_width numeric)
-  RETURNS TABLE(
-    osm_id bigint,
-    geometry geometry, 
-    name text, 
-    name_en text, 
-    name_de text, 
-    class text, 
-    tags hstore,
-    ele int, 
-    ele_ft int, 
-    "rank" int) AS 
-$$
-   -- etldoc: osm_peak_point -> layer_mountain_peak:z7_
-  SELECT 
-    osm_id, 
-    geometry, 
-    name, 
-    name_en, 
-    name_de, 
-    tags -> 'natural' AS class, 
-    tags,
-    ele::int, 
-    ele_ft::int, 
-    rank::int FROM (
-      SELECT osm_id, geometry, name,
-      COALESCE(NULLIF(name_en, ''), name) AS name_en,
-      COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-      tags,
-      substring(ele from E'^(-?\\d+)(\\D|$)')::int AS ele,
-      round(substring(ele from E'^(-?\\d+)(\\D|$)')::int*3.2808399)::int AS ele_ft,
-      row_number() OVER (
-          PARTITION BY LabelGrid(geometry, 100 * pixel_width)
-          ORDER BY (
-            substring(ele from E'^(-?\\d+)(\\D|$)')::int +
-            (CASE WHEN NULLIF(wikipedia, '') is not null THEN 10000 ELSE 0 END) +
-            (CASE WHEN NULLIF(name, '') is not null THEN 10000 ELSE 0 END)
-          ) DESC
-      )::int AS "rank"
-      FROM osm_peak_point
-      WHERE geometry && bbox
-        AND ele is not null
-        AND ele ~ E'^-?\\d{1,4}(\\D|$)'
-    ) AS ranked_peaks
-  WHERE zoom_level >= 7 AND (rank <= 5 OR zoom_level >= 14)
-  ORDER BY "rank" ASC;
-
-$$ LANGUAGE SQL IMMUTABLE;

layers/mountain_peak/mapping.yaml

@@ -3,7 +3,7 @@ tables:
   # etldoc: imposm3 -> osm_peak_point
   peak_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -29,3 +29,32 @@ tables:
       natural:
       - peak
       - volcano
+      - saddle
+
+  # etldoc: imposm3 -> osm_mountain_linestring
+  mountain_linestring:
+    type: linestring
+    columns:
+    - name: osm_id
+      type: id
+    - name: geometry
+      type: geometry
+    - name: name
+      key: name
+      type: string
+    - name: name_en
+      key: name:en
+      type: string
+    - name: name_de
+      key: name:de
+      type: string
+    - name: tags
+      type: hstore_tags
+    - name: wikipedia
+      key: wikipedia
+      type: string
+    mapping:
+      natural:
+      - ridge
+      - cliff
+      - arete

layers/mountain_peak/mountain_peak.sql

@@ -0,0 +1,118 @@
+-- etldoc: osm_peak_point -> peak_point
+-- etldoc: ne_10m_admin_0_countries -> peak_point
+CREATE OR REPLACE VIEW peak_point AS
+(
+SELECT pp.osm_id,
+       pp.geometry,
+       pp.name,
+       pp.name_en,
+       pp.name_de,
+       pp.tags,
+       pp.ele,
+       ne.iso_a2,
+       pp.wikipedia
+FROM osm_peak_point pp, ne_10m_admin_0_countries ne
+WHERE ST_Intersects(pp.geometry, ne.geometry)
+    );
+
+
+
+-- etldoc: layer_mountain_peak[shape=record fillcolor=lightpink,
+-- etldoc:     style="rounded,filled", label="layer_mountain_peak | <z7_> z7+ | <z13_> z13+" ] ;
+
+CREATE OR REPLACE FUNCTION layer_mountain_peak(bbox geometry,
+                                               zoom_level integer,
+                                               pixel_width numeric)
+    RETURNS TABLE
+            (
+                osm_id          bigint,
+                geometry        geometry,
+                name            text,
+                name_en         text,
+                name_de         text,
+                class           text,
+                tags            hstore,
+                ele             int,
+                ele_ft          int,
+                customary_ft    int,
+                "rank"          int
+            )
+AS
+$$
+SELECT
+    -- etldoc: peak_point -> layer_mountain_peak:z7_
+    osm_id,
+    geometry,
+    name,
+    name_en,
+    name_de,
+    tags->'natural' AS class,
+    tags,
+    ele::int,
+    ele_ft::int,
+    customary_ft,
+    rank::int
+FROM (
+         SELECT osm_id,
+                geometry,
+                name,
+                COALESCE(NULLIF(name_en, ''), name) AS name_en,
+                COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+                tags,
+                substring(ele FROM E'^(-?\\d+)(\\D|$)')::int AS ele,
+                round(substring(ele FROM E'^(-?\\d+)(\\D|$)')::int * 3.2808399)::int AS ele_ft,
+                CASE WHEN iso_a2 = 'US' THEN 1 END AS customary_ft,
+                row_number() OVER (
+                    PARTITION BY LabelGrid(geometry, 100 * pixel_width)
+                    ORDER BY (
+                            substring(ele FROM E'^(-?\\d+)(\\D|$)')::int +
+                            (CASE WHEN wikipedia <> '' THEN 10000 ELSE 0 END) +
+                            (CASE WHEN name <> '' THEN 10000 ELSE 0 END)
+                        ) DESC
+                    )::int AS "rank"
+         FROM peak_point
+         WHERE geometry && bbox
+           AND ele IS NOT NULL
+           AND ele ~ E'^-?\\d{1,4}(\\D|$)'
+     ) AS ranked_peaks
+WHERE zoom_level >= 7
+  AND (rank <= 5 OR zoom_level >= 14)
+
+UNION ALL
+
+SELECT
+    -- etldoc: osm_mountain_linestring -> layer_mountain_peak:z13_
+    osm_id,
+    geometry,
+    name,
+    name_en,
+    name_de,
+    tags->'natural' AS class,
+    tags,
+    NULL AS ele,
+    NULL AS ele_ft,
+    NULL AS customary_ft,
+    rank::int
+FROM (
+         SELECT osm_id,
+                geometry,
+                name,
+                COALESCE(NULLIF(name_en, ''), name) AS name_en,
+                COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+                tags,
+                row_number() OVER (
+                    PARTITION BY LabelGrid(geometry, 100 * pixel_width)
+                    ORDER BY (
+                            (CASE WHEN wikipedia <> '' THEN 10000 ELSE 0 END) +
+                            (CASE WHEN name <> '' THEN 10000 ELSE 0 END)
+                        ) DESC
+                    )::int AS "rank"
+         FROM osm_mountain_linestring
+         WHERE geometry && bbox
+     ) AS ranked_mountain_linestring
+WHERE zoom_level >= 13
+ORDER BY "rank" ASC;
+
+$$ LANGUAGE SQL STABLE
+                PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/mountain_peak/mountain_peak.yaml

@@ -1,31 +1,45 @@
 layer:
   id: "mountain_peak"
+  requires:
+    tables:
+      - ne_10m_admin_0_countries
   description: |
       [Natural peaks](http://wiki.openstreetmap.org/wiki/Tag:natural%3Dpeak)
   buffer_size: 64
   srs: +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over
   fields:
-    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the peak.
-    name_en: English name `name:en` if available, otherwise `name`.
-    name_de: German name `name:de` if available, otherwise `name` or `name:en`.
+    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the peak. Language-specific values are in `name:xx`.
+    name_en: English name `name:en` if available, otherwise `name`. This is deprecated and will be removed in a future release in favor of `name:en`.
+    name_de: German name `name:de` if available, otherwise `name` or `name:en`. This is deprecated and will be removed in a future release in favor of `name:de`.
     class:
       description: |
-        Use the **class** to differentiate between mountain peak and volcano.
+        Use the **class** to differentiate between natural objects.
       values:
       - peak
       - volcano
+      - saddle
+      - ridge
+      - cliff
+      - arete
     ele: Elevation (`ele`) in meters.
-    ele_ft: Elevation (`ele`) in feets.
+    ele_ft: Elevation (`ele`) in feet.
+    customary_ft:
+      description: |
+        Value 1 for peaks in location where feet is used as customary unit (USA).
+      values:
+      - 1
+      - NULL
     rank: Rank of the peak within one tile (starting at 1 that is the most important peak).
   datasource:
     geometry_field: geometry
     key_field: osm_id
     key_field_as_attribute: no
     srid: 900913
-    query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, ele, ele_ft, rank FROM layer_mountain_peak(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
+    query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, ele, ele_ft, customary_ft, rank FROM layer_mountain_peak(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
 schema:
   - ./update_peak_point.sql
-  - ./layer.sql
+  - ./update_mountain_linestring.sql
+  - ./mountain_peak.sql
 datasources:
   - type: imposm3
     mapping_file: ./mapping.yaml

layers/mountain_peak/style.json

@@ -0,0 +1,101 @@
+{
+  "layers": [
+    {
+      "id": "mountain_peak",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "mountain_peak",
+      "maxzoom": 16,
+      "filter": [
+        "all",
+        [
+          "!in",
+          "class",
+          "cliff",
+          "volcano"
+        ]
+      ],
+      "layout": {
+        "text-size": 10,
+        "icon-image": "peak",
+        "text-field": {
+          "stops": [
+            [
+              6,
+              " "
+            ],
+            [
+              12,
+              "{name} {ele}m"
+            ]
+          ]
+        },
+        "text-anchor": "top",
+        "text-offset": [
+          0,
+          0.5
+        ],
+        "text-max-width": 6,
+        "text-line-height": 1.1,
+        "text-font": [
+          "Noto Sans Regular",
+          "Noto Sans Italic"
+        ]
+      },
+      "paint": {
+        "text-color": "#6e441e",
+        "text-halo-color": "rgba(255, 255, 255, .8)",
+        "text-halo-width": 1
+      },
+      "order": 197
+    },
+    {
+      "id": "mountain_peak_volcano",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "mountain_peak",
+      "maxzoom": 16,
+      "filter": [
+        "all",
+        [
+          "==",
+          "class",
+          "volcano"
+        ]
+      ],
+      "layout": {
+        "text-size": 10,
+        "icon-image": "volcano",
+        "text-field": {
+          "stops": [
+            [
+              6,
+              " "
+            ],
+            [
+              12,
+              "{name} {ele}m"
+            ]
+          ]
+        },
+        "text-anchor": "top",
+        "text-offset": [
+          0,
+          0.5
+        ],
+        "text-max-width": 6,
+        "text-line-height": 1.1,
+        "text-font": [
+          "Noto Sans Regular",
+          "Noto Sans Italic"
+        ]
+      },
+      "paint": {
+        "text-color": "#d40000",
+        "text-halo-color": "rgba(255, 255, 255, .8)",
+        "text-halo-width": 1
+      },
+      "order": 198
+    }
+  ]
+}

layers/mountain_peak/update_mountain_linestring.sql

@@ -0,0 +1,87 @@
+DROP TRIGGER IF EXISTS trigger_flag ON osm_mountain_linestring;
+DROP TRIGGER IF EXISTS trigger_store ON osm_mountain_linestring;
+DROP TRIGGER IF EXISTS trigger_refresh ON mountain_linestring.updates;
+
+CREATE SCHEMA IF NOT EXISTS mountain_linestring;
+
+CREATE TABLE IF NOT EXISTS mountain_linestring.osm_ids
+(
+    osm_id bigint PRIMARY KEY
+);
+
+-- etldoc:  osm_mountain_linestring ->  osm_mountain_linestring
+CREATE OR REPLACE FUNCTION update_osm_mountain_linestring(full_update boolean) RETURNS void AS
+$$
+    UPDATE osm_mountain_linestring
+    SET tags = update_tags(tags, geometry)
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM mountain_linestring.osm_ids))
+      AND COALESCE(tags -> 'name:latin', tags -> 'name:nonlatin', tags -> 'name_int') IS NULL
+      AND tags != update_tags(tags, geometry)
+$$ LANGUAGE SQL;
+
+SELECT update_osm_mountain_linestring(true);
+
+-- Handle updates
+
+CREATE OR REPLACE FUNCTION mountain_linestring.store() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO mountain_linestring.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TABLE IF NOT EXISTS mountain_linestring.updates
+(
+    id serial PRIMARY KEY,
+    t  text,
+    UNIQUE (t)
+);
+CREATE OR REPLACE FUNCTION mountain_linestring.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO mountain_linestring.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION mountain_linestring.refresh() RETURNS trigger AS
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+    RAISE LOG 'Refresh mountain_linestring';
+
+    -- Analyze tracking and source tables before performing update
+    ANALYZE mountain_linestring.osm_ids;
+    ANALYZE osm_mountain_linestring;
+
+    PERFORM update_osm_mountain_linestring(false);
+    -- noinspection SqlWithoutWhere
+    DELETE FROM mountain_linestring.osm_ids;
+    -- noinspection SqlWithoutWhere
+    DELETE FROM mountain_linestring.updates;
+
+    RAISE LOG 'Refresh mountain_linestring done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_store
+    AFTER INSERT OR UPDATE
+    ON osm_mountain_linestring
+    FOR EACH ROW
+EXECUTE PROCEDURE mountain_linestring.store();
+
+CREATE TRIGGER trigger_flag
+    AFTER INSERT OR UPDATE
+    ON osm_mountain_linestring
+    FOR EACH STATEMENT
+EXECUTE PROCEDURE mountain_linestring.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+    AFTER INSERT
+    ON mountain_linestring.updates
+    INITIALLY DEFERRED
+    FOR EACH ROW
+EXECUTE PROCEDURE mountain_linestring.refresh();

layers/mountain_peak/update_peak_point.sql

@@ -1,48 +1,87 @@
 DROP TRIGGER IF EXISTS trigger_flag ON osm_peak_point;
+DROP TRIGGER IF EXISTS trigger_store ON osm_peak_point;
 DROP TRIGGER IF EXISTS trigger_refresh ON mountain_peak_point.updates;
 
--- etldoc:  osm_peak_point ->  osm_peak_point
-CREATE OR REPLACE FUNCTION update_osm_peak_point() RETURNS VOID AS $$
-BEGIN
-  UPDATE osm_peak_point
-  SET tags = update_tags(tags, geometry)
-  WHERE COALESCE(tags->'name:latin', tags->'name:nonlatin', tags->'name_int') IS NULL;
-
-END;
-$$ LANGUAGE plpgsql;
-
-SELECT update_osm_peak_point();
-
--- Handle updates
-
 CREATE SCHEMA IF NOT EXISTS mountain_peak_point;
 
-CREATE TABLE IF NOT EXISTS mountain_peak_point.updates(id serial primary key, t text, unique (t));
-CREATE OR REPLACE FUNCTION mountain_peak_point.flag() RETURNS trigger AS $$
+CREATE TABLE IF NOT EXISTS mountain_peak_point.osm_ids
+(
+    osm_id bigint PRIMARY KEY
+);
+
+-- etldoc:  osm_peak_point ->  osm_peak_point
+CREATE OR REPLACE FUNCTION update_osm_peak_point(full_update boolean) RETURNS void AS
+$$
+    UPDATE osm_peak_point
+    SET tags = update_tags(tags, geometry)
+    WHERE (full_update OR osm_id IN (SELECT osm_id FROM mountain_peak_point.osm_ids))
+      AND COALESCE(tags -> 'name:latin', tags -> 'name:nonlatin', tags -> 'name_int') IS NULL
+      AND tags != update_tags(tags, geometry)
+$$ LANGUAGE SQL;
+
+SELECT update_osm_peak_point(true);
+
+-- Handle updates
+
+CREATE OR REPLACE FUNCTION mountain_peak_point.store() RETURNS trigger AS
+$$
 BEGIN
-    INSERT INTO mountain_peak_point.updates(t) VALUES ('y')  ON CONFLICT(t) DO NOTHING;
-    RETURN null;
+    INSERT INTO mountain_peak_point.osm_ids VALUES (NEW.osm_id) ON CONFLICT (osm_id) DO NOTHING;
+    RETURN NULL;
 END;
-$$ language plpgsql;
+$$ LANGUAGE plpgsql;
+
+CREATE TABLE IF NOT EXISTS mountain_peak_point.updates
+(
+    id serial PRIMARY KEY,
+    t  text,
+    UNIQUE (t)
+);
+CREATE OR REPLACE FUNCTION mountain_peak_point.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO mountain_peak_point.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION mountain_peak_point.refresh() RETURNS trigger AS
-  $BODY$
-  BEGIN
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
     RAISE LOG 'Refresh mountain_peak_point';
-    PERFORM update_osm_peak_point();
+
+    -- Analyze tracking and source tables before performing update
+    ANALYZE mountain_peak_point.osm_ids;
+    ANALYZE osm_peak_point;
+
+    PERFORM update_osm_peak_point(false);
+    -- noinspection SqlWithoutWhere
+    DELETE FROM mountain_peak_point.osm_ids;
+    -- noinspection SqlWithoutWhere
     DELETE FROM mountain_peak_point.updates;
-    RETURN null;
-  END;
-  $BODY$
-language plpgsql;
+
+    RAISE LOG 'Refresh mountain_peak_point done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_store
+    AFTER INSERT OR UPDATE
+    ON osm_peak_point
+    FOR EACH ROW
+EXECUTE PROCEDURE mountain_peak_point.store();
 
 CREATE TRIGGER trigger_flag
-    AFTER INSERT OR UPDATE OR DELETE ON osm_peak_point
+    AFTER INSERT OR UPDATE
+    ON osm_peak_point
     FOR EACH STATEMENT
-    EXECUTE PROCEDURE mountain_peak_point.flag();
+EXECUTE PROCEDURE mountain_peak_point.flag();
 
 CREATE CONSTRAINT TRIGGER trigger_refresh
-    AFTER INSERT ON mountain_peak_point.updates
+    AFTER INSERT
+    ON mountain_peak_point.updates
     INITIALLY DEFERRED
     FOR EACH ROW
-    EXECUTE PROCEDURE mountain_peak_point.refresh();
+EXECUTE PROCEDURE mountain_peak_point.refresh();

layers/park/layer.sql

@@ -1,125 +0,0 @@
--- etldoc: layer_park[shape=record fillcolor=lightpink, style="rounded,filled",
--- etldoc:     label="layer_park |<z6> z6 |<z7> z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14> z14+" ] ;
-
-CREATE OR REPLACE FUNCTION layer_park(bbox geometry, zoom_level int, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, class text, name text, name_en text, name_de text, tags hstore, rank int) AS $$
-    SELECT osm_id, geometry, class, name, name_en, name_de, tags, rank
-    FROM (
-    SELECT osm_id, geometry,
-        COALESCE(NULLIF(boundary, ''), NULLIF(leisure, '')) AS class,
-        name, name_en, name_de, tags,
-        NULL::int as rank
-        FROM (
-        -- etldoc: osm_park_polygon_gen8 -> layer_park:z6
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen8
-        WHERE zoom_level = 6 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen7 -> layer_park:z7
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen7
-        WHERE zoom_level = 7 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen6 -> layer_park:z8
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen6
-        WHERE zoom_level = 8 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen5 -> layer_park:z9
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen5
-        WHERE zoom_level = 9 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen4 -> layer_park:z10
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen4
-        WHERE zoom_level = 10 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen3 -> layer_park:z11
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen3
-        WHERE zoom_level = 11 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen2 -> layer_park:z12
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen2
-        WHERE zoom_level = 12 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon_gen1 -> layer_park:z13
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon_gen1
-        WHERE zoom_level = 13 AND geometry && bbox
-        UNION ALL
-        -- etldoc: osm_park_polygon -> layer_park:z14
-        SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, NULL::int as scalerank
-        FROM osm_park_polygon
-        WHERE zoom_level >= 14 AND geometry && bbox
-    ) AS park_polygon
-
-    UNION ALL
-    SELECT osm_id, geometry_point AS geometry,
-        COALESCE(NULLIF(boundary, ''), NULLIF(leisure, '')) AS class,
-        name, name_en, name_de, tags,
-        row_number() OVER (
-           PARTITION BY LabelGrid(geometry_point, 100 * pixel_width)
-           ORDER BY
-               (CASE WHEN boundary = 'national_park' THEN true ELSE false END) DESC,
-               (COALESCE(NULLIF(tags->'wikipedia', ''), NULLIF(tags->'wikidata', '')) IS NOT NULL) DESC,
-               area DESC
-        )::int AS "rank"
-        FROM (
-        -- etldoc: osm_park_polygon_gen8 -> layer_park:z6
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen8
-        WHERE zoom_level = 6 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen7 -> layer_park:z7
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen7
-        WHERE zoom_level = 7 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen6 -> layer_park:z8
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen6
-        WHERE zoom_level = 8 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen5 -> layer_park:z9
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen5
-        WHERE zoom_level = 9 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen4 -> layer_park:z10
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen4
-        WHERE zoom_level = 10 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen3 -> layer_park:z11
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen3
-        WHERE zoom_level = 11 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen2 -> layer_park:z12
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen2
-        WHERE zoom_level = 12 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon_gen1 -> layer_park:z13
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon_gen1
-        WHERE zoom_level = 13 AND geometry_point && bbox
-        UNION ALL
-
-        -- etldoc: osm_park_polygon -> layer_park:z14
-        SELECT osm_id, geometry_point, name, name_en, name_de, tags, leisure, boundary, area
-        FROM osm_park_polygon
-        WHERE zoom_level >= 14 AND geometry_point && bbox
-    ) AS park_point
-    ) AS park_all;
-$$ LANGUAGE SQL IMMUTABLE;

layers/park/mapping.yaml

@@ -1,48 +1,60 @@
 generalized_tables:
-  # etldoc: imposm3 -> osm_park_polygon_gen8
-  park_polygon_gen8:
-    source: park_polygon_gen7
+  # etldoc: osm_park_polygon_gen_z5 -> osm_park_polygon_gen_z4
+  park_polygon_gen_z4:
+    source: park_polygon_gen_z5
+    sql_filter: area>power(ZRES3,2)
+    tolerance: ZRES4
+
+  # etldoc: osm_park_polygon_gen_z6 -> osm_park_polygon_gen_z5
+  park_polygon_gen_z5:
+    source: park_polygon_gen_z6
+    sql_filter: area>power(ZRES4,2)
+    tolerance: ZRES5
+
+  # etldoc: osm_park_polygon_gen_z7 -> osm_park_polygon_gen_z6
+  park_polygon_gen_z6:
+    source: park_polygon_gen_z7
     sql_filter: area>power(ZRES5,2)
-    tolerance: ZRES8
+    tolerance: ZRES6
 
-  # etldoc: imposm3 -> osm_park_polygon_gen7
-  park_polygon_gen7:
-    source: park_polygon_gen6
+  # etldoc: osm_park_polygon_gen_z8 -> osm_park_polygon_gen_z7
+  park_polygon_gen_z7:
+    source: park_polygon_gen_z8
     sql_filter: area>power(ZRES6,2)
+    tolerance: ZRES7
+
+  # etldoc: osm_park_polygon_gen_z9 -> osm_park_polygon_gen_z8
+  park_polygon_gen_z8:
+    source: park_polygon_gen_z9
+    sql_filter: area>power(ZRES7,2)
     tolerance: ZRES8
 
-  # etldoc: imposm3 -> osm_park_polygon_gen6
-  park_polygon_gen6:
-    source: park_polygon_gen5
-    sql_filter: area>power(ZRES7,2)
+  # etldoc: osm_park_polygon_gen_z10 -> osm_park_polygon_gen_z9
+  park_polygon_gen_z9:
+    source: park_polygon_gen_z10
+    sql_filter: area>power(ZRES8,2)
     tolerance: ZRES9
 
-  # etldoc: imposm3 -> osm_park_polygon_gen5
-  park_polygon_gen5:
-    source: park_polygon_gen4
-    sql_filter: area>power(ZRES8,2)
+  # etldoc: osm_park_polygon_gen_z11 -> osm_park_polygon_gen_z10
+  park_polygon_gen_z10:
+    source: park_polygon_gen_z11
+    sql_filter: area>power(ZRES9,2)
     tolerance: ZRES10
 
-  # etldoc: imposm3 -> osm_park_polygon_gen4
-  park_polygon_gen4:
-    source: park_polygon_gen3
-    sql_filter: area>power(ZRES9,2)
-    tolerance: ZRES11
-
-  # etldoc: imposm3 -> osm_park_polygon_gen3
-  park_polygon_gen3:
-    source: park_polygon_gen2
+  # etldoc: osm_park_polygon_gen_z12 -> osm_park_polygon_gen_z11
+  park_polygon_gen_z11:
+    source: park_polygon_gen_z12
     sql_filter: area>power(ZRES10,2)
     tolerance: ZRES11
 
-  # etldoc: imposm3 -> osm_park_polygon_gen2
-  park_polygon_gen2:
-    source: park_polygon_gen1
+  # etldoc: osm_park_polygon_gen_z13 -> osm_park_polygon_gen_z12
+  park_polygon_gen_z12:
+    source: park_polygon_gen_z13
     sql_filter: area>power(ZRES11,2)
     tolerance: ZRES12
 
-  # etldoc: imposm3 -> osm_park_polygon_gen1
-  park_polygon_gen1:
+  # etldoc: osm_park_polygon -> osm_park_polygon_gen_z13
+  park_polygon_gen_z13:
     source: park_polygon
     sql_filter: area>power(ZRES12,2) AND ST_IsValid(geometry)
     tolerance: ZRES13
@@ -52,7 +64,8 @@ tables:
   # etldoc: imposm3 -> osm_park_polygon
   park_polygon:
     type: polygon
-    fields:
+    _resolve_wikidata: false
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -77,6 +90,9 @@ tables:
     - name: boundary
       key: boundary
       type: string
+    - name: protection_title
+      key: protection_title
+      type: string
     - name: area
       type: area
     mapping:
@@ -84,3 +100,4 @@ tables:
       - nature_reserve
       boundary:
       - national_park
+      - protected_area

layers/park/park.sql

@@ -0,0 +1,386 @@
+-- etldoc: layer_park[shape=record fillcolor=lightpink, style="rounded,filled",
+-- etldoc:     label="layer_park |<z4> z4 |<z5> z5 |<z6> z6 |<z7> z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14> z14+" ] ;
+
+CREATE OR REPLACE FUNCTION layer_park(bbox geometry, zoom_level int, pixel_width numeric)
+    RETURNS TABLE
+            (
+                osm_id   bigint,
+                geometry geometry,
+                class    text,
+                name     text,
+                name_en  text,
+                name_de  text,
+                tags     hstore,
+                rank     int
+            )
+AS
+$$
+SELECT osm_id,
+       geometry,
+       class,
+       NULLIF(name, '') AS name,
+       NULLIF(name_en, '') AS name_en,
+       NULLIF(name_de, '') AS name_de,
+       tags,
+       rank
+FROM (
+         SELECT osm_id,
+                geometry,
+                COALESCE(
+                    LOWER(REPLACE(NULLIF(protection_title, ''), ' ', '_')),
+                    NULLIF(boundary, ''),
+                    NULLIF(leisure, ''))
+                AS class,
+                name,
+                name_en,
+                name_de,
+                tags,
+                NULL::int AS rank
+         FROM (
+                  -- etldoc: osm_park_polygon_dissolve_z4 -> layer_park:z4
+                  SELECT NULL::int AS osm_id,
+                         geometry,
+                         NULL AS name,
+                         NULL AS name_en,
+                         NULL AS name_de,
+                         NULL AS tags,
+                         NULL AS leisure,
+                         NULL AS boundary,
+                         NULL AS protection_title
+                  FROM osm_park_polygon_dissolve_z4
+                  WHERE zoom_level = 4
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z5 -> layer_park:z5
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z5
+                  WHERE zoom_level = 5
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z6 -> layer_park:z6
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z6
+                  WHERE zoom_level = 6
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z7 -> layer_park:z7
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z7
+                  WHERE zoom_level = 7
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z8 -> layer_park:z8
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z8
+                  WHERE zoom_level = 8
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z9 -> layer_park:z9
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z9
+                  WHERE zoom_level = 9
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z10 -> layer_park:z10
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z10
+                  WHERE zoom_level = 10
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z11 -> layer_park:z11
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z11
+                  WHERE zoom_level = 11
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z12 -> layer_park:z12
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z12
+                  WHERE zoom_level = 12
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon_gen_z13 -> layer_park:z13
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon_gen_z13
+                  WHERE zoom_level = 13
+                    AND geometry && bbox
+                  UNION ALL
+                  -- etldoc: osm_park_polygon -> layer_park:z14
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title
+                  FROM osm_park_polygon
+                  WHERE zoom_level >= 14
+                    AND geometry && bbox
+              ) AS park_polygon
+
+         UNION ALL
+         SELECT osm_id,
+                geometry_point AS geometry,
+                COALESCE(
+                        LOWER(REPLACE(NULLIF(protection_title, ''), ' ', '_')),
+                        NULLIF(boundary, ''),
+                        NULLIF(leisure, '')
+                    ) AS class,
+                name,
+                name_en,
+                name_de,
+                tags,
+                row_number() OVER (
+                    PARTITION BY LabelGrid(geometry_point, 100 * pixel_width)
+                    ORDER BY
+                        (CASE WHEN boundary = 'national_park' THEN TRUE ELSE FALSE END) DESC,
+                        (COALESCE(NULLIF(tags->'wikipedia', ''), NULLIF(tags->'wikidata', '')) IS NOT NULL) DESC,
+                        area DESC
+                    )::int AS "rank"
+         FROM (
+                  -- etldoc: osm_park_polygon_gen_z5 -> layer_park:z5
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z5
+                  WHERE zoom_level = 5
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z6 -> layer_park:z6
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z6
+                  WHERE zoom_level = 6
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z7 -> layer_park:z7
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z7
+                  WHERE zoom_level = 7
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z8 -> layer_park:z8
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z8
+                  WHERE zoom_level = 8
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z9 -> layer_park:z9
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z9
+                  WHERE zoom_level = 9
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z10 -> layer_park:z10
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z10
+                  WHERE zoom_level = 10
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z11 -> layer_park:z11
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z11
+                  WHERE zoom_level = 11
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z12 -> layer_park:z12
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z12
+                  WHERE zoom_level = 12
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon_gen_z13 -> layer_park:z13
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon_gen_z13
+                  WHERE zoom_level = 13
+                    AND geometry_point && bbox
+                    AND area > 70000*2^(20-zoom_level)
+                  UNION ALL
+
+                  -- etldoc: osm_park_polygon -> layer_park:z14
+                  SELECT osm_id,
+                         geometry_point,
+                         name,
+                         name_en,
+                         name_de,
+                         tags,
+                         leisure,
+                         boundary,
+                         protection_title,
+                         area
+                  FROM osm_park_polygon
+                  WHERE zoom_level >= 14
+                    AND geometry_point && bbox
+              ) AS park_point
+     ) AS park_all;
+$$ LANGUAGE SQL STABLE
+                PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/park/park.yaml

@@ -1,25 +1,32 @@
 layer:
   id: "park"
   description: |
-      The park layer contains parks from OpenStreetMap tagged with either [`boundary=national_park`](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dnational_park) or [`leisure=nature_reserve`](http://wiki.openstreetmap.org/wiki/Tag:leisure%3Dnature_reserve).
+    The park layer in OpenMapTiles contains natural and protected areas from OpenStreetMap, 
+    such as parks tagged with [`boundary=national_park`](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dnational_park),
+    [`boundary=protected_area`](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dprotected_area),
+    or [`leisure=nature_reserve`](https://wiki.openstreetmap.org/wiki/Tag:leisure%3Dnature_reserve).
   buffer_size: 4
   fields:
     class:
       description: |
-          Use the **class** to differentiate between different parks.
-      values:
-      - national_park
-      - nature_reserve
-    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the park (point features only).
-    name_en: English name `name:en` if available, otherwise `name` (point features only).
-    name_de: German name `name:de` if available, otherwise `name` or `name:en` (point features only).
+        Use the **class** to differentiate between different kinds of features in the `parks` layer.
+        The class for `boundary=protected_area` parks is the lower-case of the
+        [`protection_title`](http://wiki.openstreetmap.org/wiki/key:protection_title)
+        value with blanks replaced by `_`.
+        `national_park` is the class of `protection_title=National Park` and `boundary=national_park`.
+        `nature_reserve` is the class of `protection_title=Nature Reserve` and `leisure=nature_reserve`.
+        The class for other [`protection_title`](http://wiki.openstreetmap.org/wiki/key:protection_title) 
+        values is similarly assigned.
+    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the park (point features only). Language-specific values are in `name:xx`.
+    name_en: English name `name:en` if available, otherwise `name` (point features only). This is deprecated and will be removed in a future release in favor of `name:en`.
+    name_de: German name `name:de` if available, otherwise `name` or `name:en` (point features only). This is deprecated and will be removed in a future release in favor of `name:de`.
     rank: Rank of the park within one tile, starting at 1 that is the most important park (point features only).
   datasource:
     geometry_field: geometry
     query: (SELECT geometry, class, name, name_en, name_de, {name_languages}, rank FROM layer_park(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
 schema:
   - ./update_park_polygon.sql
-  - ./layer.sql
+  - ./park.sql
 datasources:
   - type: imposm3
     mapping_file: ./mapping.yaml

layers/park/style.json

@@ -0,0 +1,111 @@
+{
+  "layers": [
+    {
+      "id": "national_parks",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "park",
+      "minzoom": 8,
+      "layout": {
+        "visibility": "visible"
+      },
+      "paint": {
+        "line-color": "rgba(154, 199, 136, 1)",
+        "line-width": {
+          "base": 1,
+          "stops": [
+            [
+              8,
+              1.2
+            ],
+            [
+              9,
+              1.5
+            ],
+            [
+              10,
+              3.6
+            ],
+            [
+              24,
+              3.6
+            ]
+          ]
+        },
+        "line-offset": 1,
+        "line-opacity": 0.8
+      },
+      "order": 20
+    },
+    {
+      "id": "national_parks_thin",
+      "type": "line",
+      "source": "openmaptiles",
+      "source-layer": "park",
+      "minzoom": 10,
+      "layout": {
+        "visibility": "none"
+      },
+      "paint": {
+        "line-color": "rgba(93, 156, 76, 1)",
+        "line-width": 1.5
+      },
+      "order": 21
+    },
+    {
+      "id": "park-national",
+      "type": "symbol",
+      "source": "openmaptiles",
+      "source-layer": "park",
+      "minzoom": 7,
+      "maxzoom": 12,
+      "layout": {
+        "text-font": [
+          "Noto Sans Italic"
+        ],
+        "text-size": 12,
+        "text-field": "{name:latin}{name:nonlatin}",
+        "visibility": "visible",
+        "symbol-spacing": 150,
+        "text-allow-overlap": false
+      },
+      "paint": {
+        "text-color": {
+          "stops": [
+            [
+              7,
+              "rgba(70, 164, 70, 1)"
+            ],
+            [
+              10,
+              "#008000"
+            ]
+          ]
+        },
+        "text-halo-blur": 0.1,
+        "text-halo-color": {
+          "stops": [
+            [
+              7,
+              "rgba(241, 255, 234, 1)"
+            ],
+            [
+              10,
+              "rgba(208, 250, 200, 1)"
+            ]
+          ]
+        },
+        "text-halo-width": 0.3
+      },
+      "filter": [
+        "all",
+        [
+          "<=",
+          "rank",
+          2
+        ]
+      ],
+      "order": 195
+    }
+  ]
+}

layers/park/update_park_polygon.sql

@@ -1,140 +1,260 @@
-ALTER TABLE osm_park_polygon ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen1 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen2 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen3 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen4 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen5 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen6 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen7 ADD COLUMN IF NOT EXISTS geometry_point geometry;
-ALTER TABLE osm_park_polygon_gen8 ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z13
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z12
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z11
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z10
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z9
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z8
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z7
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z6
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+ALTER TABLE osm_park_polygon_gen_z5
+    ADD COLUMN IF NOT EXISTS geometry_point geometry;
+
+-- etldoc:  osm_park_polygon_gen_z4 -> osm_park_polygon_dissolve_z4
+DROP MATERIALIZED VIEW IF EXISTS osm_park_polygon_dissolve_z4 CASCADE;
+CREATE MATERIALIZED VIEW osm_park_polygon_dissolve_z4 AS
+(
+  SELECT min(osm_id) AS osm_id,
+         ST_Union(geometry) AS geometry,
+         boundary
+  FROM (
+        SELECT ST_ClusterDBSCAN(geometry, 0, 1) OVER() AS cluster,
+               osm_id,
+               geometry,
+               boundary
+        FROM osm_park_polygon_gen_z4
+  ) park_cluster
+  GROUP BY boundary, cluster
+);
+CREATE UNIQUE INDEX IF NOT EXISTS osm_park_polygon_dissolve_idx ON osm_park_polygon_dissolve_z4 (osm_id);
+
 DROP TRIGGER IF EXISTS update_row ON osm_park_polygon;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen1;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen2;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen3;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen4;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen5;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen6;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen7;
-DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen8;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z13;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z12;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z11;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z10;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z9;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z8;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z7;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z6;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z5;
+DROP TRIGGER IF EXISTS update_row ON osm_park_polygon_gen_z4;
+DROP TRIGGER IF EXISTS trigger_flag ON osm_park_polygon;
+DROP TRIGGER IF EXISTS trigger_refresh ON park_polygon.updates;
 
 -- etldoc:  osm_park_polygon ->  osm_park_polygon
--- etldoc:  osm_park_polygon_gen1 ->  osm_park_polygon_gen1
--- etldoc:  osm_park_polygon_gen2 ->  osm_park_polygon_gen2
--- etldoc:  osm_park_polygon_gen3 ->  osm_park_polygon_gen3
--- etldoc:  osm_park_polygon_gen4 ->  osm_park_polygon_gen4
--- etldoc:  osm_park_polygon_gen5 ->  osm_park_polygon_gen5
--- etldoc:  osm_park_polygon_gen6 ->  osm_park_polygon_gen6
--- etldoc:  osm_park_polygon_gen7 ->  osm_park_polygon_gen7
--- etldoc:  osm_park_polygon_gen8 ->  osm_park_polygon_gen8
-CREATE OR REPLACE FUNCTION update_osm_park_polygon() RETURNS VOID AS $$
+-- etldoc:  osm_park_polygon_gen_z13 ->  osm_park_polygon_gen_z13
+-- etldoc:  osm_park_polygon_gen_z12 ->  osm_park_polygon_gen_z12
+-- etldoc:  osm_park_polygon_gen_z11 ->  osm_park_polygon_gen_z11
+-- etldoc:  osm_park_polygon_gen_z10 ->  osm_park_polygon_gen_z10
+-- etldoc:  osm_park_polygon_gen_z9 ->  osm_park_polygon_gen_z9
+-- etldoc:  osm_park_polygon_gen_z8 ->  osm_park_polygon_gen_z8
+-- etldoc:  osm_park_polygon_gen_z7 ->  osm_park_polygon_gen_z7
+-- etldoc:  osm_park_polygon_gen_z6 ->  osm_park_polygon_gen_z6
+-- etldoc:  osm_park_polygon_gen_z5 ->  osm_park_polygon_gen_z5
+-- etldoc:  osm_park_polygon_gen_z4 ->  osm_park_polygon_gen_z4
+CREATE OR REPLACE FUNCTION update_osm_park_polygon() RETURNS void AS
+$$
 BEGIN
-  UPDATE osm_park_polygon
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen1
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z13
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen2
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z12
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen3
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z11
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen4
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z10
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen5
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z9
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen6
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z8
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen7
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z7
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
-  UPDATE osm_park_polygon_gen8
-  SET tags = update_tags(tags, geometry),
-      geometry_point = st_centroid(geometry);
+    UPDATE osm_park_polygon_gen_z6
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
 
+    UPDATE osm_park_polygon_gen_z5
+    SET tags           = update_tags(tags, geometry),
+        geometry_point = ST_PointOnSurface(geometry);
+
+    REFRESH MATERIALIZED VIEW CONCURRENTLY osm_park_polygon_dissolve_z4;
 END;
 $$ LANGUAGE plpgsql;
 
 SELECT update_osm_park_polygon();
-CREATE INDEX IF NOT EXISTS osm_park_polygon_point_geom_idx ON osm_park_polygon USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen1_point_geom_idx ON osm_park_polygon_gen1 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen2_point_geom_idx ON osm_park_polygon_gen2 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen3_point_geom_idx ON osm_park_polygon_gen3 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen4_point_geom_idx ON osm_park_polygon_gen4 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen5_point_geom_idx ON osm_park_polygon_gen5 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen6_point_geom_idx ON osm_park_polygon_gen6 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen7_point_geom_idx ON osm_park_polygon_gen7 USING gist(geometry_point);
-CREATE INDEX IF NOT EXISTS osm_park_polygon_gen8_point_geom_idx ON osm_park_polygon_gen8 USING gist(geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_point_geom_idx ON osm_park_polygon USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z13_point_geom_idx ON osm_park_polygon_gen_z13 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z12_point_geom_idx ON osm_park_polygon_gen_z12 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z11_point_geom_idx ON osm_park_polygon_gen_z11 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z10_point_geom_idx ON osm_park_polygon_gen_z10 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z9_point_geom_idx ON osm_park_polygon_gen_z9 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z8_point_geom_idx ON osm_park_polygon_gen_z8 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z7_point_geom_idx ON osm_park_polygon_gen_z7 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z6_point_geom_idx ON osm_park_polygon_gen_z6 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z5_point_geom_idx ON osm_park_polygon_gen_z5 USING gist (geometry_point);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_gen_z4_polygon_geom_idx ON osm_park_polygon_gen_z4 USING gist (geometry);
+CREATE INDEX IF NOT EXISTS osm_park_polygon_dissolve_z4_polygon_geom_idx ON osm_park_polygon_dissolve_z4 USING gist (geometry);
 
+CREATE SCHEMA IF NOT EXISTS park_polygon;
+
+CREATE TABLE IF NOT EXISTS park_polygon.updates
+(
+    id serial PRIMARY KEY,
+    t  text,
+    UNIQUE (t)
+);
+
+CREATE OR REPLACE FUNCTION park_polygon.flag() RETURNS trigger AS
+$$
+BEGIN
+    INSERT INTO park_polygon.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION park_polygon.refresh() RETURNS trigger AS
+$$
+DECLARE
+    t TIMESTAMP WITH TIME ZONE := clock_timestamp();
+BEGIN
+    RAISE LOG 'Refresh park_polygon';
+
+    -- Analyze tracking and source tables before performing update
+    ANALYZE osm_park_polygon_gen_z4;
+    REFRESH MATERIALIZED VIEW osm_park_polygon_dissolve_z4;
+
+    -- noinspection SqlWithoutWhere
+    DELETE FROM park_polygon.updates;
+
+    RAISE LOG 'Refresh park_polygon done in %', age(clock_timestamp(), t);
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION update_osm_park_polygon_row()
-  RETURNS TRIGGER
+    RETURNS trigger
 AS
-$BODY$
+$$
 BEGIN
-  NEW.tags = update_tags(NEW.tags, NEW.geometry);
-  NEW.geometry_point = st_centroid(NEW.geometry);
-  RETURN NEW;
+    NEW.tags = update_tags(NEW.tags, NEW.geometry);
+    NEW.geometry_point = ST_PointOnSurface(NEW.geometry);
+    RETURN NEW;
 END;
-$BODY$
-LANGUAGE plpgsql;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION update_osm_park_dissolved_polygon_row()
+    RETURNS trigger
+AS
+$$
+BEGIN
+    NEW.tags = update_tags(NEW.tags, NEW.geometry);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen1
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z13
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen2
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z12
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen3
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z11
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen4
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z10
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen5
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z9
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen6
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z8
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen7
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z7
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
 CREATE TRIGGER update_row
-BEFORE INSERT OR UPDATE ON osm_park_polygon_gen8
-FOR EACH ROW
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z6
+    FOR EACH ROW
 EXECUTE PROCEDURE update_osm_park_polygon_row();
 
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z5
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_park_polygon_row();
 
+CREATE TRIGGER update_row
+    BEFORE INSERT OR UPDATE
+    ON osm_park_polygon_gen_z4
+    FOR EACH ROW
+EXECUTE PROCEDURE update_osm_park_dissolved_polygon_row();
 
+CREATE TRIGGER trigger_flag
+    AFTER INSERT OR UPDATE OR DELETE
+    ON osm_park_polygon
+    FOR EACH STATEMENT
+EXECUTE PROCEDURE park_polygon.flag();
+
+CREATE CONSTRAINT TRIGGER trigger_refresh
+    AFTER INSERT
+    ON park_polygon.updates
+    INITIALLY DEFERRED
+    FOR EACH ROW
+EXECUTE PROCEDURE park_polygon.refresh();

layers/place/area_rank.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION area_rank(area real) RETURNS int AS
+$$
+SELECT CASE
+           WHEN area > 640000000 THEN 1
+           WHEN area > 160000000 THEN 2
+           WHEN area > 40000000 THEN 3
+           WHEN area > 15000000 THEN 4
+           WHEN area > 10000000 THEN 5
+           WHEN area > 0 THEN 6
+           ELSE 7
+           END;
+$$ LANGUAGE SQL IMMUTABLE
+                STRICT
+                PARALLEL SAFE;

layers/place/capital.sql

@@ -1,8 +1,10 @@
-CREATE OR REPLACE FUNCTION normalize_capital_level(capital TEXT)
-RETURNS INT AS $$
-    SELECT CASE
-        WHEN capital IN ('yes', '2') THEN 2
-        WHEN capital = '4' THEN 4
-        ELSE NULL
-    END;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+CREATE OR REPLACE FUNCTION normalize_capital_level(capital text)
+    RETURNS int AS
+$$
+SELECT CASE
+           WHEN capital = 'yes' THEN 2
+           WHEN capital IN ('2', '3', '4', '5', '6') THEN capital::int
+           END;
+$$ LANGUAGE SQL IMMUTABLE
+                STRICT
+                PARALLEL SAFE;

layers/place/city.sql

@@ -1,55 +1,79 @@
-
 -- etldoc: layer_city[shape=record fillcolor=lightpink, style="rounded,filled",
 -- etldoc:     label="layer_city | <z2_14> z2-z14+" ] ;
 
 -- etldoc: osm_city_point -> layer_city:z2_14
 CREATE OR REPLACE FUNCTION layer_city(bbox geometry, zoom_level int, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de text, tags hstore, place city_place, "rank" int, capital int) AS $$
-  SELECT * FROM (
-    SELECT osm_id, geometry, name,
-    COALESCE(NULLIF(name_en, ''), name) AS name_en,
-    COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-    tags,
-    place, "rank", normalize_capital_level(capital) AS capital
-    FROM osm_city_point
-    WHERE geometry && bbox
-      AND ((zoom_level = 2 AND "rank" = 1)
-        OR (zoom_level BETWEEN 3 AND 7 AND "rank" <= zoom_level + 1)
-      )
-    UNION ALL
-    SELECT osm_id, geometry, name,
-        COALESCE(NULLIF(name_en, ''), name) AS name_en,
-        COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-        tags,
-        place,
-        COALESCE("rank", gridrank + 10),
-        normalize_capital_level(capital) AS capital
-    FROM (
-      SELECT osm_id, geometry, name,
-      COALESCE(NULLIF(name_en, ''), name) AS name_en,
-      COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-      tags,
-      place, "rank", capital,
-      row_number() OVER (
-        PARTITION BY LabelGrid(geometry, 128 * pixel_width)
-        ORDER BY "rank" ASC NULLS LAST,
-        place ASC NULLS LAST,
-        population DESC NULLS LAST,
-        length(name) ASC
-      )::int AS gridrank
-        FROM osm_city_point
-        WHERE geometry && bbox
-          AND ((zoom_level = 7 AND place <= 'town'::city_place
-            OR (zoom_level BETWEEN 8 AND 10 AND place <= 'village'::city_place)
-
-            OR (zoom_level BETWEEN 11 AND 13 AND place <= 'suburb'::city_place)
-            OR (zoom_level >= 14)
-          ))
-    ) AS ranked_places
-    WHERE (zoom_level BETWEEN 7 AND 8 AND (gridrank <= 4 OR "rank" IS NOT NULL))
-       OR (zoom_level = 9 AND (gridrank <= 8 OR "rank" IS NOT NULL))
-       OR (zoom_level = 10 AND (gridrank <= 12 OR "rank" IS NOT NULL))
-       OR (zoom_level BETWEEN 11 AND 12 AND (gridrank <= 14 OR "rank" IS NOT NULL))
-       OR (zoom_level >= 13)
-  ) as city_all;
-$$ LANGUAGE SQL IMMUTABLE;
+    RETURNS TABLE
+            (
+                osm_id   bigint,
+                geometry geometry,
+                name     text,
+                name_en  text,
+                name_de  text,
+                tags     hstore,
+                place    city_place,
+                "rank"   int,
+                capital  int
+            )
+AS
+$$
+SELECT *
+FROM (
+         SELECT osm_id,
+                geometry,
+                name,
+                COALESCE(NULLIF(name_en, ''), name) AS name_en,
+                COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+                tags,
+                place,
+                "rank",
+                normalize_capital_level(capital) AS capital
+         FROM osm_city_point
+         WHERE geometry && bbox
+           AND ((zoom_level = 2 AND "rank" = 1)
+             OR (zoom_level BETWEEN 3 AND 7 AND "rank" <= zoom_level + 1)
+             )
+         UNION ALL
+         SELECT osm_id,
+                geometry,
+                name,
+                COALESCE(NULLIF(name_en, ''), name) AS name_en,
+                COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+                tags,
+                place,
+                COALESCE("rank", gridrank + 10),
+                normalize_capital_level(capital) AS capital
+         FROM (
+                  SELECT osm_id,
+                         geometry,
+                         name,
+                         COALESCE(NULLIF(name_en, ''), name) AS name_en,
+                         COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+                         tags,
+                         place,
+                         "rank",
+                         capital,
+                         row_number() OVER (
+                             PARTITION BY LabelGrid(geometry, 128 * pixel_width)
+                             ORDER BY "rank" ASC NULLS LAST,
+                                 place ASC NULLS LAST,
+                                 population DESC NULLS LAST,
+                                 length(name) ASC
+                             )::int AS gridrank
+                  FROM osm_city_point
+                  WHERE geometry && bbox
+                    AND ((zoom_level = 7 AND place <= 'town'::city_place
+                      OR (zoom_level BETWEEN 8 AND 10 AND place <= 'village'::city_place)
+                      OR (zoom_level BETWEEN 11 AND 13 AND place <= 'suburb'::city_place)
+                      OR (zoom_level >= 14)
+                      ))
+              ) AS ranked_places
+         WHERE (zoom_level BETWEEN 7 AND 8 AND (gridrank <= 4 OR "rank" IS NOT NULL))
+            OR (zoom_level = 9 AND (gridrank <= 8 OR "rank" IS NOT NULL))
+            OR (zoom_level = 10 AND (gridrank <= 12 OR "rank" IS NOT NULL))
+            OR (zoom_level BETWEEN 11 AND 12 AND (gridrank <= 14 OR "rank" IS NOT NULL))
+            OR (zoom_level >= 13)
+     ) AS city_all;
+$$ LANGUAGE SQL STABLE
+                -- STRICT
+                PARALLEL SAFE;

layers/place/island_rank.sql

@@ -1,9 +0,0 @@
-CREATE OR REPLACE FUNCTION island_rank(area REAL) RETURNS INT AS $$
-    SELECT CASE
-        WHEN area < 10000000 THEN 6
-        WHEN area BETWEEN  1000000 AND 15000000 THEN 5
-        WHEN area BETWEEN 15000000 AND 40000000 THEN 4
-        WHEN area > 40000000 THEN 3
-        ELSE 7
-    END;
-$$ LANGUAGE SQL IMMUTABLE STRICT;

layers/place/layer.sql

@@ -1,99 +0,0 @@
-
--- etldoc: layer_place[shape=record fillcolor=lightpink, style="rounded,filled",
--- etldoc:     label="layer_place | <z0_3> z0-3|<z4_7> z4-7|<z8_11> z8-11| <z12_14> z12-z14+" ] ;
-
-CREATE OR REPLACE FUNCTION layer_place(bbox geometry, zoom_level int, pixel_width numeric)
-RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text,
-    name_de text, tags hstore, class text, "rank" int, capital INT, iso_a2
-        TEXT) AS $$
-    SELECT * FROM (
-
-    -- etldoc: osm_continent_point -> layer_place:z0_3
-    SELECT
-        osm_id*10, geometry, name,
-        COALESCE(NULLIF(name_en, ''), name) AS name_en,
-        COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-        tags,
-        'continent' AS class, 1 AS "rank", NULL::int AS capital,
-        NULL::text AS iso_a2
-    FROM osm_continent_point
-    WHERE geometry && bbox AND zoom_level < 4
-    UNION ALL
-
-    -- etldoc: osm_country_point -> layer_place:z0_3
-    -- etldoc: osm_country_point -> layer_place:z4_7
-    -- etldoc: osm_country_point -> layer_place:z8_11
-    -- etldoc: osm_country_point -> layer_place:z12_14
-    SELECT
-        osm_id*10, geometry, name,
-        COALESCE(NULLIF(name_en, ''), name) AS name_en,
-        COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-        tags,
-        'country' AS class, "rank", NULL::int AS capital,
-        iso3166_1_alpha_2 AS iso_a2
-    FROM osm_country_point
-    WHERE geometry && bbox AND "rank" <= zoom_level + 1 AND name <> ''
-    UNION ALL
-
-    -- etldoc: osm_state_point  -> layer_place:z0_3
-    -- etldoc: osm_state_point  -> layer_place:z4_7
-    -- etldoc: osm_state_point  -> layer_place:z8_11
-    -- etldoc: osm_state_point  -> layer_place:z12_14
-    SELECT
-        osm_id*10, geometry, name,
-        COALESCE(NULLIF(name_en, ''), name) AS name_en,
-        COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-        tags,
-        'state' AS class, "rank", NULL::int AS capital,
-        NULL::text AS iso_a2
-    FROM osm_state_point
-    WHERE geometry && bbox AND
-          name <> '' AND
-          ("rank" + 2 <= zoom_level) AND (
-              zoom_level >= 5 OR
-              is_in_country IN ('United Kingdom', 'USA', 'Россия', 'Brasil', 'China', 'India') OR
-              is_in_country_code IN ('AU', 'CN', 'IN', 'BR', 'US'))
-    UNION ALL
-
-    -- etldoc: osm_island_point    -> layer_place:z12_14
-    SELECT
-        osm_id*10, geometry, name,
-        COALESCE(NULLIF(name_en, ''), name) AS name_en,
-        COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-        tags,
-        'island' AS class, 7 AS "rank", NULL::int AS capital,
-        NULL::text AS iso_a2
-    FROM osm_island_point
-    WHERE zoom_level >= 12
-        AND geometry && bbox
-    UNION ALL
-
-    -- etldoc: osm_island_polygon  -> layer_place:z8_11
-    -- etldoc: osm_island_polygon  -> layer_place:z12_14
-    SELECT
-        osm_id*10, geometry, name,
-        COALESCE(NULLIF(name_en, ''), name) AS name_en,
-        COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
-        tags,
-        'island' AS class, island_rank(area) AS "rank", NULL::int AS capital,
-        NULL::text AS iso_a2
-    FROM osm_island_polygon
-    WHERE geometry && bbox AND
-        ((zoom_level = 8 AND island_rank(area) <= 3)
-        OR (zoom_level = 9 AND island_rank(area) <= 4)
-        OR (zoom_level >= 10))
-    UNION ALL
-
-    -- etldoc: layer_city          -> layer_place:z0_3
-    -- etldoc: layer_city          -> layer_place:z4_7
-    -- etldoc: layer_city          -> layer_place:z8_11
-    -- etldoc: layer_city          -> layer_place:z12_14
-    SELECT
-        osm_id*10, geometry, name, name_en, name_de,
-        tags,
-        place::text AS class, "rank", capital,
-        NULL::text AS iso_a2
-    FROM layer_city(bbox, zoom_level, pixel_width)
-    ORDER BY "rank" ASC
-    ) AS place_all
-$$ LANGUAGE SQL IMMUTABLE;

layers/place/mapping.yaml

@@ -20,7 +20,7 @@ tables:
   # etldoc: imposm3 -> osm_continent_point
   continent_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -40,7 +40,7 @@ tables:
   # etldoc: imposm3 -> osm_country_point
   country_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -70,7 +70,7 @@ tables:
   # etldoc: imposm3 -> osm_island_polygon
   island_polygon:
     type: polygon
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -93,7 +93,7 @@ tables:
   # etldoc: imposm3 -> osm_island_point
   island_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -114,7 +114,7 @@ tables:
   # etldoc: imposm3 -> osm_state_point
   state_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -124,6 +124,9 @@ tables:
     - *name_de
     - name: tags
       type: hstore_tags
+    - name: place
+      key: place
+      type: string
     - name: is_in_country
       key: is_in:country
       type: string
@@ -140,11 +143,12 @@ tables:
     mapping:
       place:
       - state
+      - province
 
   # etldoc: imposm3 -> osm_city_point
   city_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -173,6 +177,8 @@ tables:
       - town
       - village
       - hamlet
+      - borough
       - suburb
+      - quarter
       - neighbourhood
       - isolated_dwelling

layers/place/place.sql

@@ -0,0 +1,165 @@
+-- etldoc: layer_place[shape=record fillcolor=lightpink, style="rounded,filled",
+-- etldoc:     label="layer_place | <z0_3> z0-3|<z4_7> z4-7|<z8_11> z8-11| <z12_14> z12-z14+" ] ;
+
+CREATE OR REPLACE FUNCTION layer_place(bbox geometry, zoom_level int, pixel_width numeric)
+    RETURNS TABLE
+            (
+                osm_id   bigint,
+                geometry geometry,
+                name     text,
+                name_en  text,
+                name_de  text,
+                tags     hstore,
+                class    text,
+                "rank"   int,
+                capital  int,
+                iso_a2   text
+            )
+AS
+$$
+SELECT *
+FROM (
+         SELECT
+             -- etldoc: osm_continent_point -> layer_place:z0_3
+             osm_id * 10 AS osm_id,
+             geometry,
+             name,
+             COALESCE(NULLIF(name_en, ''), name) AS name_en,
+             COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+             tags,
+             'continent' AS class,
+             1 AS "rank",
+             NULL::int AS capital,
+             NULL::text AS iso_a2
+         FROM osm_continent_point
+         WHERE geometry && bbox
+           AND zoom_level < 4
+
+         UNION ALL
+
+         SELECT
+             -- etldoc: osm_country_point -> layer_place:z0_3
+             -- etldoc: osm_country_point -> layer_place:z4_7
+             -- etldoc: osm_country_point -> layer_place:z8_11
+             -- etldoc: osm_country_point -> layer_place:z12_14
+             osm_id * 10 AS osm_id,
+             geometry,
+             name,
+             COALESCE(NULLIF(name_en, ''), name) AS name_en,
+             COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+             tags,
+             'country' AS class,
+             "rank",
+             NULL::int AS capital,
+             iso3166_1_alpha_2 AS iso_a2
+         FROM osm_country_point
+         WHERE geometry && bbox
+           AND "rank" <= zoom_level + 1
+           AND name <> ''
+
+         UNION ALL
+
+         SELECT
+             -- etldoc: osm_state_point  -> layer_place:z0_3
+             -- etldoc: osm_state_point  -> layer_place:z4_7
+             -- etldoc: osm_state_point  -> layer_place:z8_11
+             -- etldoc: osm_state_point  -> layer_place:z12_14
+             osm_id * 10 AS osm_id,
+             geometry,
+             name,
+             COALESCE(NULLIF(name_en, ''), name) AS name_en,
+             COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+             tags,
+             place::text AS class,
+             "rank",
+             NULL::int AS capital,
+             NULL::text AS iso_a2
+         FROM osm_state_point
+         WHERE geometry && bbox
+           AND name <> ''
+           AND zoom_level > 1
+
+         UNION ALL
+
+         SELECT
+             -- etldoc: osm_island_point    -> layer_place:z12_14
+             osm_id * 10 AS osm_id,
+             geometry,
+             name,
+             COALESCE(NULLIF(name_en, ''), name) AS name_en,
+             COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+             tags,
+             'island' AS class,
+             7 AS "rank",
+             NULL::int AS capital,
+             NULL::text AS iso_a2
+         FROM osm_island_point
+         WHERE zoom_level >= 12
+           AND geometry && bbox
+
+         UNION ALL
+
+         SELECT
+             -- etldoc: osm_island_polygon  -> layer_place:z8_11
+             -- etldoc: osm_island_polygon  -> layer_place:z12_14
+             osm_id * 10 AS osm_id,
+             geometry,
+             name,
+             COALESCE(NULLIF(name_en, ''), name) AS name_en,
+             COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
+             tags,
+             'island' AS class,
+             area_rank(area) AS "rank",
+             NULL::int AS capital,
+             NULL::text AS iso_a2
+         FROM osm_island_polygon
+         WHERE geometry && bbox
+           AND ((zoom_level = 8 AND area_rank(area) <= 3)
+             OR (zoom_level = 9 AND area_rank(area) <= 4)
+             OR (zoom_level >= 10))
+
+         UNION ALL
+
+         SELECT
+             -- etldoc: osm_boundary_polygon  -> layer_place:z6_11
+             -- etldoc: osm_boundary_polygon  -> layer_place:z12_14
+             osm_id * 10 AS osm_id,
+             geometry_point,
+             name,
+             NULL::text AS name_en, -- deprecated
+             NULL::text AS name_de, -- deprecated
+             tags,
+             'aboriginal_lands' AS class,
+             area_rank(area) AS "rank",
+             NULL::int AS capital,
+             NULL::text AS iso_a2
+         FROM osm_boundary_polygon
+         WHERE geometry_point && bbox
+           AND ((zoom_level = 6 AND area_rank(area) <= 1)
+             OR (zoom_level = 7 AND area_rank(area) <= 2)
+             OR (zoom_level = 8 AND area_rank(area) <= 3)
+             OR (zoom_level = 9 AND area_rank(area) <= 4)
+             OR (zoom_level >= 10))
+         UNION ALL
+
+         SELECT
+             -- etldoc: layer_city          -> layer_place:z0_3
+             -- etldoc: layer_city          -> layer_place:z4_7
+             -- etldoc: layer_city          -> layer_place:z8_11
+             -- etldoc: layer_city          -> layer_place:z12_14
+             osm_id * 10 AS osm_id,
+             geometry,
+             name,
+             name_en,
+             name_de,
+             tags,
+             place::text AS class,
+             "rank",
+             capital,
+             NULL::text AS iso_a2
+         FROM layer_city(bbox, zoom_level, pixel_width)
+         ORDER BY "rank" ASC
+     ) AS place_all
+$$ LANGUAGE SQL STABLE
+                PARALLEL SAFE;
+-- TODO: Check if the above can be made STRICT -- i.e. if pixel_width could be NULL

layers/place/place.yaml

@@ -1,49 +1,64 @@
 layer:
   id: "place"
+  requires:
+    tables:
+      - ne_10m_admin_1_states_provinces
+      - ne_10m_admin_0_countries
+      - ne_10m_populated_places
+    layers:
+      - boundary
   description: |
       The place layer consists out of [countries](http://wiki.openstreetmap.org/wiki/Tag:place%3Dcountry),
-      [states](http://wiki.openstreetmap.org/wiki/Tag:place%3Dstate) and [cities](http://wiki.openstreetmap.org/wiki/Key:place).
+      [states](http://wiki.openstreetmap.org/wiki/Tag:place%3Dstate), [cities](http://wiki.openstreetmap.org/wiki/Key:place)
+      and [islands](https://wiki.openstreetmap.org/wiki/Tag:place%3Disland).
       Apart from the roads this is also one of the more important layers to create a beautiful map.
       We suggest you use different font styles and sizes to create a text hierarchy.
   fields:
-    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the POI.
-    name_en: English name `name:en` if available, otherwise `name`.
-    name_de: German name `name:de` if available, otherwise `name` or `name:en`.
+    name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the place. Language-specific values are in `name:xx`.
+    name_en: English name `name:en` if available, otherwise `name`. This is deprecated and will be removed in a future release in favor of `name:en`.
+    name_de: German name `name:de` if available, otherwise `name` or `name:en`. This is deprecated and will be removed in a future release in favor of `name:de`.
     capital:
       description: |
           The **capital** field marks the
           [`admin_level`](http://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative#admin_level)
           of the boundary the place is a capital of.
-      values: [2, 4]
+      values: [2, 3, 4, 5, 6]
     class:
       description: |
           Original value of the
           [`place`](http://wiki.openstreetmap.org/wiki/Key:place) tag.
-          Distinguish between continents, countries, states and
+          Distinguish between continents, countries, states, islands and
           places like settlements or smaller entities.
           Use **class** to separately style the different places and build
-          a text hierarchy according to their importance.
+          a text hierarchy according to their importance. For places derived 
+          from boundaries, the original value of the 
+          [`boundary`](http://wiki.openstreetmap.org/wiki/Key:boundary) tag.
       values:
       - continent
       - country
       - state
+      - province
       - city
       - town
       - village
       - hamlet
+      - borough
       - suburb
+      - quarter
       - neighbourhood
       - isolated_dwelling
+      - island
+      - aboriginal_lands
     iso_a2:
       description: |
-          Two-letter country code [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2).
+          Two-letter country code [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2). Available only for `class=country`.
           Original value of the
           [`country_code_iso3166_1_alpha_2`](http://wiki.openstreetmap.org/wiki/Tag:place%3Dcountry) tag.
     rank:
       description: |
           Countries, states and the most important cities all have a
           **rank** to boost their importance on the map.
-          The **rank** field for counries and states ranges from
+          The **rank** field for countries and states ranges from
           `1` to `6` while the **rank** field for cities ranges from
           `1` to `10` for the most important cities
           and continues from `10` serially based on the
@@ -65,14 +80,14 @@ schema:
   - ./types.sql
   - ./capital.sql
   - ./city.sql
-  - ./island_rank.sql
+  - ./area_rank.sql
   - ./update_continent_point.sql
   - ./update_country_point.sql
   - ./update_island_polygon.sql
   - ./update_island_point.sql
   - ./update_state_point.sql
   - ./update_city_point.sql
-  - ./layer.sql
+  - ./place.sql
 datasources:
   - type: imposm3
     mapping_file: ./mapping.yaml