kopia lustrzana https://github.com/onthegomap/planetiler
rodzic
798bc544a6
commit
39b7eca2a1
|
@ -0,0 +1,32 @@
|
|||
---
|
||||
name: Bug report
|
||||
about: Create a report to help improve Flatmap
|
||||
title: "[BUG] "
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Download data from '...'
|
||||
2. Run command '....'
|
||||
3. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem (include tile IDs or latitude/longitude for visual issues with generated maps)
|
||||
|
||||
**Environment (please complete the following information):**
|
||||
- Hardware: [e.g. 2015 Macbook Pro]
|
||||
- OS: [e.g. MacOS 10.15.7]
|
||||
- Java version and distribution: [e.g. Eclipse Temurin 17.35]
|
||||
- Maven version: [e.g. 3.8.1]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for Flatmap
|
||||
title: "[FEATURE] "
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
|
@ -0,0 +1,22 @@
|
|||
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions
|
||||
name: 'Cache data/sources'
|
||||
description: 'Save/restore data/sources cache'
|
||||
inputs:
|
||||
basedir:
|
||||
description: 'Base dir for computing file hash'
|
||||
required: false
|
||||
default: ''
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Get Date
|
||||
id: get-data
|
||||
run: |
|
||||
echo "::set-output name=hash::${{ hashFiles('**/FlatmapRunner.java', '**/Downloader.java', '**/Geofabrik.java', '**/BasemapMain.java') }}"
|
||||
echo "::set-output name=date::$(date -u "+%Y-%m-%d")"
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.basedir }}
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: data/sources
|
||||
key: data-sources-${{ steps.get-data.outputs.date }}-${{ steps.get-data.outputs.hash }}
|
|
@ -0,0 +1,33 @@
|
|||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: maven
|
||||
directory: "/"
|
||||
open-pull-requests-limit: 1
|
||||
schedule:
|
||||
interval: daily
|
||||
time: "04:30"
|
||||
timezone: America/New_York
|
||||
labels:
|
||||
- dependencies
|
||||
ignore:
|
||||
- dependency-name: "com.graphhopper:graphhopper-reader-osm"
|
||||
- package-ecosystem: maven
|
||||
directory: "/flatmap-examples"
|
||||
open-pull-requests-limit: 1
|
||||
schedule:
|
||||
interval: daily
|
||||
time: "04:30"
|
||||
timezone: America/New_York
|
||||
labels:
|
||||
- dependencies
|
||||
ignore:
|
||||
- dependency-name: "com.onthegomap.flatmap:*"
|
||||
- package-ecosystem: github-actions
|
||||
directory: "/"
|
||||
open-pull-requests-limit: 1
|
||||
schedule:
|
||||
interval: daily
|
||||
time: "04:30"
|
||||
timezone: America/New_York
|
||||
labels:
|
||||
- dependencies
|
|
@ -0,0 +1,21 @@
|
|||
---
|
||||
name: Docs
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
markdown-link-check:
|
||||
name: Broken Links
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Run link check
|
||||
uses: gaurav-nelson/github-action-markdown-link-check@v1
|
||||
with:
|
||||
use-quiet-mode: 'no'
|
||||
use-verbose-mode: 'yes'
|
||||
config-file: '.github/workflows/docs_mlc_config.json'
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"ignorePatterns": [
|
||||
{
|
||||
"pattern": "^https://github.com/onthegomap/flatmap/.*$"
|
||||
},
|
||||
{
|
||||
"pattern": "^https://onthegomap.github.io/.*$"
|
||||
},
|
||||
{
|
||||
"pattern": "^http://localhost.*$"
|
||||
}
|
||||
]
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
# This workflow will build a Java project with Maven
|
||||
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
|
||||
|
||||
name: Java CI with Maven
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
|
@ -10,23 +10,94 @@ on:
|
|||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
# TODO: add format/checkstyle
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
name: Java ${{ matrix.jdk }} / ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ ubuntu-latest ]
|
||||
# os: [ ubuntu-latest, macos-latest, windows-latest ]
|
||||
jdk: [ 17 ]
|
||||
# include:
|
||||
# - os: ubuntu-latest
|
||||
# jdk: 16
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up JDK 16
|
||||
- name: Set up JDK ${{ matrix.jdk }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: '16'
|
||||
distribution: 'adopt'
|
||||
- name: Cache Maven packages
|
||||
uses: actions/cache@v2
|
||||
java-version: ${{ matrix.jdk }}
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
- name: Build with mvnw (linux/mac)
|
||||
if: ${{ !contains(matrix.os, 'windows') }}
|
||||
run: ./mvnw --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
|
||||
- name: Build with mvnw.cmd (windows)
|
||||
if: ${{ contains(matrix.os, 'windows') }}
|
||||
run: mvnw.cmd --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
|
||||
shell: cmd
|
||||
|
||||
examples:
|
||||
name: Example project
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: ${{ runner.os }}-m2
|
||||
- name: Build with Maven
|
||||
run: mvn -B package --file pom.xml
|
||||
java-version: 17
|
||||
distribution: 'temurin'
|
||||
# do not cache to ensure we can resolve dependencies
|
||||
server-id: 'github-flatmap'
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
- name: Build and test
|
||||
run: mvn --batch-mode -no-transfer-progress package --file pom.xml
|
||||
working-directory: flatmap-examples
|
||||
# workaround for github maven packages not supporting anonymous access (https://github.community/t/download-from-github-package-registry-without-authentication/14407/111)
|
||||
env:
|
||||
MAVEN_USERNAME: 'flatmapbot'
|
||||
MAVEN_PASSWORD: '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'
|
||||
- name: Find jar
|
||||
run: mv target/*with-deps.jar ./run.jar
|
||||
working-directory: flatmap-examples
|
||||
- name: Run
|
||||
run: java -jar run.jar --osm-path=../flatmap-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles
|
||||
working-directory: flatmap-examples
|
||||
- name: Verify
|
||||
run: java -cp run.jar com.onthegomap.flatmap.mbtiles.Verify data/out.mbtiles
|
||||
working-directory: flatmap-examples
|
||||
|
||||
run:
|
||||
name: Build / Run
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Cache data/sources
|
||||
uses: ./.github/cache-sources-action
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build this branch
|
||||
run: ./mvnw -DskipTests -Dimage.version=CI_ONLY --batch-mode -no-transfer-progress package jib:dockerBuild --file pom.xml
|
||||
|
||||
- name: Download data (java)
|
||||
run: java -jar flatmap-dist/target/*with-deps.jar --only-download --area=monaco
|
||||
|
||||
- name: Download wikidata (java)
|
||||
run: java -jar flatmap-dist/target/*with-deps.jar --only-fetch-wikidata --area=monaco
|
||||
|
||||
- name: Verify build
|
||||
run: ./scripts/test-release.sh CI_ONLY
|
||||
env:
|
||||
SKIP_EXAMPLE_PROJECT: true
|
||||
|
|
|
@ -0,0 +1,92 @@
|
|||
# This workflow builds a map using the base and branch commit of a PR and uploads
|
||||
# the logs as an artifact that update-pr.yml uses to add back as a comment.
|
||||
|
||||
name: Performance
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
env:
|
||||
# For performance tests, run this branch against main with:
|
||||
AREA: rhode island
|
||||
RAM: 4g
|
||||
# Also pick up a good chunk of the atlantic ocean to catch any regressions in repeated tile performance
|
||||
# Omit to infer from .osm.pbf bounds
|
||||
BOUNDS_ARG: "--bounds=-74.07,21.34,-17.84,43.55"
|
||||
|
||||
jobs:
|
||||
performance:
|
||||
name: Performance Test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: 'Cancel previous runs'
|
||||
uses: styfle/cancel-workflow-action@0.9.1
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- name: 'Checkout branch'
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
path: branch
|
||||
- name: 'Checkout base'
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
path: base
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Cache data/sources
|
||||
uses: ./branch/.github/cache-sources-action
|
||||
with:
|
||||
basedir: branch
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '14'
|
||||
- run: npm install -g strip-ansi-cli@3.0.2
|
||||
|
||||
- name: 'Build branch'
|
||||
run: ./scripts/build.sh
|
||||
working-directory: branch
|
||||
- name: 'Build base'
|
||||
run: ./scripts/build.sh
|
||||
working-directory: base
|
||||
|
||||
- name: 'Download data'
|
||||
run: |
|
||||
set -eo pipefail
|
||||
cp base/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
|
||||
cp branch/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
|
||||
|
||||
- name: 'Store build info'
|
||||
run: |
|
||||
mkdir build-info
|
||||
echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha
|
||||
echo "${{ github.sha }}" > build-info/branch_sha
|
||||
echo "${{ github.event.number }}" > build-info/pull_request_number
|
||||
|
||||
- name: 'Run branch'
|
||||
run: |
|
||||
rm -f data/out.mbtiles
|
||||
cp branch/flatmap-dist/target/*with-deps.jar run.jar
|
||||
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
|
||||
ls -alh run.jar | tee -a log
|
||||
cat log | strip-ansi > build-info/branchlogs.txt
|
||||
- name: 'Run base'
|
||||
run: |
|
||||
rm -f data/out.mbtiles
|
||||
cp base/flatmap-dist/target/*with-deps.jar run.jar
|
||||
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
|
||||
ls -alh run.jar | tee -a log
|
||||
cat log | strip-ansi > build-info/baselogs.txt
|
||||
|
||||
- name: 'Upload build-info'
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: build-info
|
||||
path: ./build-info
|
|
@ -0,0 +1,74 @@
|
|||
name: Publish a Release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version without leading "v" (1.0, 2.3.4, 0.2-alpha, 5.9-beta.3)'
|
||||
required: true
|
||||
default: ''
|
||||
image_tags:
|
||||
description: 'Extra docker image tags ("latest,test")'
|
||||
required: true
|
||||
default: 'latest'
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
steps:
|
||||
- name: Ensure version does not start with 'v'
|
||||
uses: actions/github-script@v5
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
script: |
|
||||
version = context.payload.inputs.version;
|
||||
if (/^v/.test(version)) throw new Error("Bad version number: " + version)
|
||||
- uses: actions/checkout@v2
|
||||
- name: Cache data/sources
|
||||
uses: ./.github/cache-sources-action
|
||||
- uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
|
||||
- name: Check tag does not exist yet
|
||||
run: if git rev-list "v${{ github.event.inputs.version }}"; then echo "Tag already exists. Aborting the release process."; exit 1; fi
|
||||
|
||||
- run: ./scripts/set-versions.sh "${{ github.event.inputs.version }}"
|
||||
- run: ./scripts/build-release.sh
|
||||
- run: ./scripts/test-release.sh "${{ github.event.inputs.version }}"
|
||||
- name: Create tag
|
||||
uses: actions/github-script@v5
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
script: |
|
||||
github.rest.git.createRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: "refs/tags/v${{ github.event.inputs.version }}",
|
||||
sha: context.sha
|
||||
})
|
||||
- run: mv flatmap-dist/target/*with-deps.jar flatmap.jar
|
||||
- name: Log in to the Container Registry
|
||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
fail_on_unmatched_files: true
|
||||
tag_name: v${{ github.event.inputs.version }}
|
||||
draft: true
|
||||
files: |
|
||||
flatmap.jar
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- run: ./scripts/push-release.sh ${{ github.event.inputs.version }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
IMAGE_TAGS: ${{ github.event.inputs.image_tags }}
|
|
@ -0,0 +1,42 @@
|
|||
# This workflow builds a map using the base and branch commit of a PR and uploads
|
||||
# the logs as an artifact that update-pr.yml uses to add back as a comment.
|
||||
|
||||
name: Publish a Snapshot
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image_tags:
|
||||
description: 'Extra docker image tags ("latest,test")'
|
||||
required: true
|
||||
default: 'latest,snapshot'
|
||||
|
||||
jobs:
|
||||
snapshot:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Cache data/sources
|
||||
uses: ./.github/cache-sources-action
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
- run: ./scripts/build-release.sh
|
||||
- run: ./scripts/test-release.sh
|
||||
- name: Log in to the Container Registry
|
||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- run: ./scripts/push-release.sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
IMAGE_TAGS: ${{ github.event.inputs.image_tags }}
|
|
@ -0,0 +1,89 @@
|
|||
# This workflow posts the result of a performance test back to the pull request as a comment.
|
||||
# Needs to be separate from CI because it has elevated privileges so only runs from main branch.
|
||||
|
||||
name: Update PR
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [ "Performance" ]
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
updatepr:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }}
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
# report status back to pull request
|
||||
- uses: haya14busa/action-workflow_run-status@v1
|
||||
- uses: actions/checkout@v2
|
||||
- name: 'Download branch build info'
|
||||
uses: dawidd6/action-download-artifact@v2
|
||||
with:
|
||||
workflow: ${{ github.event.workflow_run.workflow_id }}
|
||||
run_id: ${{ github.event.workflow_run.id }}
|
||||
name: build-info
|
||||
path: build-info
|
||||
repo: onthegomap/flatmap
|
||||
- name: 'Get build info'
|
||||
id: build_info
|
||||
run: echo "::set-output name=pr_number::$(cat build-info/pull_request_number)"
|
||||
- name: 'Build comment-body'
|
||||
run: |
|
||||
cat build-info/branchlogs.txt | sed -n '/^.*Tile stats/,$p' > branchsummary.txt
|
||||
cat build-info/branchlogs.txt | sed -n '/^.*Exception in thread/,$p' >> branchsummary.txt
|
||||
cat build-info/baselogs.txt | sed -n '/^.*Tile stats:/,$p' > basesummary.txt
|
||||
cat build-info/baselogs.txt | sed -n '/^.*Exception in thread/,$p' >> basesummary.txt
|
||||
|
||||
cat << EOF > comment-body.txt
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Base $(cat build-info/base_sha)</th>
|
||||
<th>This Branch $(cat build-info/branch_sha)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
\`\`\`
|
||||
$(cat basesummary.txt)
|
||||
\`\`\`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
\`\`\`
|
||||
$(cat build-info/branchlogs.txt | sed -n '/^.*Tile stats:/,$p')
|
||||
\`\`\`
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
https://github.com/onthegomap/flatmap/actions/runs/${{ github.event.workflow_run.id }}
|
||||
|
||||
<details><summary>ℹ️ <strong>Base Logs $(cat build-info/base_sha)</strong></summary>
|
||||
|
||||
\`\`\`
|
||||
$(cat build-info/baselogs.txt)
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
<details><summary>ℹ️ <strong>This Branch Logs $(cat build-info/branch_sha)</strong></summary>
|
||||
|
||||
\`\`\`
|
||||
$(cat build-info/branchlogs.txt)
|
||||
\`\`\`
|
||||
</details>
|
||||
EOF
|
||||
|
||||
- name: 'Dump comment body'
|
||||
run: cat comment-body.txt
|
||||
|
||||
- uses: marocchino/sticky-pull-request-comment@v2
|
||||
with:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
path: comment-body.txt
|
||||
header: performance-tests
|
||||
number: ${{ steps.build_info.outputs.pr_number }}
|
|
@ -2,8 +2,10 @@
|
|||
|
||||
target/
|
||||
*.jar
|
||||
!.mvn/wrapper/*.jar
|
||||
*.log
|
||||
|
||||
*/.idea
|
||||
.idea/*
|
||||
*.iml
|
||||
!.idea/codeStyles
|
||||
|
|
|
@ -105,6 +105,9 @@
|
|||
<Python>
|
||||
<option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true" />
|
||||
</Python>
|
||||
<ScalaCodeStyleSettings>
|
||||
<option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
|
||||
</ScalaCodeStyleSettings>
|
||||
<TypeScriptCodeStyleSettings>
|
||||
<option name="INDENT_CHAINED_CALLS" value="false" />
|
||||
</TypeScriptCodeStyleSettings>
|
||||
|
@ -204,6 +207,9 @@
|
|||
<option name="TAB_SIZE" value="2" />
|
||||
</indentOptions>
|
||||
</codeStyleSettings>
|
||||
<codeStyleSettings language="Markdown">
|
||||
<option name="RIGHT_MARGIN" value="120" />
|
||||
</codeStyleSettings>
|
||||
<codeStyleSettings language="ObjectiveC">
|
||||
<option name="RIGHT_MARGIN" value="80" />
|
||||
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
|
||||
|
|
|
@ -0,0 +1,117 @@
|
|||
/*
|
||||
* Copyright 2007-present the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import java.net.*;
|
||||
import java.io.*;
|
||||
import java.nio.channels.*;
|
||||
import java.util.Properties;
|
||||
|
||||
public class MavenWrapperDownloader {
|
||||
|
||||
private static final String WRAPPER_VERSION = "0.5.6";
|
||||
/**
|
||||
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
|
||||
*/
|
||||
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
|
||||
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
|
||||
|
||||
/**
|
||||
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
|
||||
* use instead of the default one.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
|
||||
".mvn/wrapper/maven-wrapper.properties";
|
||||
|
||||
/**
|
||||
* Path where the maven-wrapper.jar will be saved to.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_JAR_PATH =
|
||||
".mvn/wrapper/maven-wrapper.jar";
|
||||
|
||||
/**
|
||||
* Name of the property which should be used to override the default download url for the wrapper.
|
||||
*/
|
||||
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
|
||||
|
||||
public static void main(String args[]) {
|
||||
System.out.println("- Downloader started");
|
||||
File baseDirectory = new File(args[0]);
|
||||
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
|
||||
|
||||
// If the maven-wrapper.properties exists, read it and check if it contains a custom
|
||||
// wrapperUrl parameter.
|
||||
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
|
||||
String url = DEFAULT_DOWNLOAD_URL;
|
||||
if(mavenWrapperPropertyFile.exists()) {
|
||||
FileInputStream mavenWrapperPropertyFileInputStream = null;
|
||||
try {
|
||||
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
|
||||
Properties mavenWrapperProperties = new Properties();
|
||||
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
|
||||
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
|
||||
} catch (IOException e) {
|
||||
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
|
||||
} finally {
|
||||
try {
|
||||
if(mavenWrapperPropertyFileInputStream != null) {
|
||||
mavenWrapperPropertyFileInputStream.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Ignore ...
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading from: " + url);
|
||||
|
||||
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
|
||||
if(!outputFile.getParentFile().exists()) {
|
||||
if(!outputFile.getParentFile().mkdirs()) {
|
||||
System.out.println(
|
||||
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
|
||||
try {
|
||||
downloadFileFromURL(url, outputFile);
|
||||
System.out.println("Done");
|
||||
System.exit(0);
|
||||
} catch (Throwable e) {
|
||||
System.out.println("- Error downloading");
|
||||
e.printStackTrace();
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
|
||||
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
|
||||
String username = System.getenv("MVNW_USERNAME");
|
||||
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
|
||||
Authenticator.setDefault(new Authenticator() {
|
||||
@Override
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(username, password);
|
||||
}
|
||||
});
|
||||
}
|
||||
URL website = new URL(urlString);
|
||||
ReadableByteChannel rbc;
|
||||
rbc = Channels.newChannel(website.openStream());
|
||||
FileOutputStream fos = new FileOutputStream(destination);
|
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
||||
fos.close();
|
||||
rbc.close();
|
||||
}
|
||||
|
||||
}
|
Plik binarny nie jest wyświetlany.
|
@ -0,0 +1,2 @@
|
|||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.2/apache-maven-3.8.2-bin.zip
|
||||
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
|
|
@ -0,0 +1,119 @@
|
|||
# Flatmap Architecture
|
||||
|
||||
![Architecture Diagram](diagrams/architecture.png)
|
||||
|
||||
Flatmap builds a map in 3 phases:
|
||||
|
||||
1. [Process Input Files](#process-input-files) according to
|
||||
the [Profile](flatmap-core/src/main/java/com/onthegomap/flatmap/Profile.java) and write rendered tile features to
|
||||
intermediate files on disk
|
||||
2. [Sort Features](#sort-features) by tile ID
|
||||
3. [Emit Vector Tiles](#emit-vector-tiles) by iterating through sorted features to group by tile ID, encoding, and
|
||||
writing to the output MBTiles file
|
||||
|
||||
## 1) Process Input Files
|
||||
|
||||
First, Flatmap reads [SourceFeatures](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/SourceFeature.java) from
|
||||
each input source:
|
||||
|
||||
- For "simple
|
||||
sources" [NaturalEarthReader](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java)
|
||||
or [ShapefileReader](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java) can get the
|
||||
latitude/longitude geometry directly from each feature
|
||||
- For OpenStreetMap `.osm.pbf` files,
|
||||
[OsmReader](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmReader.java)
|
||||
needs to make 2 passes through the input file to construct feature geometries:
|
||||
- pass 1:
|
||||
- nodes: store node latitude/longitude locations in-memory or on disk
|
||||
using [LongLongMap](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/LongLongMap.java)
|
||||
- ways: nothing
|
||||
- relations: call `preprocessOsmRelation` on the profile and store information returned for each relation of
|
||||
interest, along with relation member IDs in-memory using
|
||||
a [LongLongMultimap](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/LongLongMultimap.java).
|
||||
- pass 2:
|
||||
- nodes: emit a point source feature
|
||||
- ways:
|
||||
- lookup the latitude/longitude for each node ID to get the way geometry and relations that the way is contained
|
||||
in
|
||||
- emit a source feature with the reconstructed geometry which can either be a line or polygon, depending on
|
||||
the `area` tag and whether the way is closed
|
||||
- if this way is part of a multipolygon, also save the way geometry in-memory for later in
|
||||
a [LongLongMultimap](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/LongLongMultimap.java)
|
||||
- relations: for any multipolygon relation, fetch the member geometries and attempt to reconstruct the multipolygon
|
||||
geometry
|
||||
using [OsmMultipolygon](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmMultipolygon.java), then
|
||||
emit a polygon source feature with the reconstructed geometry if successfule
|
||||
|
||||
Then, for each [SourceFeature](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/SourceFeature.java), render
|
||||
vector tile features according to the [Profile](flatmap-core/src/main/java/com/onthegomap/flatmap/Profile.java) in a
|
||||
worker thread (default 1 per core):
|
||||
|
||||
- Call `processFeature` method on the profile for each source feature
|
||||
- For every vector tile feature added to
|
||||
the [FeatureCollector](flatmap-core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java):
|
||||
- Call [FeatureRenderer#accept](flatmap-core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java)
|
||||
which for each zoom level the feature appears in:
|
||||
- Scale the geometry to that zoom level
|
||||
- Simplify it in screen pixel coordinates
|
||||
- Use [TiledGeometry](flatmap-core/src/main/java/com/onthegomap/flatmap/render/TiledGeometry.java)
|
||||
to slice the geometry into subcomponents that appear in every tile it touches using the stripe clipping algorithm
|
||||
derived from [geojson-vt](https://github.com/mapbox/geojson-vt):
|
||||
- `sliceX` splits the geometry into vertical slices for each "column" representing the X coordinate of a vector
|
||||
tile
|
||||
- `sliceY` splits each "column" into "rows" representing the Y coordinate of a vector tile
|
||||
- Uses an [IntRangeSet](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/IntRangeSet.java) to
|
||||
optimize processing for large filled areas (like oceans)
|
||||
- If any features wrapped past -180 or 180 degrees, then repeat with a 360 or -360 degree offset
|
||||
- Reassemble each vector tile geometry and round to tile precision (4096x4096)
|
||||
- For polygons, [GeoUtils#snapAndFixPolygon](flatmap-core/src/main/java/com/onthegomap/flatmap/geo/GeoUtils.java)
|
||||
uses [JTS](https://github.com/locationtech/jts) utilities to fix any topology errors (i.e. self-intersections)
|
||||
introduced by rounding. This is very expensive, but necessary since clients
|
||||
like [MapLibre GL JS](https://github.com/maplibre/maplibre-gl-js) produce rendering artifacts for invalid
|
||||
polygons.
|
||||
- Encode the feature into compact binary format
|
||||
using [FeatureGroup#newRenderedFeatureEncoder](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/FeatureGroup.java)
|
||||
consisting of a sortable 64-bit `long` key (zoom, x, y, layer, sort order) and a binary value encoded
|
||||
using [MessagePack](https://msgpack.org/) (feature group/limit, feature ID, geometry type, tags, geometry)
|
||||
- Add the encoded feature to a [WorkQueue](flatmap-core/src/main/java/com/onthegomap/flatmap/worker/WorkQueue.java)
|
||||
|
||||
Finally, a single-threaded writer reads encoded features off of the work queue and writes them to disk
|
||||
using [ExternalMergeSort#add](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java)
|
||||
|
||||
- Write features to a "chunk" file until that file hits a size limit (i.e. 1GB) then start writing to a new file
|
||||
|
||||
## 2) Sort Features
|
||||
|
||||
[ExternalMergeSort](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java) sorts all of
|
||||
the intermediate features using a worker thread per core:
|
||||
|
||||
- Read each "chunk" file into memory
|
||||
- Sort the features it contains by 64-bit `long` key
|
||||
- Write the chunk back to disk
|
||||
|
||||
## 3) Emit Vector Tiles
|
||||
|
||||
[MbtilesWriter](flatmap-core/src/main/java/com/onthegomap/flatmap/mbtiles/MbtilesWriter.java) is the main driver. First,
|
||||
a single-threaded reader reads features from disk:
|
||||
|
||||
- [ExternalMergeSort](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java) emits sorted
|
||||
features by doing a k-way merge using a priority queue of iterators reading from each sorted chunk
|
||||
- [FeatureGroup](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/FeatureGroup.java) collects consecutive
|
||||
features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit
|
||||
to limit point label density
|
||||
- Then [MbtilesWriter](flatmap-core/src/main/java/com/onthegomap/flatmap/mbtiles/MbtilesWriter.java) groups tiles into
|
||||
variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the
|
||||
writer thread waits for finished tiles in order)
|
||||
|
||||
Then, process tile batches in worker threads (default 1 per core):
|
||||
|
||||
- For each tile in the batch, first check to see if it has the same contents as the previous tile to avoid re-encoding
|
||||
the same thing for large filled areas (i.e. oceans)
|
||||
- Encode using [VectorTile](flatmap-core/src/main/java/com/onthegomap/flatmap/VectorTile.java)
|
||||
- gzip each encoded tile
|
||||
- Pass the batch of encoded vector tiles to the writer thread
|
||||
|
||||
Finally, a single-threaded writer writes encoded vector tiles to the output MBTiles file:
|
||||
|
||||
- Create the largest prepared statement supported by SQLite (999 parameters)
|
||||
- Iterate through finished vector tile batches until the prepared statement is full, flush to disk, then repeat
|
||||
- Then flush any remaining tiles at the end
|
|
@ -0,0 +1,35 @@
|
|||
# Contributing to Flatmap
|
||||
|
||||
Pull requests are welcome! To set up your development environment:
|
||||
|
||||
- Fork the repo
|
||||
- [Install Java 16 or later](https://adoptium.net/installation.html)
|
||||
- Build and run the tests ([mvnw](https://github.com/takari/maven-wrapper) automatically downloads maven the first time
|
||||
you run it):
|
||||
- on max/linux: `./mvnw clean test`
|
||||
- on windows: `mvnw.cmd clean test`
|
||||
- or if you already have maven installed globally on your machine: `mvn clean test`
|
||||
|
||||
To edit the code:
|
||||
|
||||
- [Install IntelliJ IDE](https://www.jetbrains.com/help/idea/installation-guide.html)
|
||||
- In IntelliJ, click `Open`, navigate to the the `pom.xml` file in the local copy of this repo, and `Open`
|
||||
then `Open as Project`
|
||||
- If IntelliJ asks (and you trust the code) then click `Trust Project`
|
||||
- If any java source files show "Cannot resolve symbol..." errors for Flatmap classes, you might need to
|
||||
select: `File -> Invalidate Caches... -> Just Restart`.
|
||||
- If you see a "Project JDK is not defined" error, then choose `Setup SDK` and point IntelliJ at the Java 16 or later
|
||||
installed on your system
|
||||
- To verify everything works correctly, right click on `flatmap-core/src/test/java` folder and click `Run 'All Tests'`
|
||||
|
||||
Any pull request should:
|
||||
|
||||
- Include at least one unit test to verify the change in behavior
|
||||
- Include an end-to-end test in [FlatmapTests.java](flatmap-core/src/test/java/com/onthegomap/flatmap/FlatmapTests.java)
|
||||
to verify any major new user-facing features work
|
||||
- Use IntelliJ's auto-formatting for modified files (this should get enabled automatically)
|
||||
- Be free of IntelliJ warnings for modified files
|
||||
|
||||
GitHub Workflows will run regression tests on any pull request.
|
||||
|
||||
TODO: Set up checkstyle and an auto-formatter to enforce standards, so you can use any IDE.
|
10
NOTICE.md
10
NOTICE.md
|
@ -25,12 +25,10 @@ The `flatmap-core` module includes the following software:
|
|||
- `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)
|
||||
- `TiledGeometry` from [geojson-vt](https://github.com/mapbox/geojson-vt) (ISC license)
|
||||
- `VectorTileEncoder`
|
||||
from [java-vector-tile](https://github.com/ElectronicChartCentre/java-vector-tile) (Apache
|
||||
license)
|
||||
from [java-vector-tile](https://github.com/ElectronicChartCentre/java-vector-tile) (Apache license)
|
||||
- `Imposm3Parsers` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)
|
||||
|
||||
Additionally, the `flatmap-openmaptiles` module is a Java port
|
||||
of [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
|
||||
Additionally, the `flatmap-basemap` module is based on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
|
||||
|
||||
- Maven Dependencies:
|
||||
- org.yaml:snakeyaml (Apache license)
|
||||
|
@ -41,8 +39,8 @@ of [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
|
|||
- `LanguageUtils` ported from OpenMapTiles
|
||||
- Schema
|
||||
- The cartography and visual design features of the map tile schema are licensed
|
||||
under [CC-BY 4.0](http://creativecommons.org/licenses/by/4.0/). Products or services using maps
|
||||
derived from OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference
|
||||
under [CC-BY 4.0](http://creativecommons.org/licenses/by/4.0/). Products or services using maps derived from
|
||||
OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference
|
||||
"OpenMapTiles" with a link to http://openmaptiles.org/. More
|
||||
details [here](https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md#design-license-cc-by-40)
|
||||
|
||||
|
|
|
@ -0,0 +1,142 @@
|
|||
# Generating a Map of the World
|
||||
|
||||
To generate a map of the world using the built-in [basemap profile](flatmap-basemap) based
|
||||
on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles), you will need a machine with java 16 or later installed
|
||||
and at least 10x as much disk space and 1.5x as much RAM as the `planet.osm.pbf` file you start from. All testing has
|
||||
been done using Digital Ocean droplets with dedicated vCPUs ([referral link](https://m.do.co/c/a947e99aab25)) and
|
||||
OpenJDK installed through `apt`. Flatmap splits work among cores so the more you have, the less time it takes.
|
||||
|
||||
### 1) Choose the Data Source
|
||||
|
||||
First decide where to get the `planet.osm.pbf` file:
|
||||
|
||||
- One of the [official mirrors](https://wiki.openstreetmap.org/wiki/Planet.osm)
|
||||
- The [AWS Registry of Open Data](https://registry.opendata.aws/osm/) public S3 mirror (default)
|
||||
- Or a [Daylight Distribution](https://daylightmap.org/) snapshot from Facebook that lags a bit, but includes extra
|
||||
quality/consistency checks, and add-ons like ML-detected roads and buildings. NOTE: you need at least
|
||||
`admin.osc.bz2` then combine and re-number using [osmium-tool](https://osmcode.org/osmium-tool/):
|
||||
```bash
|
||||
osmium apply-changes daylight.osm.pbf admin.osc.bz2 <buildings.osc.bz2, ...> -o everything.osm.pbf
|
||||
osmium renumber everything.osm.pbf -o planet.osm.pbf
|
||||
```
|
||||
This takes about 2.5 hours and needs as much RAM as the `planet.osm.pbf` size.
|
||||
|
||||
### 2) Run Flatmap
|
||||
|
||||
Download the [latest release](https://github.com/onthegomap/flatmap/releases/latest) of `flatmap.jar`.
|
||||
|
||||
Then run `java -Xms100g -Xmx100g -jar flatmap.jar` (replacing `100g` with 1.5x the `planet.osm.pbf` size)
|
||||
with these options:
|
||||
|
||||
- `--bounds=world` to set bounding box to the entire planet
|
||||
- `--nodemap-type=sparsearray` to store node locations in a sparse array instead of a sorted table (sorted table is only
|
||||
more efficient for extracts)
|
||||
- `--nodemap-storage=ram` to store all node locations in RAM instead of a memory-mapped file (when using `ram` give the
|
||||
JVM 1.5x the input file size instead of 0.5x when using `mmap`)
|
||||
- `--download` to fetch [other data sources](NOTICE.md#data) automatically
|
||||
- One of these to point flatmap at your data source:
|
||||
- `--osm-path=path/to/planet.osm.pbf` to point Flatmap at a file you downloaded
|
||||
- `--osm-url=http://url/of/planet.osm.pbf` to download automatically
|
||||
- `--area=planet` to use the file in `./data/sources/planet.osm.pbf` or download the latest snapshot from AWS S3
|
||||
mirror if missing.
|
||||
|
||||
Run with `--help` to see all available arguments.
|
||||
|
||||
## Example
|
||||
|
||||
To generate the tiles shown on https://onthegomap.github.io/flatmap-demo/ I used the `planet-211011.osm.pbf` (64.7GB) S3
|
||||
snapshot, then ran Flatmap on a Digital Ocean Memory-Optimized droplet with 16 CPUs, 128GB RAM, and 1.17TB disk running
|
||||
Ubuntu 21.04 x64 in the nyc3 location.
|
||||
|
||||
First, I installed java 17 jre and screen:
|
||||
|
||||
```bash
|
||||
apt-get update && apt-get install -y openjdk-17-jre-headless screen
|
||||
```
|
||||
|
||||
Then I added a script `runworld.sh` to run with 100GB of RAM:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
java -Xmx100g -Xms100g \
|
||||
-XX:OnOutOfMemoryError="kill -9 %p" \
|
||||
-jar flatmap.jar \
|
||||
`# Download the latest planet.osm.pbf from s3://osm-pds bucket` \
|
||||
--area=planet --bounds=world --download \
|
||||
`# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
|
||||
--download-threads=10 --download-chunk-size-mb=1000 \
|
||||
`# Also download name translations from wikidata` \
|
||||
--fetch-wikidata \
|
||||
`# Personal preference overrides to default OpenMapTiles schema` \
|
||||
--transportation-name-brunnel=false --transportation-z13-paths=true \
|
||||
--mbtiles=output.mbtiles \
|
||||
--nodemap-type=sparsearray --nodemap-storage=ram 2>&1 | tee logs.txt
|
||||
```
|
||||
|
||||
Then I ran this in the background using screen, so it would continue if my shell exited:
|
||||
|
||||
```bash
|
||||
screen -d -m "./runworld.sh"
|
||||
tail -f logs.txt
|
||||
```
|
||||
|
||||
It took 3h21m (including 12 minutes downloading source data) to generate a 99GB `output.mbtiles` file. See
|
||||
the [full logs](planet-logs/planet-logs.txt) from this run or this summary that it printed at the end. Notice that it
|
||||
spent almost an hour emitting z13 tiles. That is because the default basemap profile merges nearby building polygons at
|
||||
z13 which is very expensive. You can disable this behavior by setting `--building-merge-z13=false`.
|
||||
|
||||
```
|
||||
3:21:03 DEB [mbtiles] - Tile stats:
|
||||
3:21:03 DEB [mbtiles] - z0 avg:71k max:71k
|
||||
3:21:03 DEB [mbtiles] - z1 avg:171k max:192k
|
||||
3:21:03 DEB [mbtiles] - z2 avg:258k max:449k
|
||||
3:21:03 DEB [mbtiles] - z3 avg:117k max:479k
|
||||
3:21:03 DEB [mbtiles] - z4 avg:51k max:541k
|
||||
3:21:03 DEB [mbtiles] - z5 avg:23k max:537k
|
||||
3:21:03 DEB [mbtiles] - z6 avg:14k max:354k
|
||||
3:21:03 DEB [mbtiles] - z7 avg:11k max:451k
|
||||
3:21:03 DEB [mbtiles] - z8 avg:6.5k max:356k
|
||||
3:21:03 DEB [mbtiles] - z9 avg:6k max:485k
|
||||
3:21:03 DEB [mbtiles] - z10 avg:2.7k max:285k
|
||||
3:21:03 DEB [mbtiles] - z11 avg:1.3k max:168k
|
||||
3:21:03 DEB [mbtiles] - z12 avg:741 max:247k
|
||||
3:21:03 DEB [mbtiles] - z13 avg:388 max:286k
|
||||
3:21:03 DEB [mbtiles] - z14 avg:340 max:1.7M
|
||||
3:21:03 DEB [mbtiles] - all avg:395 max:0
|
||||
3:21:03 DEB [mbtiles] - # features: 2,832,396,934
|
||||
3:21:03 DEB [mbtiles] - # tiles: 264,204,266
|
||||
3:21:03 INF [mbtiles] - Finished in 4,668s cpu:66,977s avg:14.3
|
||||
|
||||
3:21:03 INF - Finished in 12,064s cpu:156,169s avg:12.9
|
||||
|
||||
3:21:03 INF - FINISHED!
|
||||
3:21:03 INF - ----------------------------------------
|
||||
3:21:03 INF - overall 12,064s cpu:156,169s avg:12.9
|
||||
3:21:03 INF - download 169s cpu:1,070s avg:6.3
|
||||
3:21:03 INF - wikidata 553s cpu:3,825s avg:6.9
|
||||
3:21:03 INF - lake_centerlines 0.9s cpu:2s avg:1.8
|
||||
3:21:03 INF - water_polygons 96s cpu:1,150s avg:12
|
||||
3:21:03 INF - natural_earth 6s cpu:21s avg:3.7
|
||||
3:21:03 INF - osm_pass1 921s cpu:5,177s avg:5.6
|
||||
3:21:03 INF - osm_pass2 5,234s cpu:73,527s avg:14
|
||||
3:21:03 INF - boundaries 14s cpu:18s avg:1.3
|
||||
3:21:03 INF - sort 407s cpu:4,403s avg:10.8
|
||||
3:21:03 INF - mbtiles 4,668s cpu:66,977s avg:14.3
|
||||
3:21:03 INF - ----------------------------------------
|
||||
3:21:03 INF - features 192GB
|
||||
3:21:03 INF - mbtiles 99GB
|
||||
```
|
||||
|
||||
Then to generate the extract for [the demo](https://onthegomap.github.io/flatmap-demo/) I ran:
|
||||
|
||||
```bash
|
||||
# install node and tilelive-copy
|
||||
curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
|
||||
apt-get install -y nodejs
|
||||
npm install -g @mapbox/tilelive @mapbox/mbtiles
|
||||
# Extract z0-4 for the world
|
||||
tilelive-copy --minzoom=0 --maxzoom=4 --bounds=-180,-90,180,90 output.mbtiles demo.mbtiles
|
||||
# Extract z0-14 for just southern New England
|
||||
tilelive-copy --minzoom=0 --maxzoom=14 --bounds=-73.6346,41.1055,-69.5464,42.9439 output.mbtiles demo.mbtiles
|
||||
```
|
|
@ -0,0 +1,226 @@
|
|||
# Flatmap
|
||||
|
||||
Flatmap is a tool that generates [Mapbox Vector Tiles](https://github.com/mapbox/vector-tile-spec/tree/master/2.1) from
|
||||
geographic data sources like [OpenStreetMap](https://www.openstreetmap.org/). Flatmap aims to be fast and
|
||||
memory-efficient so that you can build a map of the world in a few hours on a single machine without any external tools
|
||||
or database.
|
||||
|
||||
Vector tiles contain raw point, line, and polygon geometries that clients like [MapLibre](https://github.com/maplibre)
|
||||
can use to render custom maps in the browser, native apps, or on a server. Flatmap packages tiles into
|
||||
an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite) file that can be served using
|
||||
tools like [TileServer GL](https://github.com/maptiler/tileserver-gl) or even
|
||||
[queried directly from the browser](https://github.com/phiresky/sql.js-httpvfs).
|
||||
See [awesome-vector-tiles](https://github.com/mapbox/awesome-vector-tiles) for more projects that work with data in this
|
||||
format.
|
||||
|
||||
Flatmap is named after the "flatmap" operation that it performs: *map* input elements to rendered tile features,
|
||||
*flatten* them into a big list, and sort by tile ID to group into tiles. The output is also a "flat map" where zoom
|
||||
level 0 contains the entire planet in a 256x256 px tile, and each zoom level splits parent tiles into 4 quadrants,
|
||||
revealing more detail.
|
||||
|
||||
See [ARCHITECTURE.md](ARCHITECTURE.md) for more details on how Flatmap works.
|
||||
|
||||
## Demo
|
||||
|
||||
See the [live demo](https://onthegomap.github.io/flatmap-demo/) of vector tiles created by Flatmap.
|
||||
|
||||
[![Flatmap Demo Screenshot](./diagrams/demo.png)](https://onthegomap.github.io/flatmap-demo/)
|
||||
Style [© OpenMapTiles](https://www.openmaptiles.org/)
|
||||
· Data [© OpenStreetMap contributors](https://www.openstreetmap.org/copyright)
|
||||
|
||||
## Usage
|
||||
|
||||
To generate a map of an area using the [basemap profile](flatmap-basemap), you will need:
|
||||
|
||||
- [Java 16+](https://adoptium.net/installation.html) or [Docker](https://docs.docker.com/get-docker/)
|
||||
- at least 1GB of free disk space plus 5-10x the size of the `.osm.pbf` file
|
||||
- at least 1.5x as much free RAM as the input `.osm.pbf` file size
|
||||
|
||||
#### To build the map:
|
||||
|
||||
Using Java, download `flatmap.jar` from the [latest release](https://github.com/onthegomap/flatmap/releases/latest)
|
||||
and run it:
|
||||
|
||||
```bash
|
||||
wget https://github.com/onthegomap/flatmap/releases/latest/download/flatmap.jar
|
||||
java -Xmx1g -jar flatmap.jar --download --area=monaco
|
||||
```
|
||||
|
||||
Or using Docker:
|
||||
|
||||
```bash
|
||||
docker run -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/flatmap:latest --download --area=monaco
|
||||
```
|
||||
|
||||
#### To view tiles locally:
|
||||
|
||||
Using [Node.js](https://nodejs.org/en/download/):
|
||||
|
||||
```bash
|
||||
npm install -g tileserver-gl-light
|
||||
tileserver-gl-light --mbtiles data/output.mbtiles
|
||||
```
|
||||
|
||||
Or using [Docker](https://docs.docker.com/get-docker/):
|
||||
|
||||
```bash
|
||||
docker run --rm -it -v "$(pwd)/data":/data -p 8080:8080 maptiler/tileserver-gl -p 8080
|
||||
```
|
||||
|
||||
Then open http://localhost:8080 to view tiles.
|
||||
|
||||
Some common arguments:
|
||||
|
||||
- `--download` downloads input sources automatically and `--only-download` exits after downloading
|
||||
- `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/)
|
||||
- `--osm-path=path/to/file.osm.pbf` points Flatmap at an existing OSM extract on disk
|
||||
- `-Xmx1g` controls how much RAM to give the JVM (recommended: 0.5x the input .osm.pbf file size to leave room for
|
||||
memory-mapped files)
|
||||
- `--force` overwrites the output file
|
||||
- `--help` shows all of the options and exits
|
||||
|
||||
## Generating a Map of the World
|
||||
|
||||
See [PLANET.md](PLANET.md).
|
||||
|
||||
## Creating a Custom Map
|
||||
|
||||
See the [flatmap-examples](flatmap-examples) project.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Some example runtimes (excluding downloading resources):
|
||||
|
||||
| Input | Input Size | Profile | Machine | Time | mbtiles size | Logs |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap | DO 16cpu 128GB RAM | 3h9m (cpu: 42h1m @ 13.3) | 99.6GB | [logs](planet-logs/planet-logs.txt) [VisualVM Profile](planet-logs/planet.nps) |
|
||||
| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries | 68.6GB | Basemap | DO 16cpu 128GB RAM | 3h13m (cpu: 43h40m @ 13.5) | 101.4GB | [logs](planet-logs/logs-daylight.txt) |
|
||||
| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB RAM) | 59m26s (cpu: 27h6m @ 27.4) | 97.3GB | [logs](planet-logs/planet-logs-c5ad.txt) |
|
||||
|
||||
## Alternatives
|
||||
|
||||
Some other tools that generate vector tiles from OpenStreetMap data:
|
||||
|
||||
- [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) is the reference implementation of
|
||||
the [OpenMapTiles schema](https://openmaptiles.org/schema/) that the [basemap profile](flatmap-basemap) is based on.
|
||||
It uses an intermediate postgres database and operates in two modes:
|
||||
1. Import data into database (~1 day) then serve vector tiles directly from the database. Tile serving is slower and
|
||||
requires bigger machines, but lets you easily incorporate realtime updates
|
||||
2. Import data into database (~1 day) then prerender every tile for the planet into an mbtiles file which
|
||||
takes [over 100 days](https://github.com/openmaptiles/openmaptiles/issues/654#issuecomment-724606293)
|
||||
or a cluster of machines, but then tiles can be served faster on smaller machines
|
||||
- [Tilemaker](https://github.com/systemed/tilemaker) uses a similar approach to Flatmap (no intermediate database), is
|
||||
more mature, and has a convenient lua API for building custom profiles without recompiling the tool, but takes
|
||||
[about a day](https://github.com/systemed/tilemaker/issues/315#issue-994322040) to generate a map of the world
|
||||
|
||||
Some companies that generate and host tiles for you:
|
||||
|
||||
- [Mapbox](https://www.mapbox.com/) - data from the pioneer of vector tile technologies
|
||||
- [Maptiler](https://www.maptiler.com/) - data from the creator of OpenMapTiles schema
|
||||
- [Stadia Maps](https://stadiamaps.com/) - what [onthegomap.com](https://onthegomap.com/) uses in production
|
||||
|
||||
If you want to host tiles yourself but have someone else generate them for you, those companies also offer plans to
|
||||
download regularly-updated tilesets.
|
||||
|
||||
## Features
|
||||
|
||||
- Supports [Natural Earth](https://www.naturalearthdata.com/),
|
||||
OpenStreetMap [.osm.pbf](https://wiki.openstreetmap.org/wiki/PBF_Format),
|
||||
and [Esri Shapefiles](https://en.wikipedia.org/wiki/Shapefile) data sources
|
||||
- Java-based [Profile API](flatmap-core/src/main/java/com/onthegomap/flatmap/Profile.java) to customize how source
|
||||
elements map to vector tile features, and post-process generated tiles
|
||||
using [JTS geometry utilities](https://github.com/locationtech/jts)
|
||||
- Automatically fixes self-intersecting polygons
|
||||
- Built-in basemap profile based on [OpenMapTiles](https://openmaptiles.org/) v3.12.2
|
||||
- Optionally download additional name translations for elements from Wikidata
|
||||
- Export real-time stats to a [prometheus push gateway](https://github.com/prometheus/pushgateway) using
|
||||
`--pushgateway=http://user:password@ip` argument (and a [grafana dashboard](grafana.json) for viewing)
|
||||
- Automatically downloads region extracts from [Geofabrik](https://download.geofabrik.de/)
|
||||
using `geofabrik:australia` shortcut as a source URL
|
||||
- Unit-test profiles to verify mapping logic, or integration-test to verify the actual contents of a generated mbtiles
|
||||
file ([example](flatmap-examples/src/test/java/com/onthegomap/flatmap/examples/BikeRouteOverlayTest.java))
|
||||
|
||||
## Limitations
|
||||
|
||||
- It is harder to join and group data than when using database. To join input data sources, profiles must explicitly
|
||||
store data when processing a feature to use with later features, or apply post-processing to rendered features
|
||||
immediately before emitting the vector tile.
|
||||
- Flatmap only does full imports from `.osm.pbf` snapshots, there is no way to incorporate real-time updates.
|
||||
|
||||
## Roadmap
|
||||
|
||||
- [x] Enough `flatmap-core` functionality to support basemap profile based on OpenMapTiles
|
||||
- [ ] Basemap profile based on OpenMapTiles v3.12.2
|
||||
- [x] Port all layers
|
||||
- [x] Download name translations from wikidata
|
||||
- [x] Merge buildings at z13
|
||||
- [x] `adm0_l`/`adm0_r` boundary labels
|
||||
- [ ] Abbreviate road names to improve visibility
|
||||
- [ ] Poi layer `agg_stop` tag
|
||||
- [ ] Get `flatmap-core` into Maven Central
|
||||
- [ ] Remove geotools dependency for reading shapefiles (not in Maven Central)
|
||||
- [ ] Remove graphhopper dependency for reading OSM files
|
||||
- [ ] "Sparse mode" to only store node and relation data for elements used by a profile
|
||||
- [ ] Support zoom levels higher than 14
|
||||
- [ ] Handle nodes and relations in relations (only ways handled now)
|
||||
- [ ] Lake centerline support in `flatmap-core`
|
||||
- [ ] Improve line merging to combine nearby parallel roads
|
||||
- [ ] Basemap schema improvements for [onthegomap.com](https://onthegomap.com)
|
||||
- [ ] Accept other kinds of data sources
|
||||
- [ ] Extract reusable utilities for complex schemas from `flatmap-basemap` to `flatmap-core`
|
||||
- [ ] Other schemas
|
||||
|
||||
## Contributing
|
||||
|
||||
Pull requests are welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
|
||||
|
||||
## Support
|
||||
|
||||
Have a question or want to share something you've built? Start
|
||||
a [GitHub discussion](https://github.com/onthegomap/flatmap/discussions).
|
||||
|
||||
Found a bug or have a feature request? Open a [GitHub issue](https://github.com/onthegomap/flatmap/issues) to report.
|
||||
|
||||
This is a side project, so support is limited. If you have the time and ability, feel free to open a pull request to fix
|
||||
issues or implement new features.
|
||||
|
||||
## Acknowledgement
|
||||
|
||||
Flatmap is made possible by these awesome open source projects:
|
||||
|
||||
- [OpenMapTiles](https://openmaptiles.org/) for the [schema](https://openmaptiles.org/schema/)
|
||||
and [reference implementation](https://github.com/openmaptiles/openmaptiles)
|
||||
that the [basemap profile](flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap/layers)
|
||||
is based on
|
||||
- [Graphhopper](https://www.graphhopper.com/) for utilities to process OpenStreetMap data in Java
|
||||
- [JTS Topology Suite](https://github.com/locationtech/jts) for working with vector geometries
|
||||
- [Geotools](https://github.com/geotools/geotools) for shapefile processing
|
||||
- [SQLite JDBC Driver](https://github.com/xerial/sqlite-jdbc) for reading Natural Earth data and writing MBTiles files
|
||||
- [MessagePack](https://msgpack.org/) for compact binary encoding of intermediate map features
|
||||
- [geojson-vt](https://github.com/mapbox/geojson-vt) for the basis of
|
||||
the [stripe clipping algorithm](flatmap-core/src/main/java/com/onthegomap/flatmap/render/TiledGeometry.java)
|
||||
that flatmap uses to slice geometries into tiles
|
||||
- [java-vector-tile](https://github.com/ElectronicChartCentre/java-vector-tile) for the basis of
|
||||
the [vector tile encoder](flatmap-core/src/main/java/com/onthegomap/flatmap/VectorTile.java)
|
||||
- [imposm3](https://github.com/omniscale/imposm3) for the basis
|
||||
of [OSM multipolygon processing](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmMultipolygon.java)
|
||||
and [tag parsing utilities](flatmap-core/src/main/java/com/onthegomap/flatmap/util/Imposm3Parsers.java)
|
||||
|
||||
See [NOTICE.md](NOTICE.md) for a full list and license details.
|
||||
|
||||
## Author
|
||||
|
||||
Flatmap was created by [Michael Barry](https://github.com/msbarry) for future use generating custom basemaps or overlays
|
||||
for [On The Go Map](https://onthegomap.com).
|
||||
|
||||
## License and Attribution
|
||||
|
||||
Flatmap source code is licensed under the [Apache 2.0 License](LICENSE), so it can be used and modified in commercial or
|
||||
other open source projects according to the license guidelines.
|
||||
|
||||
Maps built using flatmap do not require any special attribution, but the data or schema used might. Any maps generated
|
||||
from OpenStreetMap data must [visibly credit OpenStreetMap contributors](https://www.openstreetmap.org/copyright). Any
|
||||
map generated with the profile based on OpenMapTiles or a derivative
|
||||
must [visibly credit OpenMapTiles](https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md#design-license-cc-by-40)
|
||||
as well.
|
||||
|
|
@ -11,8 +11,11 @@
|
|||
# download=true
|
||||
# only_download=true
|
||||
|
||||
# To override the user-agent header when downloading resources:
|
||||
# To override request parameters when downloading resources:
|
||||
# http_user_agent=Flatmap downloader (https://github.com/onthegomap/flatmap)
|
||||
# http_timeout=10s
|
||||
# download_threads=10
|
||||
# download_chunk_size=1000
|
||||
|
||||
# Map bounds are inferred from OSM input file bounds, but to override use:
|
||||
# bounds=W,S,E,N
|
||||
|
@ -28,6 +31,12 @@
|
|||
# Change how often to log progress messages:
|
||||
# loginterval=1m
|
||||
|
||||
# Send stats to a prometheus push gateway:
|
||||
# pushgateway=https://user:password@ip
|
||||
# "job" tag to include in every push to prometheus (to separate multiple flatmap instances):
|
||||
# pushgateway.job=flatmap
|
||||
# pushgateway.interval=15s
|
||||
|
||||
# Override the output location for mbtiles file:
|
||||
# mbtiles=data/output.mbtiles
|
||||
# Set the location where temporary files are stored (node map or intermediate features)
|
||||
|
@ -71,7 +80,7 @@
|
|||
# mbtiles_version=0.1
|
||||
# mbtiles_type=baselayer OR overlay
|
||||
|
||||
################# OpenMapTiles profile config #################
|
||||
################# Basemap profile config #################
|
||||
|
||||
# Set the input file name and automatically determine the https://download.geofabrik.de/ download URL for a region by name:
|
||||
# area=monaco
|
||||
|
|
Plik binarny nie jest wyświetlany.
Po Szerokość: | Wysokość: | Rozmiar: 238 KiB |
|
@ -0,0 +1,107 @@
|
|||
@startuml
|
||||
!includeurl https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/ce5a3ea16aee1b07487f9960633057d102b626a6/C4_Container.puml
|
||||
skinparam WrapWidth 80
|
||||
skinparam ranksep 10
|
||||
skinparam dpi 200
|
||||
|
||||
LAYOUT_LANDSCAPE()
|
||||
HIDE_STEREOTYPE()
|
||||
|
||||
skinparam rectangle {
|
||||
BorderColor<<diagram>> transparent
|
||||
BackgroundColor<<diagram>> transparent
|
||||
}
|
||||
sprite slicing slicing.png
|
||||
|
||||
Boundary(input, "**1) Process Input Files**") {
|
||||
Boundary(simple, "Read Simple Sources", "Natural Earth & Shapefiles") {
|
||||
System(sr, "Reader")
|
||||
}
|
||||
|
||||
Boundary(osm, "Read OpenStreetMap Data", ".osm.pbf files") {
|
||||
System(profile2, "Profile")
|
||||
Boundary(pass1, "pass 1") {
|
||||
System(osm1n, "Nodes")
|
||||
System(osm1w, "Ways")
|
||||
System(osm1r, "Relations")
|
||||
' enforce ordering
|
||||
osm1r -[hidden]> osm1w
|
||||
osm1w -[hidden]> osm1n
|
||||
}
|
||||
BiRel_Up(osm1r, profile2, " ")
|
||||
SystemDb(osmnl, "Node locations")
|
||||
SystemDb(osmmem, "Relation Members")
|
||||
SystemDb(osmmp, "Multipolygon geometries")
|
||||
SystemDb(osmrl, "Relation Info")
|
||||
Boundary(pass2, "pass 2") {
|
||||
System(osm2n, "Nodes")
|
||||
System(osm2w, "Ways")
|
||||
System(osm2r, "Relations")
|
||||
' enforce ordering
|
||||
osm2r -[hidden]> osm2w
|
||||
osm2w -[hidden]> osm2n
|
||||
}
|
||||
Rel(osm1n, osmnl, " ")
|
||||
Rel(osm1r, osmmem, " ")
|
||||
Rel(osm1r, osmrl, " ")
|
||||
|
||||
Rel(osmnl, osm2w, " ")
|
||||
Rel(osmmem, osm2w, " ")
|
||||
Rel(osmrl, osm2w, " ")
|
||||
Rel_Left(osm2w, osmmp, " ")
|
||||
Rel(osmmp, osm2r, " ")
|
||||
}
|
||||
|
||||
Boundary(workers, "Process Workers", "1 per core") {
|
||||
System(profile, "Profile")
|
||||
System(render, "Render Vector Tile Features")
|
||||
}
|
||||
|
||||
' slicing diagram
|
||||
rectangle "<$slicing{scale=0.5}>" as slicing <<diagram>>
|
||||
slicing -[hidden]> render
|
||||
|
||||
System(writer, "Writer")
|
||||
|
||||
Rel(sr, profile, " ")
|
||||
Rel(osm2n, profile, " ")
|
||||
Rel(osm2w, profile, " ")
|
||||
Rel(osm2r, profile, " ")
|
||||
|
||||
Rel(profile, render, " ")
|
||||
Rel(render, writer, " ")
|
||||
}
|
||||
|
||||
Boundary(sort, "**2) Sort Features**") {
|
||||
SystemDb(fc1, "Chunk")
|
||||
SystemDb(fc2, "Chunk")
|
||||
SystemDb(fc3, "Chunk")
|
||||
' enforce ordering
|
||||
fc1 -[hidden]> fc2
|
||||
fc2 -[hidden]> fc3
|
||||
System(sorters, "Sort Workers\n1 per core")
|
||||
BiRel_Up(fc3, sorters, " ")
|
||||
}
|
||||
|
||||
Rel(writer, fc1, " ")
|
||||
Rel(writer, fc2, " ")
|
||||
Rel(writer, fc3, " ")
|
||||
|
||||
Boundary(mbtiles, "**3) Emit Vector Tiles**") {
|
||||
System(mbtread, "Read Features\nand group into tiles")
|
||||
System(mbtprocess, "Encode & gzip\n1 worker per core")
|
||||
System(mbtwriter, "Batched Writer")
|
||||
}
|
||||
|
||||
SystemDb(mbtilesdb, "MBTiles", "x,y,z,data")
|
||||
|
||||
Rel(fc1, mbtread, " ")
|
||||
Rel(fc2, mbtread, " ")
|
||||
Rel(fc3, mbtread, " ")
|
||||
Rel(mbtread, mbtprocess, " ")
|
||||
Rel(mbtprocess, mbtwriter, " ")
|
||||
Rel(mbtwriter, mbtilesdb, " ")
|
||||
|
||||
|
||||
@enduml
|
||||
|
Plik binarny nie jest wyświetlany.
Po Szerokość: | Wysokość: | Rozmiar: 488 KiB |
Plik binarny nie jest wyświetlany.
Po Szerokość: | Wysokość: | Rozmiar: 8.8 KiB |
|
@ -0,0 +1,48 @@
|
|||
# Flatmap Basemap Profile
|
||||
|
||||
This basemap profile is based on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) v3.12.2.
|
||||
See [README.md](../README.md) in the parent directory for instructions on how to run.
|
||||
|
||||
## Code Layout
|
||||
|
||||
[Generate.java](./src/main/java/com/onthegomap/flatmap/basemap/Generate.java) generates code in
|
||||
the [generated](./src/main/java/com/onthegomap/flatmap/basemap/generated) package from an OpenMapTiles tag in GitHub:
|
||||
|
||||
- [OpenMapTilesSchema](./src/main/java/com/onthegomap/flatmap/basemap/generated/OpenMapTilesSchema.java)
|
||||
contains an interface for each layer with constants for the name, attributes, and their allowed values
|
||||
- [Tables](./src/main/java/com/onthegomap/flatmap/basemap/generated/Tables.java)
|
||||
contains a record for each table that OpenMapTiles [imposm3](https://github.com/omniscale/imposm3) configuration
|
||||
generates (along with the tag-filtering expression) so layers can listen on instances of those records instead of
|
||||
doing the tag filtering and parsing themselves
|
||||
|
||||
The [layers](./src/main/java/com/onthegomap/flatmap/basemap/layers) package contains a port of the SQL logic to generate
|
||||
each layer from OpenMapTiles. Layers define how source features (or parsed imposm3 table rows) map to vector tile
|
||||
features, and logic for post-processing tile geometries.
|
||||
|
||||
[BasemapProfile](./src/main/java/com/onthegomap/flatmap/basemap/BasemapProfile.java)
|
||||
dispatches source features to layers and merges the results.
|
||||
|
||||
[BasemapMain](./src/main/java/com/onthegomap/flatmap/basemap/BasemapMain.java) is the main driver that registers source
|
||||
data and output location.
|
||||
|
||||
## Regenerating Code
|
||||
|
||||
To run `Generate.java`, use [scripts/regenerate-openmaptiles.sh](../scripts/regenerate-openmaptiles.sh) script with the
|
||||
OpenMapTiles release tag:
|
||||
|
||||
```bash
|
||||
./scripts/regenerate-openmaptiles.sh v3.12.2
|
||||
```
|
||||
|
||||
Then follow the instructions it prints for reformatting generated code.
|
||||
|
||||
## License and Attribution
|
||||
|
||||
OpenMapTiles code is licensed under the BSD 3-Clause License, which appears at the top of any file ported from
|
||||
OpenMapTiles.
|
||||
|
||||
The OpenMapTiles schema (or "look and feel") is licensed under [CC-BY 4.0](http://creativecommons.org/licenses/by/4.0/),
|
||||
so any map derived from that schema
|
||||
must [visibly credit OpenMapTiles](https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md#design-license-cc-by-40)
|
||||
. It also uses OpenStreetMap data so you
|
||||
must [visibly credit OpenStreetMap contributors](https://www.openstreetmap.org/copyright).
|
|
@ -0,0 +1,57 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>flatmap-basemap</artifactId>
|
||||
|
||||
<parent>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-parent</artifactId>
|
||||
<version>0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-core</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.yaml</groupId>
|
||||
<artifactId>snakeyaml</artifactId>
|
||||
<version>1.29</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.commonmark</groupId>
|
||||
<artifactId>commonmark</artifactId>
|
||||
<version>0.18.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-core</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>io.github.zlika</groupId>
|
||||
<artifactId>reproducible-build-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- we don't want to deploy this module -->
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
|
@ -1,14 +1,14 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import com.onthegomap.flatmap.FlatmapRunner;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.config.Arguments;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/**
|
||||
* Main entrypoint for generating a map using the OpenMapTiles schema.
|
||||
* Main entrypoint for generating a map using the basemap schema.
|
||||
*/
|
||||
public class OpenMapTilesMain {
|
||||
public class BasemapMain {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
run(Arguments.fromArgsOrConfigFile(args));
|
||||
|
@ -21,7 +21,7 @@ public class OpenMapTilesMain {
|
|||
// will be ignored if osm_path or osm_url are set
|
||||
String area = arguments.getString(
|
||||
"area",
|
||||
"name of the geofabrik extract to download if osm_url/osm_path not specified (i.e. 'monaco' 'rhode island' or 'australia')",
|
||||
"name of the extract to download if osm_url/osm_path not specified (i.e. 'monaco' 'rhode island' 'australia' or 'planet')",
|
||||
"monaco"
|
||||
);
|
||||
|
||||
|
@ -29,22 +29,22 @@ public class OpenMapTilesMain {
|
|||
.setDefaultLanguages(OpenMapTilesSchema.LANGUAGES)
|
||||
.fetchWikidataNameTranslations(sourcesDir.resolve("wikidata_names.json"))
|
||||
// defer creation of the profile because it depends on data from the runner
|
||||
.setProfile(OpenMapTilesProfile::new)
|
||||
.setProfile(BasemapProfile::new)
|
||||
// override any of these with arguments: --osm_path=... or --osm_url=...
|
||||
// or OSM_PATH=... OSM_URL=... environmental argument
|
||||
// or osm_path=... osm_url=... in a config file
|
||||
.addShapefileSource("EPSG:3857", OpenMapTilesProfile.LAKE_CENTERLINE_SOURCE,
|
||||
.addShapefileSource("EPSG:3857", BasemapProfile.LAKE_CENTERLINE_SOURCE,
|
||||
sourcesDir.resolve("lake_centerline.shp.zip"),
|
||||
"https://github.com/lukasmartinelli/osm-lakelines/releases/download/v0.9/lake_centerline.shp.zip")
|
||||
.addShapefileSource(OpenMapTilesProfile.WATER_POLYGON_SOURCE,
|
||||
.addShapefileSource(BasemapProfile.WATER_POLYGON_SOURCE,
|
||||
sourcesDir.resolve("water-polygons-split-3857.zip"),
|
||||
"https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip")
|
||||
.addNaturalEarthSource(OpenMapTilesProfile.NATURAL_EARTH_SOURCE,
|
||||
.addNaturalEarthSource(BasemapProfile.NATURAL_EARTH_SOURCE,
|
||||
sourcesDir.resolve("natural_earth_vector.sqlite.zip"),
|
||||
"https://naturalearth.s3.amazonaws.com/packages/natural_earth_vector.sqlite.zip") // TODO: go back to "https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip")
|
||||
.addOsmSource(OpenMapTilesProfile.OSM_SOURCE,
|
||||
"https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip")
|
||||
.addOsmSource(BasemapProfile.OSM_SOURCE,
|
||||
sourcesDir.resolve(area.replaceAll("[^a-zA-Z]+", "_") + ".osm.pbf"),
|
||||
"geofabrik:" + area)
|
||||
"planet".equalsIgnoreCase(area) ? ("aws:latest") : ("geofabrik:" + area))
|
||||
// override with --mbtiles=... argument or MBTILES=... env var or mbtiles=... in a config file
|
||||
.setOutput("mbtiles", dataDir.resolve("output.mbtiles"))
|
||||
.run();
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import static com.onthegomap.flatmap.geo.GeoUtils.EMPTY_LINE;
|
||||
import static com.onthegomap.flatmap.geo.GeoUtils.EMPTY_POINT;
|
||||
|
@ -8,10 +8,10 @@ import com.onthegomap.flatmap.FeatureCollector;
|
|||
import com.onthegomap.flatmap.FlatmapRunner;
|
||||
import com.onthegomap.flatmap.ForwardingProfile;
|
||||
import com.onthegomap.flatmap.Profile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.reader.SimpleFeature;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmElement;
|
||||
|
@ -21,8 +21,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Delegates the logic for generating a map using OpenMapTiles vector schema to individual implementations in the {@code
|
||||
* layers} package.
|
||||
* Delegates the logic for generating a map to individual implementations in the {@code layers} package.
|
||||
* <p>
|
||||
* Layer implementations extend these interfaces to subscribe to elements from different sources:
|
||||
* <ul>
|
||||
|
@ -40,7 +39,7 @@ import java.util.List;
|
|||
* {@link FinishHandler} or post-process features in that layer before rendering the output tile by implementing
|
||||
* {@link FeaturePostProcessor}.
|
||||
*/
|
||||
public class OpenMapTilesProfile extends ForwardingProfile {
|
||||
public class BasemapProfile extends ForwardingProfile {
|
||||
|
||||
// IDs used in stats and logs for each input source, as well as argument/config file overrides to source locations
|
||||
public static final String LAKE_CENTERLINE_SOURCE = "lake_centerlines";
|
||||
|
@ -52,11 +51,11 @@ public class OpenMapTilesProfile extends ForwardingProfile {
|
|||
/** Index variant that filters out any table only used by layers that implement IgnoreWikidata class. */
|
||||
private final MultiExpression.Index<Boolean> wikidataMappings;
|
||||
|
||||
public OpenMapTilesProfile(FlatmapRunner runner) {
|
||||
public BasemapProfile(FlatmapRunner runner) {
|
||||
this(runner.translations(), runner.config(), runner.stats());
|
||||
}
|
||||
|
||||
public OpenMapTilesProfile(Translations translations, FlatmapConfig config, Stats stats) {
|
||||
public BasemapProfile(Translations translations, FlatmapConfig config, Stats stats) {
|
||||
List<String> onlyLayers = config.arguments().getList("only_layers", "Include only certain layers", List.of());
|
||||
List<String> excludeLayers = config.arguments().getList("exclude_layers", "Exclude certain layers", List.of());
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import static com.onthegomap.flatmap.expression.Expression.*;
|
||||
import static java.util.stream.Collectors.joining;
|
||||
|
@ -9,12 +9,13 @@ import com.fasterxml.jackson.databind.JsonNode;
|
|||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.CaseFormat;
|
||||
import com.onthegomap.flatmap.config.Arguments;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.Expression;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.util.Downloader;
|
||||
import com.onthegomap.flatmap.util.FileUtils;
|
||||
import com.onthegomap.flatmap.util.Format;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
|
@ -54,6 +55,7 @@ public class Generate {
|
|||
private static final Logger LOGGER = LoggerFactory.getLogger(Generate.class);
|
||||
private static final ObjectMapper mapper = new ObjectMapper();
|
||||
private static final Yaml yaml;
|
||||
private static final String LINE_SEPARATOR = System.lineSeparator();
|
||||
private static final String GENERATED_FILE_HEADER = """
|
||||
/*
|
||||
Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors.
|
||||
|
@ -102,9 +104,9 @@ public class Generate {
|
|||
yaml = new Yaml(options);
|
||||
}
|
||||
|
||||
private static <T> T loadAndParseYaml(URL url, Class<T> clazz) throws IOException {
|
||||
private static <T> T loadAndParseYaml(String url, FlatmapConfig config, Class<T> clazz) throws IOException {
|
||||
LOGGER.info("reading " + url);
|
||||
try (var stream = url.openStream()) {
|
||||
try (var stream = Downloader.openStream(url, config)) {
|
||||
// Jackson yaml parsing does not handle anchors and references, so first parse the input
|
||||
// using SnakeYAML, then parse SnakeYAML's output using Jackson to get it into our records.
|
||||
Map<String, Object> parsed = yaml.load(stream);
|
||||
|
@ -125,20 +127,21 @@ public class Generate {
|
|||
|
||||
public static void main(String[] args) throws IOException {
|
||||
Arguments arguments = Arguments.fromArgsOrConfigFile(args);
|
||||
FlatmapConfig flatmapConfig = FlatmapConfig.from(arguments);
|
||||
String tag = arguments.getString("tag", "openmaptiles tag to use", "v3.12.2");
|
||||
String base = "https://raw.githubusercontent.com/openmaptiles/openmaptiles/" + tag + "/";
|
||||
|
||||
// start crawling from openmaptiles.yaml
|
||||
// then crawl schema from each layers/<layer>/<layer>.yaml file that it references
|
||||
// then crawl table definitions from each layers/<layer>/mapping.yaml file that the layer references
|
||||
var rootUrl = new URL(base + "openmaptiles.yaml");
|
||||
OpenmaptilesConfig config = loadAndParseYaml(rootUrl, OpenmaptilesConfig.class);
|
||||
String rootUrl = base + "openmaptiles.yaml";
|
||||
OpenmaptilesConfig config = loadAndParseYaml(rootUrl, flatmapConfig, OpenmaptilesConfig.class);
|
||||
|
||||
List<LayerConfig> layers = new ArrayList<>();
|
||||
Set<String> imposm3MappingFiles = new LinkedHashSet<>();
|
||||
for (String layerFile : config.tileset.layers) {
|
||||
URL layerURL = new URL(base + layerFile);
|
||||
LayerConfig layer = loadAndParseYaml(layerURL, LayerConfig.class);
|
||||
String layerURL = base + layerFile;
|
||||
LayerConfig layer = loadAndParseYaml(layerURL, flatmapConfig, LayerConfig.class);
|
||||
layers.add(layer);
|
||||
for (Datasource datasource : layer.datasources) {
|
||||
if ("imposm3".equals(datasource.type)) {
|
||||
|
@ -152,13 +155,13 @@ public class Generate {
|
|||
|
||||
Map<String, Imposm3Table> tables = new LinkedHashMap<>();
|
||||
for (String uri : imposm3MappingFiles) {
|
||||
Imposm3Mapping layer = loadAndParseYaml(new URL(uri), Imposm3Mapping.class);
|
||||
Imposm3Mapping layer = loadAndParseYaml(uri, flatmapConfig, Imposm3Mapping.class);
|
||||
tables.putAll(layer.tables);
|
||||
}
|
||||
|
||||
String packageName = "com.onthegomap.flatmap.openmaptiles.generated";
|
||||
String packageName = "com.onthegomap.flatmap.basemap.generated";
|
||||
String[] packageParts = packageName.split("\\.");
|
||||
Path output = Path.of("flatmap-openmaptiles", "src", "main", "java")
|
||||
Path output = Path.of("flatmap-basemap", "src", "main", "java")
|
||||
.resolve(Path.of(packageParts[0], Arrays.copyOfRange(packageParts, 1, packageParts.length)));
|
||||
|
||||
FileUtils.deleteDirectory(output);
|
||||
|
@ -183,7 +186,7 @@ public class Generate {
|
|||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.Layer;
|
||||
import com.onthegomap.flatmap.basemap.Layer;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -220,9 +223,9 @@ public class Generate {
|
|||
info.languages.stream().map(Format::quote).collect(joining(", ")),
|
||||
layers.stream()
|
||||
.map(
|
||||
l -> "new com.onthegomap.flatmap.openmaptiles.layers.%s(translations, config, stats)"
|
||||
l -> "new com.onthegomap.flatmap.basemap.layers.%s(translations, config, stats)"
|
||||
.formatted(lowerUnderscoreToUpperCamel(l.layer.id)))
|
||||
.collect(joining(",\n"))
|
||||
.collect(joining("," + LINE_SEPARATOR))
|
||||
.indent(6).trim()
|
||||
));
|
||||
for (var layer : layers) {
|
||||
|
@ -263,7 +266,7 @@ public class Generate {
|
|||
* </ul>
|
||||
*/
|
||||
""".stripTrailing().formatted(javadocDescription,
|
||||
valuesForComment.stream().map(v -> "<li>" + v).collect(joining("\n * "))),
|
||||
valuesForComment.stream().map(v -> "<li>" + v).collect(joining(LINE_SEPARATOR + " * "))),
|
||||
name.toUpperCase(Locale.ROOT),
|
||||
Format.quote(name)
|
||||
).indent(4));
|
||||
|
@ -277,7 +280,7 @@ public class Generate {
|
|||
.map(v -> "public static final String %s = %s;"
|
||||
.formatted(name.toUpperCase(Locale.ROOT) + "_" + v.toUpperCase(Locale.ROOT).replace('-', '_'),
|
||||
Format.quote(v)))
|
||||
.collect(joining("\n")).indent(2).strip()
|
||||
.collect(joining(LINE_SEPARATOR)).indent(2).strip()
|
||||
.indent(4));
|
||||
fieldValues.append("public static final Set<String> %s = Set.of(%s);".formatted(
|
||||
name.toUpperCase(Locale.ROOT) + "_VALUES",
|
||||
|
@ -287,7 +290,7 @@ public class Generate {
|
|||
|
||||
if (valuesNode != null && valuesNode.isObject()) {
|
||||
MultiExpression<String> mapping = generateFieldMapping(valuesNode);
|
||||
fieldMappings.append(" public static final MultiExpression<String> %s = %s;\n"
|
||||
fieldMappings.append(" public static final MultiExpression<String> %s = %s;%n"
|
||||
.formatted(lowerUnderscoreToUpperCamel(name), generateJavaCode(mapping)));
|
||||
}
|
||||
});
|
||||
|
@ -477,7 +480,7 @@ public class Generate {
|
|||
interfaceName,
|
||||
type,
|
||||
attrName);
|
||||
}).collect(joining("\n")).indent(2));
|
||||
}).collect(joining(LINE_SEPARATOR)).indent(2));
|
||||
|
||||
tablesClass.append("""
|
||||
/** Index to efficiently choose which imposm3 "tables" an element should appear in based on its attributes. */
|
||||
|
@ -488,7 +491,7 @@ public class Generate {
|
|||
classNames.stream().map(
|
||||
className -> "MultiExpression.entry(new RowClassAndConstructor(%s.class, %s::new), %s.MAPPING)".formatted(
|
||||
className, className, className))
|
||||
.collect(joining(",\n")).indent(2).strip()
|
||||
.collect(joining("," + LINE_SEPARATOR)).indent(2).strip()
|
||||
).indent(2));
|
||||
|
||||
String handlerCondition = classNames.stream().map(className ->
|
||||
|
@ -496,7 +499,7 @@ public class Generate {
|
|||
if (handler instanceof %s.Handler typedHandler) {
|
||||
result.computeIfAbsent(%s.class, cls -> new ArrayList<>()).add(new RowHandlerAndClass<>(typedHandler.getClass(), typedHandler::process));
|
||||
}""".formatted(className, className)
|
||||
).collect(joining("\n"));
|
||||
).collect(joining(LINE_SEPARATOR));
|
||||
tablesClass.append("""
|
||||
/**
|
||||
* Returns a map from imposm3 "table row" class to the layers that have a handler for it from a list of layer
|
||||
|
@ -663,11 +666,11 @@ public class Generate {
|
|||
|
||||
/** Renders {@code markdown} as HTML and returns comment text safe to insert in generated javadoc. */
|
||||
private static String markdownToJavadoc(String markdown) {
|
||||
return Stream.of(markdown.strip().split("\n\n+"))
|
||||
return Stream.of(markdown.strip().split("[\r\n][\r\n]+"))
|
||||
.map(p -> parser.parse(p.strip()))
|
||||
.map(node -> escapeJavadoc(renderer.render(node)))
|
||||
.map(p -> p.replaceAll("(^<p>|</p>$)", "").strip())
|
||||
.collect(joining("\n<p>\n"));
|
||||
.collect(joining(LINE_SEPARATOR + "<p>" + LINE_SEPARATOR));
|
||||
}
|
||||
|
||||
/** Returns {@code comment} text safe to insert in generated javadoc. */
|
|
@ -1,8 +1,8 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import com.onthegomap.flatmap.ForwardingProfile;
|
||||
|
||||
/** Interface for all vector tile layer implementations that {@link OpenMapTilesProfile} delegates to. */
|
||||
/** Interface for all vector tile layer implementations that {@link BasemapProfile} delegates to. */
|
||||
public interface Layer extends
|
||||
ForwardingProfile.Handler,
|
||||
ForwardingProfile.HandlerForLayer {}
|
|
@ -35,16 +35,16 @@ See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for deta
|
|||
*/
|
||||
// AUTOGENERATED BY Generate.java -- DO NOT MODIFY
|
||||
|
||||
package com.onthegomap.flatmap.openmaptiles.generated;
|
||||
package com.onthegomap.flatmap.basemap.generated;
|
||||
|
||||
import static com.onthegomap.flatmap.expression.Expression.FALSE;
|
||||
import static com.onthegomap.flatmap.expression.Expression.and;
|
||||
import static com.onthegomap.flatmap.expression.Expression.matchAny;
|
||||
import static com.onthegomap.flatmap.expression.Expression.or;
|
||||
|
||||
import com.onthegomap.flatmap.basemap.Layer;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.Layer;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
import java.util.List;
|
||||
|
@ -53,8 +53,7 @@ import java.util.Set;
|
|||
/**
|
||||
* All vector tile layer definitions, attributes, and allowed values generated from the
|
||||
* <a href="https://github.com/openmaptiles/openmaptiles/blob/v3.12.2/openmaptiles.yaml">OpenMapTiles vector tile
|
||||
* schema
|
||||
* v3.12.2</a>.
|
||||
* schema v3.12.2</a>.
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
public class OpenMapTilesSchema {
|
||||
|
@ -72,22 +71,22 @@ public class OpenMapTilesSchema {
|
|||
/** Returns a list of expected layer implementation instances from the {@code layers} package. */
|
||||
public static List<Layer> createInstances(Translations translations, FlatmapConfig config, Stats stats) {
|
||||
return List.of(
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Water(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Waterway(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Landcover(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Landuse(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.MountainPeak(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Park(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Boundary(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Aeroway(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Transportation(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Building(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.WaterName(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.TransportationName(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Place(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Housenumber(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.Poi(translations, config, stats),
|
||||
new com.onthegomap.flatmap.openmaptiles.layers.AerodromeLabel(translations, config, stats)
|
||||
new com.onthegomap.flatmap.basemap.layers.Water(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Waterway(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Landcover(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Landuse(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.MountainPeak(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Park(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Boundary(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Aeroway(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Transportation(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Building(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.WaterName(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.TransportationName(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Place(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Housenumber(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.Poi(translations, config, stats),
|
||||
new com.onthegomap.flatmap.basemap.layers.AerodromeLabel(translations, config, stats)
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -1492,9 +1491,8 @@ public class OpenMapTilesSchema {
|
|||
* from population and city class). You can use the <strong>rank</strong> to limit density of labels or improve
|
||||
* the text hierarchy. The rank value is a combination of the Natural Earth <code>scalerank</code>,
|
||||
* <code>labelrank</code> and <code>datarank</code> values for countries and states and for cities consists out
|
||||
* of
|
||||
* a shifted Natural Earth <code>scalerank</code> combined with a local rank within a grid for cities that do not
|
||||
* have a Natural Earth <code>scalerank</code>.
|
||||
* of a shifted Natural Earth <code>scalerank</code> combined with a local rank within a grid for cities that do
|
||||
* not have a Natural Earth <code>scalerank</code>.
|
||||
*/
|
||||
public static final String RANK = "rank";
|
||||
}
|
|
@ -35,7 +35,7 @@ See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for deta
|
|||
*/
|
||||
// AUTOGENERATED BY Generate.java -- DO NOT MODIFY
|
||||
|
||||
package com.onthegomap.flatmap.openmaptiles.generated;
|
||||
package com.onthegomap.flatmap.basemap.generated;
|
||||
|
||||
import static com.onthegomap.flatmap.expression.Expression.*;
|
||||
|
|
@ -33,17 +33,17 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.basemap.util.Utils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.Utils;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
|
@ -33,12 +33,12 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
|
@ -33,7 +33,7 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.util.MemoryEstimator.CLASS_HEADER_BYTES;
|
||||
import static com.onthegomap.flatmap.util.MemoryEstimator.POINTER_BYTES;
|
||||
|
@ -46,16 +46,17 @@ import com.graphhopper.coll.GHLongObjectHashMap;
|
|||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.FeatureMerge;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.reader.SimpleFeature;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmElement;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Format;
|
||||
import com.onthegomap.flatmap.util.MemoryEstimator;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -90,11 +91,11 @@ import org.slf4j.LoggerFactory;
|
|||
*/
|
||||
public class Boundary implements
|
||||
OpenMapTilesSchema.Boundary,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
OpenMapTilesProfile.OsmRelationPreprocessor,
|
||||
OpenMapTilesProfile.OsmAllProcessor,
|
||||
OpenMapTilesProfile.FeaturePostProcessor,
|
||||
OpenMapTilesProfile.FinishHandler {
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.OsmRelationPreprocessor,
|
||||
BasemapProfile.OsmAllProcessor,
|
||||
BasemapProfile.FeaturePostProcessor,
|
||||
BasemapProfile.FinishHandler {
|
||||
|
||||
/*
|
||||
* Uses natural earth at lower zoom levels and OpenStreetMap at higher zoom levels.
|
||||
|
@ -292,7 +293,7 @@ public class Boundary implements
|
|||
@Override
|
||||
public void finish(String sourceName, FeatureCollector.Factory featureCollectors,
|
||||
Consumer<FeatureCollector.Feature> emit) {
|
||||
if (OpenMapTilesProfile.OSM_SOURCE.equals(sourceName)) {
|
||||
if (BasemapProfile.OSM_SOURCE.equals(sourceName)) {
|
||||
var timer = stats.startStage("boundaries");
|
||||
LongObjectMap<PreparedGeometry> countryBoundaries = prepareRegionPolygons();
|
||||
|
||||
|
@ -374,11 +375,10 @@ public class Boundary implements
|
|||
|
||||
if (left == null && right == null) {
|
||||
Coordinate point = GeoUtils.worldToLatLonCoords(GeoUtils.pointAlongOffset(lineString, 0.5, 0)).getCoordinate();
|
||||
LOGGER.warn("no left or right country for border between OSM country relations: %s around %.5f, %.5f"
|
||||
LOGGER.warn("no left or right country for border between OSM country relations: %s around %s"
|
||||
.formatted(
|
||||
validRegions,
|
||||
point.getX(),
|
||||
point.getY()
|
||||
Format.osmDebugUrl(10, point)
|
||||
));
|
||||
}
|
||||
|
|
@ -33,9 +33,9 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.util.MemoryEstimator.CLASS_HEADER_BYTES;
|
||||
import static com.onthegomap.flatmap.util.Parse.parseDoubleOrNull;
|
||||
import static java.util.Map.entry;
|
||||
|
@ -43,11 +43,11 @@ import static java.util.Map.entry;
|
|||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.FeatureMerge;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmElement;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
|
@ -66,8 +66,8 @@ import java.util.Map;
|
|||
public class Building implements
|
||||
OpenMapTilesSchema.Building,
|
||||
Tables.OsmBuildingPolygon.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor,
|
||||
OpenMapTilesProfile.OsmRelationPreprocessor {
|
||||
BasemapProfile.FeaturePostProcessor,
|
||||
BasemapProfile.OsmRelationPreprocessor {
|
||||
|
||||
/*
|
||||
* Emit all buildings from OSM data at z14.
|
|
@ -33,12 +33,12 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
|
@ -33,17 +33,17 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.FeatureMerge;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -62,9 +62,9 @@ import java.util.Set;
|
|||
*/
|
||||
public class Landcover implements
|
||||
OpenMapTilesSchema.Landcover,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
Tables.OsmLandcoverPolygon.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor {
|
||||
BasemapProfile.FeaturePostProcessor {
|
||||
|
||||
/*
|
||||
* Large ice areas come from natural earth and the rest come from OpenStreetMap at higher zoom
|
|
@ -33,16 +33,16 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
|
@ -60,7 +60,7 @@ import java.util.Set;
|
|||
*/
|
||||
public class Landuse implements
|
||||
OpenMapTilesSchema.Landuse,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
Tables.OsmLandusePolygon.Handler {
|
||||
|
||||
private static final ZoomFunction<Number> MIN_PIXEL_SIZE_THRESHOLDS = ZoomFunction.fromMaxZoomThresholds(Map.of(
|
|
@ -33,21 +33,21 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.elevationTags;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.elevationTags;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.carrotsearch.hppc.LongIntMap;
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -65,7 +65,7 @@ import org.locationtech.jts.geom.Point;
|
|||
public class MountainPeak implements
|
||||
OpenMapTilesSchema.MountainPeak,
|
||||
Tables.OsmPeakPoint.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor {
|
||||
BasemapProfile.FeaturePostProcessor {
|
||||
|
||||
/*
|
||||
* Mountain peaks come from OpenStreetMap data and are ranked by importance (based on if they
|
|
@ -33,24 +33,24 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_BITS;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.carrotsearch.hppc.LongIntMap;
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.geo.GeometryType;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.SortKey;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -67,7 +67,7 @@ import java.util.Locale;
|
|||
public class Park implements
|
||||
OpenMapTilesSchema.Park,
|
||||
Tables.OsmParkPolygon.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor {
|
||||
BasemapProfile.FeaturePostProcessor {
|
||||
|
||||
// constants for packing the minimum zoom ordering of park labels into the sort-key field
|
||||
private static final int PARK_NATIONAL_PARK_BOOST = 1 << (SORT_KEY_BITS - 1);
|
|
@ -33,26 +33,26 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullOrEmpty;
|
||||
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_BITS;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullOrEmpty;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.carrotsearch.hppc.LongIntMap;
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.geo.PointIndex;
|
||||
import com.onthegomap.flatmap.geo.PolygonIndex;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
|
@ -81,14 +81,14 @@ import org.locationtech.jts.geom.Point;
|
|||
*/
|
||||
public class Place implements
|
||||
OpenMapTilesSchema.Place,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
Tables.OsmContinentPoint.Handler,
|
||||
Tables.OsmCountryPoint.Handler,
|
||||
Tables.OsmStatePoint.Handler,
|
||||
Tables.OsmIslandPoint.Handler,
|
||||
Tables.OsmIslandPolygon.Handler,
|
||||
Tables.OsmCityPoint.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor {
|
||||
BasemapProfile.FeaturePostProcessor {
|
||||
|
||||
/*
|
||||
* Place labels locations and names come from OpenStreetMap, but we also join with natural
|
|
@ -33,24 +33,24 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIf;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullOrEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIf;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullOrEmpty;
|
||||
import static java.util.Map.entry;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.carrotsearch.hppc.LongIntMap;
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -68,7 +68,7 @@ public class Poi implements
|
|||
OpenMapTilesSchema.Poi,
|
||||
Tables.OsmPoiPoint.Handler,
|
||||
Tables.OsmPoiPolygon.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor {
|
||||
BasemapProfile.FeaturePostProcessor {
|
||||
|
||||
/*
|
||||
* process() creates the raw POI feature from OSM elements and postProcess()
|
|
@ -33,19 +33,19 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.*;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.*;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.FeatureMerge;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -69,8 +69,8 @@ public class Transportation implements
|
|||
Tables.OsmRailwayLinestring.Handler,
|
||||
Tables.OsmShipwayLinestring.Handler,
|
||||
Tables.OsmHighwayPolygon.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor,
|
||||
OpenMapTilesProfile.IgnoreWikidata {
|
||||
BasemapProfile.FeaturePostProcessor,
|
||||
BasemapProfile.IgnoreWikidata {
|
||||
|
||||
/*
|
||||
* Generates the shape for roads, trails, ferries, railways with detailed
|
||||
|
@ -273,6 +273,7 @@ public class Transportation implements
|
|||
.setAttrWithMinzoom(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()), 10)
|
||||
.setAttrWithMinzoom(Fields.LAYER, nullIf(element.layer(), 0), 9)
|
||||
.setSortKey(element.zOrder())
|
||||
.setMinPixelSize(0) // merge during post-processing, then limit by size
|
||||
.setMinZoom(minzoom);
|
||||
}
|
||||
}
|
||||
|
@ -288,6 +289,7 @@ public class Transportation implements
|
|||
.setAttr(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()))
|
||||
.setAttr(Fields.LAYER, nullIf(element.layer(), 0))
|
||||
.setSortKey(element.zOrder())
|
||||
.setMinPixelSize(0) // merge during post-processing, then limit by size
|
||||
.setMinZoom(12);
|
||||
}
|
||||
|
||||
|
@ -302,6 +304,7 @@ public class Transportation implements
|
|||
.setAttr(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()))
|
||||
.setAttr(Fields.LAYER, nullIf(element.layer(), 0))
|
||||
.setSortKey(element.zOrder())
|
||||
.setMinPixelSize(0) // merge during post-processing, then limit by size
|
||||
.setMinZoom(11);
|
||||
}
|
||||
|
|
@ -33,15 +33,15 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.layers.Transportation.highwayClass;
|
||||
import static com.onthegomap.flatmap.openmaptiles.layers.Transportation.highwaySubclass;
|
||||
import static com.onthegomap.flatmap.openmaptiles.layers.Transportation.isFootwayOrSteps;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.brunnel;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIf;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.layers.Transportation.highwayClass;
|
||||
import static com.onthegomap.flatmap.basemap.layers.Transportation.highwaySubclass;
|
||||
import static com.onthegomap.flatmap.basemap.layers.Transportation.isFootwayOrSteps;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.brunnel;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIf;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.util.MemoryEstimator.CLASS_HEADER_BYTES;
|
||||
import static com.onthegomap.flatmap.util.MemoryEstimator.POINTER_BYTES;
|
||||
import static com.onthegomap.flatmap.util.MemoryEstimator.estimateSize;
|
||||
|
@ -49,13 +49,13 @@ import static com.onthegomap.flatmap.util.MemoryEstimator.estimateSize;
|
|||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.FeatureMerge;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmElement;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmReader;
|
||||
|
@ -88,10 +88,10 @@ import org.slf4j.LoggerFactory;
|
|||
public class TransportationName implements
|
||||
OpenMapTilesSchema.TransportationName,
|
||||
Tables.OsmHighwayLinestring.Handler,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
OpenMapTilesProfile.FeaturePostProcessor,
|
||||
OpenMapTilesProfile.OsmRelationPreprocessor,
|
||||
OpenMapTilesProfile.IgnoreWikidata {
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.FeaturePostProcessor,
|
||||
BasemapProfile.OsmRelationPreprocessor,
|
||||
BasemapProfile.IgnoreWikidata {
|
||||
|
||||
/*
|
||||
* Generate road names from OSM data. Route network and ref are copied
|
|
@ -33,15 +33,15 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.Utils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.Utils;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -55,8 +55,8 @@ import com.onthegomap.flatmap.util.Translations;
|
|||
public class Water implements
|
||||
OpenMapTilesSchema.Water,
|
||||
Tables.OsmWaterPolygon.Handler,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
OpenMapTilesProfile.OsmWaterPolygonProcessor {
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.OsmWaterPolygonProcessor {
|
||||
|
||||
/*
|
||||
* At low zoom levels, use natural earth for oceans and major lakes, and at high zoom levels
|
|
@ -33,20 +33,20 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.carrotsearch.hppc.LongObjectMap;
|
||||
import com.graphhopper.coll.GHLongObjectHashMap;
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
|
@ -68,8 +68,8 @@ public class WaterName implements
|
|||
OpenMapTilesSchema.WaterName,
|
||||
Tables.OsmMarinePoint.Handler,
|
||||
Tables.OsmWaterPolygon.Handler,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor,
|
||||
OpenMapTilesProfile.LakeCenterlineProcessor {
|
||||
BasemapProfile.NaturalEarthProcessor,
|
||||
BasemapProfile.LakeCenterlineProcessor {
|
||||
|
||||
/*
|
||||
* Labels for lakes and oceans come primarily from OpenStreetMap data, but we also join
|
|
@ -33,19 +33,19 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.FeatureMerge;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.basemap.generated.Tables;
|
||||
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.basemap.util.Utils;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
|
||||
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
|
||||
import com.onthegomap.flatmap.openmaptiles.util.Utils;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
|
@ -62,8 +62,8 @@ import java.util.Map;
|
|||
public class Waterway implements
|
||||
OpenMapTilesSchema.Waterway,
|
||||
Tables.OsmWaterwayLinestring.Handler,
|
||||
OpenMapTilesProfile.FeaturePostProcessor,
|
||||
OpenMapTilesProfile.NaturalEarthProcessor {
|
||||
BasemapProfile.FeaturePostProcessor,
|
||||
BasemapProfile.NaturalEarthProcessor {
|
||||
|
||||
/*
|
||||
* Uses Natural Earth at lower zoom-levels and OpenStreetMap at higher zoom levels.
|
|
@ -33,10 +33,10 @@ Design license: CC-BY 4.0
|
|||
|
||||
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
|
||||
*/
|
||||
package com.onthegomap.flatmap.openmaptiles.util;
|
||||
package com.onthegomap.flatmap.basemap.util;
|
||||
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
|
||||
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
|
||||
|
||||
import com.onthegomap.flatmap.util.Translations;
|
||||
import java.util.HashMap;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.util;
|
||||
package com.onthegomap.flatmap.basemap.util;
|
||||
|
||||
import com.onthegomap.flatmap.util.Parse;
|
||||
import java.util.Map;
|
|
@ -0,0 +1,44 @@
|
|||
package com.onthegomap.flatmap.basemap.util;
|
||||
|
||||
import com.onthegomap.flatmap.mbtiles.Mbtiles;
|
||||
import com.onthegomap.flatmap.mbtiles.Verify;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.locationtech.jts.geom.LineString;
|
||||
import org.locationtech.jts.geom.Point;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
|
||||
/**
|
||||
* A utility to check the contents of an mbtiles file generated for Monaco.
|
||||
*/
|
||||
public class VerifyMonaco {
|
||||
|
||||
public static final Envelope MONACO_BOUNDS = new Envelope(7.40921, 7.44864, 43.72335, 43.75169);
|
||||
|
||||
/**
|
||||
* Returns a verification result with a basic set of checks against an openmaptiles map built from an extract for
|
||||
* Monaco.
|
||||
*/
|
||||
public static Verify verify(Mbtiles mbtiles) {
|
||||
Verify verify = Verify.verify(mbtiles);
|
||||
verify.checkMinFeatureCount(MONACO_BOUNDS, "building", Map.of(), 13, 14, 100, Polygon.class);
|
||||
verify.checkMinFeatureCount(MONACO_BOUNDS, "transportation", Map.of(), 10, 14, 5, LineString.class);
|
||||
verify.checkMinFeatureCount(MONACO_BOUNDS, "landcover", Map.of(
|
||||
"class", "grass",
|
||||
"subclass", "park"
|
||||
), 14, 10, Polygon.class);
|
||||
verify.checkMinFeatureCount(MONACO_BOUNDS, "water", Map.of("class", "ocean"), 0, 14, 1, Polygon.class);
|
||||
verify.checkMinFeatureCount(MONACO_BOUNDS, "place", Map.of("class", "country"), 2, 14, 1, Point.class);
|
||||
return verify;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
try (var mbtiles = Mbtiles.newReadOnlyDatabase(Path.of(args[0]))) {
|
||||
var result = verify(mbtiles);
|
||||
result.print();
|
||||
result.failIfErrors();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
@ -11,12 +11,12 @@ import com.onthegomap.flatmap.util.Wikidata;
|
|||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class OpenMapTilesProfileTest {
|
||||
public class BasemapProfileTest {
|
||||
|
||||
private final Wikidata.WikidataTranslations wikidataTranslations = new Wikidata.WikidataTranslations();
|
||||
private final Translations translations = Translations.defaultProvider(List.of("en", "es", "de"))
|
||||
.addTranslationProvider(wikidataTranslations);
|
||||
private final OpenMapTilesProfile profile = new OpenMapTilesProfile(translations, FlatmapConfig.defaults(),
|
||||
private final BasemapProfile profile = new BasemapProfile(translations, FlatmapConfig.defaults(),
|
||||
Stats.inMemory());
|
||||
|
||||
@Test
|
|
@ -1,49 +1,53 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.assertContains;
|
||||
import static com.onthegomap.flatmap.TestUtils.assertFeatureNear;
|
||||
import static com.onthegomap.flatmap.TestUtils.gunzip;
|
||||
import static com.onthegomap.flatmap.basemap.util.VerifyMonaco.MONACO_BOUNDS;
|
||||
import static com.onthegomap.flatmap.util.Gzip.gunzip;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
|
||||
|
||||
import com.onthegomap.flatmap.TestUtils;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.util.VerifyMonaco;
|
||||
import com.onthegomap.flatmap.config.Arguments;
|
||||
import com.onthegomap.flatmap.mbiles.Mbtiles;
|
||||
import com.onthegomap.flatmap.mbtiles.Mbtiles;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.DynamicTest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestFactory;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.locationtech.jts.geom.LineString;
|
||||
import org.locationtech.jts.geom.Point;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
|
||||
/**
|
||||
* End-to-end tests for OpenMapTiles map generation.
|
||||
* End-to-end tests for basemap generation.
|
||||
* <p>
|
||||
* Generates an entire map for the smallest openstreetmap extract available (Monaco) and asserts that expected output
|
||||
* features exist
|
||||
*/
|
||||
public class OpenMapTilesTest {
|
||||
public class BasemapTest {
|
||||
|
||||
@TempDir
|
||||
static Path tmpDir;
|
||||
private static Mbtiles mbtiles;
|
||||
|
||||
private static final Envelope monacoBounds = new Envelope(7.40921, 7.44864, 43.72335, 43.75169);
|
||||
|
||||
@BeforeAll
|
||||
public static void runFlatmap() throws Exception {
|
||||
Path dbPath = tmpDir.resolve("output.mbtiles");
|
||||
OpenMapTilesMain.run(Arguments.of(
|
||||
BasemapMain.run(Arguments.of(
|
||||
// Override input source locations
|
||||
"osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"),
|
||||
"natural_earth_path", TestUtils.pathToResource("natural_earth_vector.sqlite"),
|
||||
"natural_earth_path", TestUtils.pathToResource("natural_earth_vector.sqlite.zip"),
|
||||
"water_polygons_path", TestUtils.pathToResource("water-polygons-split-3857.zip"),
|
||||
// no centerlines in monaco - so fake it out with an empty source
|
||||
"lake_centerlines_path", TestUtils.pathToResource("water-polygons-split-3857.zip"),
|
||||
|
@ -206,8 +210,16 @@ public class OpenMapTilesTest {
|
|||
), 14, 6, LineString.class);
|
||||
}
|
||||
|
||||
@TestFactory
|
||||
public Stream<DynamicTest> testVerifyChecks() {
|
||||
return VerifyMonaco.verify(mbtiles).results().stream()
|
||||
.map(check -> dynamicTest(check.name(), () -> {
|
||||
check.error().ifPresent(Assertions::fail);
|
||||
}));
|
||||
}
|
||||
|
||||
private static void assertNumFeatures(String layer, Map<String, Object> attrs, int zoom,
|
||||
int expected, Class<? extends Geometry> clazz) {
|
||||
TestUtils.assertNumFeatures(mbtiles, layer, zoom, attrs, monacoBounds, expected, clazz);
|
||||
TestUtils.assertNumFeatures(mbtiles, layer, zoom, attrs, MONACO_BOUNDS, expected, clazz);
|
||||
}
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.openmaptiles;
|
||||
package com.onthegomap.flatmap.basemap;
|
||||
|
||||
import static com.onthegomap.flatmap.basemap.Generate.parseYaml;
|
||||
import static com.onthegomap.flatmap.expression.Expression.*;
|
||||
import static com.onthegomap.flatmap.openmaptiles.Generate.parseYaml;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
|
||||
|
|
@ -1,20 +1,20 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.assertSubmap;
|
||||
import static com.onthegomap.flatmap.TestUtils.newLineString;
|
||||
import static com.onthegomap.flatmap.TestUtils.newPoint;
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.TestUtils;
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.reader.SimpleFeature;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmReader;
|
||||
|
@ -35,7 +35,7 @@ public abstract class AbstractLayerTest {
|
|||
.addTranslationProvider(wikidataTranslations);
|
||||
|
||||
final FlatmapConfig params = FlatmapConfig.defaults();
|
||||
final OpenMapTilesProfile profile = new OpenMapTilesProfile(translations, FlatmapConfig.defaults(),
|
||||
final BasemapProfile profile = new BasemapProfile(translations, FlatmapConfig.defaults(),
|
||||
Stats.inMemory());
|
||||
final Stats stats = Stats.inMemory();
|
||||
final FeatureCollector.Factory featureCollectorFactory = new FeatureCollector.Factory(params, stats);
|
||||
|
@ -61,8 +61,8 @@ public abstract class AbstractLayerTest {
|
|||
if (vals[i - 1] > vals[i]) {
|
||||
fail(
|
||||
Arrays.toString(vals) +
|
||||
"\nelement at " + (i - 1) + " (" + vals[i - 1] + ") is greater than element at " + i + " (" + vals[i]
|
||||
+ ")");
|
||||
System.lineSeparator() + "element at " + (i - 1) + " (" + vals[i - 1] + ") is greater than element at " + i
|
||||
+ " (" + vals[i] + ")");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -91,7 +91,7 @@ public abstract class AbstractLayerTest {
|
|||
for (int zoom = feature.getMinZoom(); zoom <= feature.getMaxZoom(); zoom++) {
|
||||
Map<String, Object> map = TestUtils.toMap(feature, zoom);
|
||||
if (zooms[zoom] != null) {
|
||||
fail("Multiple features at z" + zoom + ":\n" + zooms[zoom] + "\n" + map);
|
||||
fail("Multiple features at z" + zoom + ":" + System.lineSeparator() + zooms[zoom] + "\n" + map);
|
||||
}
|
||||
zooms[zoom] = map;
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
|
@ -1,9 +1,9 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newLineString;
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTile;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTile;
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import com.onthegomap.flatmap.reader.SimpleFeature;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newPoint;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
||||
import java.util.List;
|
|
@ -1,12 +1,12 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newPoint;
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.layers.Place.getSortKey;
|
||||
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_MAX;
|
||||
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_MIN;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.layers.Place.getSortKey;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newLineString;
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
|
||||
import com.onthegomap.flatmap.FeatureCollector;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
|
@ -1,10 +1,10 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newLineString;
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.LAKE_CENTERLINE_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.LAKE_CENTERLINE_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
|
||||
import com.onthegomap.flatmap.TestUtils;
|
||||
import com.onthegomap.flatmap.geo.GeoUtils;
|
|
@ -1,9 +1,9 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.rectangle;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.WATER_POLYGON_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.WATER_POLYGON_SOURCE;
|
||||
|
||||
import com.onthegomap.flatmap.reader.SimpleFeature;
|
||||
import java.util.List;
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.layers;
|
||||
package com.onthegomap.flatmap.basemap.layers;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.newLineString;
|
||||
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
|
||||
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTile;
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.openmaptiles.util;
|
||||
package com.onthegomap.flatmap.basemap.util;
|
||||
|
||||
import static com.onthegomap.flatmap.TestUtils.assertSubmap;
|
||||
import static com.onthegomap.flatmap.openmaptiles.util.LanguageUtils.containsOnlyLatinCharacters;
|
||||
import static com.onthegomap.flatmap.basemap.util.LanguageUtils.containsOnlyLatinCharacters;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
@ -0,0 +1,62 @@
|
|||
package com.onthegomap.flatmap.basemap.util;
|
||||
|
||||
import static com.onthegomap.flatmap.geo.GeoUtils.point;
|
||||
import static com.onthegomap.flatmap.util.Gzip.gzip;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import com.onthegomap.flatmap.mbtiles.Mbtiles;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class VerifyMonacoTest {
|
||||
|
||||
private Mbtiles mbtiles;
|
||||
|
||||
@BeforeEach
|
||||
public void setup() {
|
||||
mbtiles = Mbtiles.newInMemoryDatabase();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void teardown() throws IOException {
|
||||
mbtiles.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptyFileInvalid() {
|
||||
assertInvalid(mbtiles);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptyTablesInvalid() {
|
||||
mbtiles.createTables().addTileIndex();
|
||||
assertInvalid(mbtiles);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStilInvalidWithOneTile() throws IOException {
|
||||
mbtiles.createTables().addTileIndex();
|
||||
mbtiles.metadata().setName("name");
|
||||
try (var writer = mbtiles.newBatchedTileWriter()) {
|
||||
VectorTile tile = new VectorTile();
|
||||
tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
|
||||
"layer",
|
||||
1,
|
||||
VectorTile.encodeGeometry(point(0, 0)),
|
||||
Map.of()
|
||||
)));
|
||||
writer.write(TileCoord.ofXYZ(0, 0, 0), gzip(tile.encode()));
|
||||
}
|
||||
assertInvalid(mbtiles);
|
||||
}
|
||||
|
||||
private void assertInvalid(Mbtiles mbtiles) {
|
||||
assertTrue(VerifyMonaco.verify(mbtiles).numErrors() > 0);
|
||||
}
|
||||
}
|
|
@ -7,20 +7,20 @@
|
|||
<artifactId>flatmap-benchmarks</artifactId>
|
||||
|
||||
<parent>
|
||||
<groupId>com.onthegomap</groupId>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-parent</artifactId>
|
||||
<version>0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.onthegomap</groupId>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-core</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.onthegomap</groupId>
|
||||
<artifactId>flatmap-openmaptiles</artifactId>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-basemap</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
@ -31,10 +31,10 @@
|
|||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
<!-- for fatjar assembly descriptor -->
|
||||
<!-- for with-deps assembly descriptor -->
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.onthegomap</groupId>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-core</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
</dependency>
|
||||
|
@ -47,20 +47,17 @@
|
|||
</manifestEntries>
|
||||
</archive>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>fatjar</descriptorRef>
|
||||
<descriptorRef>with-deps</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<configuration>
|
||||
<!-- we don't want to deploy this module -->
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
|
|
@ -4,9 +4,9 @@ import com.graphhopper.reader.ReaderElementUtils;
|
|||
import com.graphhopper.reader.ReaderNode;
|
||||
import com.graphhopper.reader.ReaderRelation;
|
||||
import com.graphhopper.reader.ReaderWay;
|
||||
import com.onthegomap.flatmap.basemap.BasemapProfile;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.expression.MultiExpression;
|
||||
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmInputFile;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
|
@ -23,10 +23,10 @@ import org.locationtech.jts.geom.Geometry;
|
|||
* Performance tests for {@link MultiExpression}. Times how long a sample of elements from an OSM input file take to
|
||||
* match.
|
||||
*/
|
||||
public class OpenMapTilesMapping {
|
||||
public class BasemapMapping {
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
var profile = new OpenMapTilesProfile(Translations.nullProvider(List.of()), FlatmapConfig.defaults(),
|
||||
var profile = new BasemapProfile(Translations.nullProvider(List.of()), FlatmapConfig.defaults(),
|
||||
Stats.inMemory());
|
||||
var random = new Random(0);
|
||||
var input = new OsmInputFile(Path.of("data", "sources", "north-america_us_massachusetts.pbf"));
|
|
@ -6,17 +6,20 @@
|
|||
|
||||
<artifactId>flatmap-core</artifactId>
|
||||
|
||||
<name>Flatmap Core</name>
|
||||
<description>Flatmap Core</description>
|
||||
|
||||
<parent>
|
||||
<groupId>com.onthegomap</groupId>
|
||||
<groupId>com.onthegomap.flatmap</groupId>
|
||||
<artifactId>flatmap-parent</artifactId>
|
||||
<version>0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<graphhopper.version>2.3</graphhopper.version>
|
||||
<geotools.version>25.0</geotools.version>
|
||||
<graphhopper.version>2.4</graphhopper.version>
|
||||
<geotools.version>26.0</geotools.version>
|
||||
<log4j.version>2.14.1</log4j.version>
|
||||
<prometheus.version>0.11.0</prometheus.version>
|
||||
<prometheus.version>0.12.0</prometheus.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
@ -28,7 +31,7 @@
|
|||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>1.18.0</version>
|
||||
<version>1.18.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.geotools</groupId>
|
||||
|
@ -43,17 +46,17 @@
|
|||
<dependency>
|
||||
<groupId>org.xerial</groupId>
|
||||
<artifactId>sqlite-jdbc</artifactId>
|
||||
<version>3.34.0</version>
|
||||
<version>3.36.0.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.msgpack</groupId>
|
||||
<artifactId>msgpack-core</artifactId>
|
||||
<version>0.8.22</version>
|
||||
<version>0.9.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.30</version>
|
||||
<version>1.7.32</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
|
@ -118,7 +121,7 @@
|
|||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>30.1.1-jre</version>
|
||||
<version>31.0.1-jre</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
@ -134,7 +137,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.1.2</version>
|
||||
<version>3.2.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
|
@ -142,8 +145,30 @@
|
|||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<!-- reduce the size of the sources jar a bit (leave behind monaco.osm.pbf file) -->
|
||||
<excludes>
|
||||
<exclude>*.wkb</exclude>
|
||||
<exclude>*.mbtiles</exclude>
|
||||
<exclude>*.sqlite</exclude>
|
||||
<exclude>*.zip</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>io.github.zlika</groupId>
|
||||
<artifactId>reproducible-build-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
|
|
@ -5,7 +5,7 @@ import com.onthegomap.flatmap.collection.LongLongMap;
|
|||
import com.onthegomap.flatmap.config.Arguments;
|
||||
import com.onthegomap.flatmap.config.FlatmapConfig;
|
||||
import com.onthegomap.flatmap.config.MbtilesMetadata;
|
||||
import com.onthegomap.flatmap.mbiles.MbtilesWriter;
|
||||
import com.onthegomap.flatmap.mbtiles.MbtilesWriter;
|
||||
import com.onthegomap.flatmap.reader.NaturalEarthReader;
|
||||
import com.onthegomap.flatmap.reader.ShapefileReader;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmInputFile;
|
||||
|
@ -53,7 +53,7 @@ public class FlatmapRunner {
|
|||
private static final Logger LOGGER = LoggerFactory.getLogger(FlatmapRunner.class);
|
||||
private final List<Stage> stages = new ArrayList<>();
|
||||
private final List<ToDownload> toDownload = new ArrayList<>();
|
||||
private final List<Path> inputPaths = new ArrayList<>();
|
||||
private final List<InputPath> inputPaths = new ArrayList<>();
|
||||
private final Timers.Finishable overallTimer;
|
||||
private final Arguments arguments;
|
||||
private final Stats stats;
|
||||
|
@ -135,7 +135,9 @@ public class FlatmapRunner {
|
|||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set. As a shortcut, can use "geofabrik:monaco" or
|
||||
* "geofabrik:australia" shorthand to find an extract by name from <a
|
||||
* href="https://download.geofabrik.de/">Geofabrik download site</a>.
|
||||
* href="https://download.geofabrik.de/">Geofabrik download site</a> or "aws:latest" to download
|
||||
* the latest {@code planet.osm.pbf} file from <a href="https://registry.opendata.aws/osm/">AWS
|
||||
* Open Data Registry</a>.
|
||||
* @return this runner instance for chaining
|
||||
* @see OsmInputFile
|
||||
* @see OsmReader
|
||||
|
@ -333,7 +335,6 @@ public class FlatmapRunner {
|
|||
* As long as {@code use_wikidata} is not set to false, then previously-downloaded wikidata translations will be
|
||||
* loaded from the cache file so you can run with {@code fetch_wikidata=true} once, then without it each subsequent
|
||||
* run to only download translations once.
|
||||
* </ul>
|
||||
*
|
||||
* @param defaultWikidataCache Path to store downloaded wikidata name translations to, and to read them from on
|
||||
* subsequent runs. Overridden by {@code wikidata_cache} argument value.
|
||||
|
@ -439,12 +440,14 @@ public class FlatmapRunner {
|
|||
ran = true;
|
||||
MbtilesMetadata mbtilesMetadata = new MbtilesMetadata(profile, config.arguments());
|
||||
|
||||
if (onlyDownloadSources) {
|
||||
if (arguments.getBoolean("help", "show arguments then exit", false)) {
|
||||
System.exit(0);
|
||||
} else if (onlyDownloadSources) {
|
||||
// don't check files if not generating map
|
||||
} else if (overwrite || config.forceOverwrite()) {
|
||||
FileUtils.deleteFile(output);
|
||||
} else if (Files.exists(output)) {
|
||||
throw new IllegalArgumentException(output + " already exists, use force to overwrite.");
|
||||
throw new IllegalArgumentException(output + " already exists, use the --force argument to overwrite.");
|
||||
}
|
||||
|
||||
LOGGER.info(
|
||||
|
@ -556,24 +559,26 @@ public class FlatmapRunner {
|
|||
toDownload.add(new ToDownload(name, url, path));
|
||||
}
|
||||
}
|
||||
inputPaths.add(path);
|
||||
inputPaths.add(new InputPath(name, path));
|
||||
return path;
|
||||
}
|
||||
|
||||
private void download() {
|
||||
var timer = stats.startStage("download");
|
||||
Downloader downloader = Downloader.create(config());
|
||||
Downloader downloader = Downloader.create(config(), stats());
|
||||
for (ToDownload toDownload : toDownload) {
|
||||
downloader.add(toDownload.id, toDownload.url, toDownload.path);
|
||||
if (profile.caresAboutSource(toDownload.id)) {
|
||||
downloader.add(toDownload.id, toDownload.url, toDownload.path);
|
||||
}
|
||||
}
|
||||
downloader.run();
|
||||
timer.stop();
|
||||
}
|
||||
|
||||
private void ensureInputFilesExist() {
|
||||
for (Path path : inputPaths) {
|
||||
if (!Files.exists(path)) {
|
||||
throw new IllegalArgumentException(path + " does not exist");
|
||||
for (InputPath inputPath : inputPaths) {
|
||||
if (profile.caresAboutSource(inputPath.id) && !Files.exists(inputPath.path)) {
|
||||
throw new IllegalArgumentException(inputPath.path + " does not exist");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -586,4 +591,6 @@ public class FlatmapRunner {
|
|||
}
|
||||
|
||||
private static record ToDownload(String id, String url, Path path) {}
|
||||
|
||||
private static record InputPath(String id, Path path) {}
|
||||
}
|
||||
|
|
|
@ -90,7 +90,7 @@ public abstract class ForwardingProfile implements Profile {
|
|||
if (handlers != null) {
|
||||
for (var handler : handlers) {
|
||||
handler.processFeature(sourceFeature, features);
|
||||
// TODO extract common handling for expression-based filtering from openmaptiles to this
|
||||
// TODO extract common handling for expression-based filtering from basemap to this
|
||||
// common profile when we have another use-case for it.
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap;
|
||||
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.mbiles.Mbtiles;
|
||||
import com.onthegomap.flatmap.mbtiles.Mbtiles;
|
||||
import com.onthegomap.flatmap.reader.SourceFeature;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmElement;
|
||||
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
|
||||
|
@ -37,11 +37,12 @@ public interface Profile {
|
|||
* passed along to {@link #processFeature(SourceFeature, FeatureCollector)} for any OSM element in that relation.
|
||||
* <p>
|
||||
* The result of this method is stored in memory.
|
||||
* <p>
|
||||
* The default implementation returns {@code null} to ignore all relations
|
||||
*
|
||||
* @param relation the OSM relation
|
||||
* @return a list of relation info instances with information extracted from the relation to pass to {@link
|
||||
* #processFeature(SourceFeature, FeatureCollector)}, or {@code null} to ignore.
|
||||
* @implNote The default implementation returns {@code null} to ignore all relations
|
||||
*/
|
||||
default List<OsmRelationInfo> preprocessOsmRelation(OsmElement.Relation relation) {
|
||||
return null;
|
||||
|
@ -72,6 +73,8 @@ public interface Profile {
|
|||
* linestrings/polygons.
|
||||
* <p>
|
||||
* Many threads invoke this method concurrently so ensure thread-safe access to any shared data structures.
|
||||
* <p>
|
||||
* The default implementation passes through input features unaltered
|
||||
*
|
||||
* @param layer the output layer name
|
||||
* @param zoom zoom level of the tile
|
||||
|
@ -80,10 +83,9 @@ public interface Profile {
|
|||
* {@code null} if they should be ignored.
|
||||
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
|
||||
* the original input features, and continue processing other tiles
|
||||
* @implSpec The default implementation passes through input features unaltered
|
||||
*/
|
||||
default List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTile.Feature> items) throws GeometryException {
|
||||
List<VectorTile.Feature> items) throws GeometryException {
|
||||
return items;
|
||||
}
|
||||
|
||||
|
@ -126,8 +128,9 @@ public interface Profile {
|
|||
/**
|
||||
* Returns {@code true} to set {@code type="overlay"} in {@link Mbtiles} metadata otherwise sets {@code
|
||||
* type="baselayer"}
|
||||
* <p>
|
||||
* The default implementation sets {@code type="baselayer"}
|
||||
*
|
||||
* @implSpec The default implementation sets {@code type="baselayer"}
|
||||
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#metadata">MBTiles specification</a>
|
||||
*/
|
||||
default boolean isOverlay() {
|
||||
|
@ -136,10 +139,11 @@ public interface Profile {
|
|||
|
||||
/**
|
||||
* Defines whether {@link Wikidata} should fetch wikidata translations for the input element.
|
||||
* <p>
|
||||
* The default implementation returns {@code true} for all elements
|
||||
*
|
||||
* @param elem the input OSM element
|
||||
* @return {@code true} to fetch wikidata translations for {@code elem}, {@code false} to ignore
|
||||
* @implSpec the default implementation returns {@code true} for all elements
|
||||
*/
|
||||
default boolean caresAboutWikidataTranslation(OsmElement elem) {
|
||||
return true;
|
||||
|
@ -153,15 +157,16 @@ public interface Profile {
|
|||
* @param next a consumer to pass finished map features to
|
||||
*/
|
||||
default void finish(String sourceName, FeatureCollector.Factory featureCollectors,
|
||||
Consumer<FeatureCollector.Feature> next) {
|
||||
Consumer<FeatureCollector.Feature> next) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this profile will use any of the elements from an input source.
|
||||
* <p>
|
||||
* The default implementation returns true.
|
||||
*
|
||||
* @param name the input source name
|
||||
* @return {@code true} if this profile uses that source, {@code false} if it is safe to ignore
|
||||
* @implSpec the default implementation returns true
|
||||
*/
|
||||
default boolean caresAboutSource(String name) {
|
||||
return true;
|
||||
|
@ -178,7 +183,7 @@ public interface Profile {
|
|||
|
||||
@Override
|
||||
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
|
||||
List<VectorTile.Feature> items) {
|
||||
List<VectorTile.Feature> items) {
|
||||
return items;
|
||||
}
|
||||
|
||||
|
|
|
@ -75,15 +75,107 @@ public class VectorTile {
|
|||
// TODO make these configurable
|
||||
private static final int EXTENT = 4096;
|
||||
private static final double SIZE = 256d;
|
||||
private static final double SCALE = ((double) EXTENT) / SIZE;
|
||||
private final Map<String, Layer> layers = new LinkedHashMap<>();
|
||||
|
||||
private static int[] getCommands(Geometry input) {
|
||||
var encoder = new CommandEncoder();
|
||||
private static int[] getCommands(Geometry input, int scale) {
|
||||
var encoder = new CommandEncoder(scale);
|
||||
encoder.accept(input);
|
||||
return encoder.result.toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scales a geometry down by a factor of {@code 2^scale} without materializing an intermediate JTS geometry and
|
||||
* returns the encoded result.
|
||||
*/
|
||||
private static int[] unscale(int[] commands, int scale, GeometryType geomType) {
|
||||
IntArrayList result = new IntArrayList();
|
||||
int geometryCount = commands.length;
|
||||
int length = 0;
|
||||
int command = 0;
|
||||
int i = 0;
|
||||
int inX = 0, inY = 0;
|
||||
int outX = 0, outY = 0;
|
||||
int startX = 0, startY = 0;
|
||||
double scaleFactor = Math.pow(2, -scale);
|
||||
int lengthIdx = 0;
|
||||
int moveToIdx = 0;
|
||||
int pointsInShape = 0;
|
||||
boolean first = true;
|
||||
while (i < geometryCount) {
|
||||
if (length <= 0) {
|
||||
length = commands[i++];
|
||||
lengthIdx = result.size();
|
||||
result.add(length);
|
||||
command = length & ((1 << 3) - 1);
|
||||
length = length >> 3;
|
||||
}
|
||||
|
||||
if (length > 0) {
|
||||
if (command == Command.MOVE_TO.value) {
|
||||
// degenerate geometry, remove it from output entirely
|
||||
if (!first && pointsInShape < geomType.minPoints()) {
|
||||
int prevCommand = result.get(lengthIdx);
|
||||
result.elementsCount = moveToIdx;
|
||||
result.add(prevCommand);
|
||||
// reset deltas
|
||||
outX = startX;
|
||||
outY = startY;
|
||||
}
|
||||
// keep track of size of next shape...
|
||||
pointsInShape = 0;
|
||||
startX = outX;
|
||||
startY = outY;
|
||||
moveToIdx = result.size() - 1;
|
||||
}
|
||||
first = false;
|
||||
if (command == Command.CLOSE_PATH.value) {
|
||||
pointsInShape++;
|
||||
length--;
|
||||
continue;
|
||||
}
|
||||
|
||||
int dx = commands[i++];
|
||||
int dy = commands[i++];
|
||||
|
||||
length--;
|
||||
|
||||
dx = zigZagDecode(dx);
|
||||
dy = zigZagDecode(dy);
|
||||
|
||||
inX = inX + dx;
|
||||
inY = inY + dy;
|
||||
|
||||
int nextX = (int) Math.round(inX * scaleFactor);
|
||||
int nextY = (int) Math.round(inY * scaleFactor);
|
||||
|
||||
if (nextX == outX && nextY == outY && command == Command.LINE_TO.value) {
|
||||
int commandLength = result.get(lengthIdx) - 8;
|
||||
if (commandLength < 8) {
|
||||
// get rid of lineto section if empty
|
||||
result.elementsCount = lengthIdx;
|
||||
} else {
|
||||
result.set(lengthIdx, commandLength);
|
||||
}
|
||||
} else {
|
||||
pointsInShape++;
|
||||
int dxOut = nextX - outX;
|
||||
int dyOut = nextY - outY;
|
||||
result.add(
|
||||
zigZagEncode(dxOut),
|
||||
zigZagEncode(dyOut)
|
||||
);
|
||||
outX = nextX;
|
||||
outY = nextY;
|
||||
}
|
||||
}
|
||||
}
|
||||
// degenerate geometry, remove it from output entirely
|
||||
if (pointsInShape < geomType.minPoints()) {
|
||||
result.elementsCount = moveToIdx;
|
||||
}
|
||||
return result.toArray();
|
||||
}
|
||||
|
||||
private static int zigZagEncode(int n) {
|
||||
// https://developers.google.com/protocol-buffers/docs/encoding#types
|
||||
return (n << 1) ^ (n >> 31);
|
||||
|
@ -94,9 +186,10 @@ public class VectorTile {
|
|||
return ((n >> 1) ^ (-(n & 1)));
|
||||
}
|
||||
|
||||
private static Geometry decodeCommands(GeometryType geomType, int[] commands) throws GeometryException {
|
||||
private static Geometry decodeCommands(GeometryType geomType, int[] commands, int scale) throws GeometryException {
|
||||
try {
|
||||
GeometryFactory gf = GeoUtils.JTS_FACTORY;
|
||||
double SCALE = (EXTENT << scale) / SIZE;
|
||||
int x = 0;
|
||||
int y = 0;
|
||||
|
||||
|
@ -219,7 +312,7 @@ public class VectorTile {
|
|||
}
|
||||
|
||||
if (geometry == null) {
|
||||
geometry = gf.createGeometryCollection(new Geometry[0]);
|
||||
geometry = GeoUtils.EMPTY_GEOMETRY;
|
||||
}
|
||||
|
||||
return geometry;
|
||||
|
@ -284,7 +377,7 @@ public class VectorTile {
|
|||
features.add(new Feature(
|
||||
layerName,
|
||||
feature.getId(),
|
||||
new VectorGeometry(Ints.toArray(feature.getGeometryList()), GeometryType.valueOf(feature.getType())),
|
||||
new VectorGeometry(Ints.toArray(feature.getGeometryList()), GeometryType.valueOf(feature.getType()), 0),
|
||||
attrs
|
||||
));
|
||||
}
|
||||
|
@ -303,7 +396,11 @@ public class VectorTile {
|
|||
* @return the geometry type and command array for the encoded geometry
|
||||
*/
|
||||
public static VectorGeometry encodeGeometry(Geometry geometry) {
|
||||
return new VectorGeometry(getCommands(geometry), GeometryType.valueOf(geometry));
|
||||
return encodeGeometry(geometry, 0);
|
||||
}
|
||||
|
||||
public static VectorGeometry encodeGeometry(Geometry geometry, int scale) {
|
||||
return new VectorGeometry(getCommands(geometry, scale), GeometryType.valueOf(geometry), scale);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -411,12 +508,28 @@ public class VectorTile {
|
|||
/**
|
||||
* A vector tile encoded as a list of commands according to the <a href="https://github.com/mapbox/vector-tile-spec/tree/master/2.1#43-geometry-encoding">vector
|
||||
* tile specification</a>.
|
||||
* <p>
|
||||
* To encode extra precision in intermediate feature geometries, the geometry contained in {@code commands} is scaled
|
||||
* to a tile extent of {@code EXTENT * 2^scale}, so when the {@code scale == 0} the extent is {@link #EXTENT} and when
|
||||
* {@code scale == 2} the extent is 4x{@link #EXTENT}. Geometries must be scaled back to 0 using {@link #unscale()}
|
||||
* before outputting to mbtiles.
|
||||
*/
|
||||
public static record VectorGeometry(int[] commands, GeometryType geomType) {
|
||||
public static record VectorGeometry(int[] commands, GeometryType geomType, int scale) {
|
||||
|
||||
public VectorGeometry {
|
||||
if (scale < 0) {
|
||||
throw new IllegalArgumentException("scale can not be less than 0, got: " + scale);
|
||||
}
|
||||
}
|
||||
|
||||
/** Converts an encoded geometry back to a JTS geometry. */
|
||||
public Geometry decode() throws GeometryException {
|
||||
return decodeCommands(geomType, commands);
|
||||
return decodeCommands(geomType, commands, scale);
|
||||
}
|
||||
|
||||
/** Returns this encoded geometry, scaled back to 0, so it is safe to emit to mbtiles output. */
|
||||
public VectorGeometry unscale() {
|
||||
return scale == 0 ? this : new VectorGeometry(VectorTile.unscale(commands, scale, geomType), geomType, 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -491,10 +604,17 @@ public class VectorTile {
|
|||
* new geometry.
|
||||
*/
|
||||
public Feature copyWithNewGeometry(Geometry newGeometry) {
|
||||
return copyWithNewGeometry(encodeGeometry(newGeometry));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a copy of this feature with {@code geometry} replaced with {@code newGeometry}.
|
||||
*/
|
||||
public Feature copyWithNewGeometry(VectorGeometry newGeometry) {
|
||||
return new Feature(
|
||||
layer,
|
||||
id,
|
||||
encodeGeometry(newGeometry),
|
||||
newGeometry,
|
||||
attrs,
|
||||
group
|
||||
);
|
||||
|
@ -521,10 +641,15 @@ public class VectorTile {
|
|||
private static class CommandEncoder {
|
||||
|
||||
final IntArrayList result = new IntArrayList();
|
||||
private final double SCALE;
|
||||
// Initial points use absolute locations, then subsequent points in a geometry use offsets so
|
||||
// need to keep track of previous x/y location during the encoding.
|
||||
int x = 0, y = 0;
|
||||
|
||||
CommandEncoder(int scale) {
|
||||
this.SCALE = (EXTENT << scale) / SIZE;
|
||||
}
|
||||
|
||||
static boolean shouldClosePath(Geometry geometry) {
|
||||
return (geometry instanceof Polygon) || (geometry instanceof LinearRing);
|
||||
}
|
||||
|
@ -536,44 +661,47 @@ public class VectorTile {
|
|||
void accept(Geometry geometry) {
|
||||
if (geometry instanceof MultiLineString multiLineString) {
|
||||
for (int i = 0; i < multiLineString.getNumGeometries(); i++) {
|
||||
encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false);
|
||||
encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false, GeometryType.LINE);
|
||||
}
|
||||
} else if (geometry instanceof Polygon polygon) {
|
||||
LineString exteriorRing = polygon.getExteriorRing();
|
||||
encode(exteriorRing.getCoordinateSequence(), true);
|
||||
encode(exteriorRing.getCoordinateSequence(), true, GeometryType.POLYGON);
|
||||
|
||||
for (int i = 0; i < polygon.getNumInteriorRing(); i++) {
|
||||
LineString interiorRing = polygon.getInteriorRingN(i);
|
||||
encode(interiorRing.getCoordinateSequence(), true);
|
||||
encode(interiorRing.getCoordinateSequence(), true, GeometryType.LINE);
|
||||
}
|
||||
} else if (geometry instanceof MultiPolygon multiPolygon) {
|
||||
for (int i = 0; i < multiPolygon.getNumGeometries(); i++) {
|
||||
accept(multiPolygon.getGeometryN(i));
|
||||
}
|
||||
} else if (geometry instanceof LineString lineString) {
|
||||
encode(lineString.getCoordinateSequence(), shouldClosePath(geometry));
|
||||
encode(lineString.getCoordinateSequence(), shouldClosePath(geometry), GeometryType.LINE);
|
||||
} else if (geometry instanceof Point point) {
|
||||
encode(point.getCoordinateSequence(), false);
|
||||
encode(point.getCoordinateSequence(), false, GeometryType.POINT);
|
||||
} else if (geometry instanceof Puntal) {
|
||||
encode(new CoordinateArraySequence(geometry.getCoordinates()), shouldClosePath(geometry),
|
||||
geometry instanceof MultiPoint);
|
||||
geometry instanceof MultiPoint, GeometryType.POINT);
|
||||
} else {
|
||||
LOGGER.warn("Unrecognized geometry type: " + geometry.getGeometryType());
|
||||
}
|
||||
}
|
||||
|
||||
void encode(CoordinateSequence cs, boolean closePathAtEnd) {
|
||||
encode(cs, closePathAtEnd, false);
|
||||
void encode(CoordinateSequence cs, boolean closePathAtEnd, GeometryType geomType) {
|
||||
encode(cs, closePathAtEnd, false, geomType);
|
||||
}
|
||||
|
||||
void encode(CoordinateSequence cs, boolean closePathAtEnd, boolean multiPoint) {
|
||||
|
||||
void encode(CoordinateSequence cs, boolean closePathAtEnd, boolean multiPoint, GeometryType geomType) {
|
||||
if (cs.size() == 0) {
|
||||
throw new IllegalArgumentException("empty geometry");
|
||||
}
|
||||
|
||||
int startIdx = result.size();
|
||||
int numPoints = 0;
|
||||
int lineToIndex = 0;
|
||||
int lineToLength = 0;
|
||||
int startX = x;
|
||||
int startY = y;
|
||||
|
||||
for (int i = 0; i < cs.size(); i++) {
|
||||
|
||||
|
@ -588,7 +716,7 @@ public class VectorTile {
|
|||
int _y = (int) Math.round(cy * SCALE);
|
||||
|
||||
// prevent point equal to the previous
|
||||
if (i > 0 && _x == x && _y == y) {
|
||||
if (i > 0 && _x == x && _y == y && !multiPoint) {
|
||||
lineToLength--;
|
||||
continue;
|
||||
}
|
||||
|
@ -602,6 +730,7 @@ public class VectorTile {
|
|||
// delta, then zigzag
|
||||
result.add(zigZagEncode(_x - x));
|
||||
result.add(zigZagEncode(_y - y));
|
||||
numPoints++;
|
||||
|
||||
x = _x;
|
||||
y = _y;
|
||||
|
@ -628,6 +757,15 @@ public class VectorTile {
|
|||
|
||||
if (closePathAtEnd) {
|
||||
result.add(commandAndLength(Command.CLOSE_PATH, 1));
|
||||
numPoints++;
|
||||
}
|
||||
|
||||
// degenerate geometry, skip emitting
|
||||
if (numPoints < geomType.minPoints()) {
|
||||
result.elementsCount = startIdx;
|
||||
// reset deltas
|
||||
x = startX;
|
||||
y = startY;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,18 +4,25 @@ import com.onthegomap.flatmap.util.FileUtils;
|
|||
import java.io.BufferedOutputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.MappedByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Arrays;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* An array of primitives backed by memory-mapped file.
|
||||
*/
|
||||
abstract class AppendStoreMmap implements AppendStore {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AppendStoreMmap.class);
|
||||
|
||||
// writes are done using a BufferedOutputStream
|
||||
final DataOutputStream outputStream;
|
||||
final int segmentBits;
|
||||
|
@ -79,6 +86,30 @@ abstract class AppendStoreMmap implements AppendStore {
|
|||
channel.close();
|
||||
}
|
||||
if (segments != null) {
|
||||
try {
|
||||
// attempt to force-unmap the file, so we can delete it later
|
||||
// https://stackoverflow.com/questions/2972986/how-to-unmap-a-file-from-memory-mapped-using-filechannel-in-java
|
||||
Class<?> unsafeClass;
|
||||
try {
|
||||
unsafeClass = Class.forName("sun.misc.Unsafe");
|
||||
} catch (Exception ex) {
|
||||
unsafeClass = Class.forName("jdk.internal.misc.Unsafe");
|
||||
}
|
||||
Method clean = unsafeClass.getMethod("invokeCleaner", ByteBuffer.class);
|
||||
clean.setAccessible(true);
|
||||
Field theUnsafeField = unsafeClass.getDeclaredField("theUnsafe");
|
||||
theUnsafeField.setAccessible(true);
|
||||
Object theUnsafe = theUnsafeField.get(null);
|
||||
for (int i = 0; i < segments.length; i++) {
|
||||
var buffer = segments[i];
|
||||
if (buffer != null) {
|
||||
clean.invoke(theUnsafe, buffer);
|
||||
segments[i] = null;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.info("Unable to unmap " + path + " " + e);
|
||||
}
|
||||
Arrays.fill(segments, null);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,6 +22,7 @@ import java.util.Arrays;
|
|||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.zip.Deflater;
|
||||
|
@ -54,6 +55,8 @@ class ExternalMergeSort implements FeatureSort {
|
|||
private final List<Chunk> chunks = new ArrayList<>();
|
||||
private final boolean gzip;
|
||||
private final FlatmapConfig config;
|
||||
private final int readerLimit;
|
||||
private final int writerLimit;
|
||||
private Chunk currentChunk;
|
||||
private volatile boolean sorted = false;
|
||||
|
||||
|
@ -83,6 +86,10 @@ class ExternalMergeSort implements FeatureSort {
|
|||
"Not enough memory to use chunk size " + chunkSizeLimit + " only have " + memory);
|
||||
}
|
||||
this.workers = workers;
|
||||
this.readerLimit = Math.max(1, config.arguments()
|
||||
.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks", 6));
|
||||
this.writerLimit = Math.max(1, config.arguments()
|
||||
.getInteger("sort_max_writers", "maximum number of concurrent write threads to use when sorting chunks", 6));
|
||||
LOGGER.info("Using merge sort feature map, chunk size=" + (chunkSizeLimit / 1_000_000) + "mb workers=" + workers);
|
||||
try {
|
||||
FileUtils.deleteDirectory(dir);
|
||||
|
@ -153,6 +160,8 @@ class ExternalMergeSort implements FeatureSort {
|
|||
}
|
||||
}
|
||||
var timer = stats.startStage("sort");
|
||||
Semaphore readSemaphore = new Semaphore(readerLimit);
|
||||
Semaphore writeSemaphore = new Semaphore(writerLimit);
|
||||
AtomicLong reading = new AtomicLong(0);
|
||||
AtomicLong writing = new AtomicLong(0);
|
||||
AtomicLong sorting = new AtomicLong(0);
|
||||
|
@ -161,10 +170,21 @@ class ExternalMergeSort implements FeatureSort {
|
|||
var pipeline = WorkerPipeline.start("sort", stats)
|
||||
.readFromTiny("item_queue", chunks)
|
||||
.sinkToConsumer("worker", workers, chunk -> {
|
||||
var toSort = time(reading, chunk::readAll);
|
||||
time(sorting, toSort::sort);
|
||||
time(writing, toSort::flush);
|
||||
doneCounter.incrementAndGet();
|
||||
try {
|
||||
readSemaphore.acquire();
|
||||
var toSort = time(reading, chunk::readAll);
|
||||
readSemaphore.release();
|
||||
|
||||
time(sorting, toSort::sort);
|
||||
|
||||
writeSemaphore.acquire();
|
||||
time(writing, toSort::flush);
|
||||
writeSemaphore.release();
|
||||
|
||||
doneCounter.incrementAndGet();
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
ProgressLoggers loggers = ProgressLoggers.create()
|
||||
|
|
|
@ -190,7 +190,7 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
|
|||
packer.packInt(group.limit());
|
||||
}
|
||||
packer.packLong(vectorTileFeature.id());
|
||||
packer.packByte(vectorTileFeature.geometry().geomType().asByte());
|
||||
packer.packByte(encodeGeomTypeAndScale(vectorTileFeature.geometry()));
|
||||
var attrs = vectorTileFeature.attrs();
|
||||
packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count());
|
||||
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
|
||||
|
@ -238,7 +238,9 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
|
|||
group = VectorTile.Feature.NO_GROUP;
|
||||
}
|
||||
long id = unpacker.unpackLong();
|
||||
byte geomType = unpacker.unpackByte();
|
||||
byte geomTypeAndScale = unpacker.unpackByte();
|
||||
GeometryType geomType = decodeGeomType(geomTypeAndScale);
|
||||
int scale = decodeScale(geomTypeAndScale);
|
||||
int mapSize = unpacker.unpackMapHeader();
|
||||
Map<String, Object> attrs = new HashMap<>(mapSize);
|
||||
for (int i = 0; i < mapSize; i++) {
|
||||
|
@ -263,7 +265,7 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
|
|||
return new VectorTile.Feature(
|
||||
layer,
|
||||
id,
|
||||
new VectorTile.VectorGeometry(commands, GeometryType.valueOf(geomType)),
|
||||
new VectorTile.VectorGeometry(commands, geomType, scale),
|
||||
attrs,
|
||||
group
|
||||
);
|
||||
|
@ -272,6 +274,20 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
|
|||
}
|
||||
}
|
||||
|
||||
static GeometryType decodeGeomType(byte geomTypeAndScale) {
|
||||
return GeometryType.valueOf((byte) (geomTypeAndScale & 0b111));
|
||||
}
|
||||
|
||||
static int decodeScale(byte geomTypeAndScale) {
|
||||
return (geomTypeAndScale & 0xff) >>> 3;
|
||||
}
|
||||
|
||||
static byte encodeGeomTypeAndScale(VectorTile.VectorGeometry geometry) {
|
||||
assert geometry.geomType().asByte() >= 0 && geometry.geomType().asByte() <= 8;
|
||||
assert geometry.scale() >= 0 && geometry.scale() < (1 << 5);
|
||||
return (byte) (geometry.geomType().asByte() | (geometry.scale() << 3));
|
||||
}
|
||||
|
||||
/** Writes a serialized binary feature to intermediate storage. */
|
||||
@Override
|
||||
public void accept(SortableFeature entry) {
|
||||
|
@ -412,12 +428,28 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
|
|||
return encoder;
|
||||
}
|
||||
|
||||
private static void unscale(List<VectorTile.Feature> features) {
|
||||
for (int i = 0; i < features.size(); i++) {
|
||||
var feature = features.get(i);
|
||||
if (feature != null) {
|
||||
VectorTile.VectorGeometry geometry = feature.geometry();
|
||||
if (geometry.scale() != 0) {
|
||||
features.set(i, feature.copyWithNewGeometry(geometry.unscale()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
|
||||
List<VectorTile.Feature> features) {
|
||||
try {
|
||||
List<VectorTile.Feature> postProcessed = profile
|
||||
.postProcessLayerFeatures(layer, tileCoord.z(), features);
|
||||
features = postProcessed == null ? features : postProcessed;
|
||||
// lines are stored using a higher precision so that rounding does not
|
||||
// introduce artificial intersections between endpoints to confuse line merging,
|
||||
// so we have to reduce the precision here, now that line merging is done.
|
||||
unscale(features);
|
||||
} catch (Throwable e) {
|
||||
// failures in tile post-processing happen very late so err on the side of caution and
|
||||
// log failures, only throwing when it's a fatal error
|
||||
|
|
|
@ -177,7 +177,7 @@ public class Arguments {
|
|||
public Envelope bounds(String key, String description) {
|
||||
String input = getArg(key);
|
||||
Envelope result = null;
|
||||
if ("world".equalsIgnoreCase(input)) {
|
||||
if ("world".equalsIgnoreCase(input) || "planet".equalsIgnoreCase(input)) {
|
||||
result = GeoUtils.WORLD_LAT_LON_BOUNDS;
|
||||
} else if (input != null) {
|
||||
double[] bounds = Stream.of(input.split("[\\s,]+")).mapToDouble(Double::parseDouble).toArray();
|
||||
|
@ -306,4 +306,16 @@ public class Arguments {
|
|||
logArgValue(key, description, parsed.get(ChronoUnit.SECONDS) + " seconds");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an argument as long.
|
||||
*
|
||||
* @throws NumberFormatException if the argument cannot be parsed as an long
|
||||
*/
|
||||
public long getLong(String key, String description, long defaultValue) {
|
||||
String value = getArg(key, Long.toString(defaultValue));
|
||||
long parsed = Long.parseLong(value);
|
||||
logArgValue(key, description, parsed);
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,6 +20,9 @@ public record FlatmapConfig(
|
|||
String nodeMapType,
|
||||
String nodeMapStorage,
|
||||
String httpUserAgent,
|
||||
Duration httpTimeout,
|
||||
long downloadChunkSizeMB,
|
||||
int downloadThreads,
|
||||
double minFeatureSizeAtMaxZoom,
|
||||
double minFeatureSizeBelowMaxZoom,
|
||||
double simplifyToleranceAtMaxZoom,
|
||||
|
@ -63,6 +66,9 @@ public record FlatmapConfig(
|
|||
arguments.getString("nodemap_storage", "storage for location map: mmap or ram", "mmap"),
|
||||
arguments.getString("http_user_agent", "User-Agent header to set when downloading files over HTTP",
|
||||
"Flatmap downloader (https://github.com/onthegomap/flatmap)"),
|
||||
arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
|
||||
arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
|
||||
arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
|
||||
arguments.getDouble("min_feature_size_at_max_zoom",
|
||||
"Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming",
|
||||
256d / 4096),
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.config;
|
||||
|
||||
import com.onthegomap.flatmap.Profile;
|
||||
import com.onthegomap.flatmap.mbiles.MbtilesWriter;
|
||||
import com.onthegomap.flatmap.mbtiles.MbtilesWriter;
|
||||
|
||||
/** Controls information that {@link MbtilesWriter} will write to the mbtiles metadata table. */
|
||||
public record MbtilesMetadata(
|
||||
|
|
|
@ -18,7 +18,7 @@ import java.util.stream.Stream;
|
|||
* Calling {@code toString()} on any expression will generate code that can be used to recreate an identical copy of the
|
||||
* original expression, assuming that the generated code includes:
|
||||
* <pre>{@code
|
||||
* import static com.onthegomap.flatmap.openmaptiles.expression.Expression.*;
|
||||
* import static com.onthegomap.flatmap.expression.Expression.*;
|
||||
* }</pre>
|
||||
*/
|
||||
public interface Expression {
|
||||
|
|
|
@ -7,15 +7,17 @@ import org.locationtech.jts.geom.Puntal;
|
|||
import vector_tile.VectorTileProto;
|
||||
|
||||
public enum GeometryType {
|
||||
UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN),
|
||||
POINT(VectorTileProto.Tile.GeomType.POINT),
|
||||
LINE(VectorTileProto.Tile.GeomType.LINESTRING),
|
||||
POLYGON(VectorTileProto.Tile.GeomType.POLYGON);
|
||||
UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0),
|
||||
POINT(VectorTileProto.Tile.GeomType.POINT, 1),
|
||||
LINE(VectorTileProto.Tile.GeomType.LINESTRING, 2),
|
||||
POLYGON(VectorTileProto.Tile.GeomType.POLYGON, 4);
|
||||
|
||||
private final VectorTileProto.Tile.GeomType protobufType;
|
||||
private int minPoints;
|
||||
|
||||
GeometryType(VectorTileProto.Tile.GeomType protobufType) {
|
||||
GeometryType(VectorTileProto.Tile.GeomType protobufType, int minPoints) {
|
||||
this.protobufType = protobufType;
|
||||
this.minPoints = minPoints;
|
||||
}
|
||||
|
||||
public static GeometryType valueOf(Geometry geom) {
|
||||
|
@ -45,4 +47,8 @@ public enum GeometryType {
|
|||
public VectorTileProto.Tile.GeomType asProtobufType() {
|
||||
return protobufType;
|
||||
}
|
||||
|
||||
public int minPoints() {
|
||||
return minPoints;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package com.onthegomap.flatmap.geo;
|
||||
|
||||
import com.onthegomap.flatmap.mbiles.Mbtiles;
|
||||
import java.text.NumberFormat;
|
||||
import com.onthegomap.flatmap.mbtiles.Mbtiles;
|
||||
import com.onthegomap.flatmap.util.Format;
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
|
@ -9,7 +9,7 @@ import org.locationtech.jts.geom.CoordinateXY;
|
|||
/**
|
||||
* The coordinate of a <a href="https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames">slippy map tile</a>.
|
||||
* <p>
|
||||
* In order to encode into a 32-bit integer, only zoom levels <= 14 are supported since we need 4 bits for the
|
||||
* In order to encode into a 32-bit integer, only zoom levels {@code <= 14} are supported since we need 4 bits for the
|
||||
* zoom-level, and 14 bits each for the x/y coordinates.
|
||||
* <p>
|
||||
* Tiles are ordered by z ascending, x ascending, y descending to match index ordering of {@link Mbtiles} sqlite
|
||||
|
@ -19,7 +19,7 @@ import org.locationtech.jts.geom.CoordinateXY;
|
|||
* @param x x coordinate of the tile where 0 is the western-most tile just to the east the international date line
|
||||
* and 2^z-1 is the eastern-most tile
|
||||
* @param y y coordinate of the tile where 0 is the northern-most tile and 2^z-1 is the southern-most tile
|
||||
* @param z zoom level (<= 14)
|
||||
* @param z zoom level ({@code <= 14})
|
||||
*/
|
||||
@Immutable
|
||||
public record TileCoord(int encoded, int x, int y, int z) implements Comparable<TileCoord> {
|
||||
|
@ -29,11 +29,6 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
|
|||
// also need to remove hardcoded z14 limits
|
||||
|
||||
private static final int XY_MASK = (1 << 14) - 1;
|
||||
private static final NumberFormat format = NumberFormat.getNumberInstance();
|
||||
|
||||
static {
|
||||
format.setMaximumFractionDigits(5);
|
||||
}
|
||||
|
||||
public TileCoord {
|
||||
assert z <= 14;
|
||||
|
@ -126,7 +121,7 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
|
|||
/** Returns a URL that displays the openstreetmap data for this tile. */
|
||||
public String getDebugUrl() {
|
||||
Coordinate coord = getLatLon();
|
||||
return "https://www.openstreetmap.org/#map=" + z + "/" + format.format(coord.y) + "/" + format.format(coord.x);
|
||||
return Format.osmDebugUrl(z, coord);
|
||||
}
|
||||
|
||||
/** Returns the pixel coordinate on this tile of a given latitude/longitude (assuming 256x256 px tiles). */
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.flatmap.mbiles;
|
||||
package com.onthegomap.flatmap.mbtiles;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
|
||||
|
||||
|
@ -539,7 +539,8 @@ public final class Mbtiles implements Closeable {
|
|||
);
|
||||
}
|
||||
} catch (SQLException throwables) {
|
||||
LOGGER.warn("Error retrieving metadata", throwables);
|
||||
LOGGER.warn("Error retrieving metadata: " + throwables);
|
||||
LOGGER.trace("Error retrieving metadata details: ", throwables);
|
||||
}
|
||||
return result;
|
||||
}
|
|
@ -1,4 +1,6 @@
|
|||
package com.onthegomap.flatmap.mbiles;
|
||||
package com.onthegomap.flatmap.mbtiles;
|
||||
|
||||
import static com.onthegomap.flatmap.util.Gzip.gzip;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.collection.FeatureGroup;
|
||||
|
@ -6,6 +8,7 @@ import com.onthegomap.flatmap.config.FlatmapConfig;
|
|||
import com.onthegomap.flatmap.config.MbtilesMetadata;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import com.onthegomap.flatmap.stats.Counter;
|
||||
import com.onthegomap.flatmap.stats.ProcessInfo;
|
||||
import com.onthegomap.flatmap.stats.ProgressLoggers;
|
||||
import com.onthegomap.flatmap.stats.Stats;
|
||||
import com.onthegomap.flatmap.stats.Timer;
|
||||
|
@ -15,7 +18,6 @@ import com.onthegomap.flatmap.util.Format;
|
|||
import com.onthegomap.flatmap.util.LayerStats;
|
||||
import com.onthegomap.flatmap.worker.WorkQueue;
|
||||
import com.onthegomap.flatmap.worker.WorkerPipeline;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayDeque;
|
||||
|
@ -32,7 +34,6 @@ import java.util.function.Consumer;
|
|||
import java.util.function.Supplier;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
@ -56,8 +57,6 @@ public class MbtilesWriter {
|
|||
private final LongAccumulator[] maxTileSizesByZoom;
|
||||
private final FeatureGroup features;
|
||||
private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
|
||||
private final LongAccumulator maxBatchLength = new LongAccumulator(Long::max, 0);
|
||||
private final LongAccumulator minBatchLength = new LongAccumulator(Long::min, Integer.MAX_VALUE);
|
||||
private final MbtilesMetadata mbtilesMetadata;
|
||||
|
||||
private MbtilesWriter(FeatureGroup features, Mbtiles db, FlatmapConfig config, MbtilesMetadata mbtilesMeatadata,
|
||||
|
@ -105,7 +104,13 @@ public class MbtilesWriter {
|
|||
|
||||
var pipeline = WorkerPipeline.start("mbtiles", stats);
|
||||
|
||||
int queueSize = 5_000;
|
||||
// a larger tile queue size helps keep cores busy, but needs a lot of RAM
|
||||
// 5k works fine with 100GB of RAM, so adjust the queue size down from there
|
||||
// but no less than 100
|
||||
int queueSize = Math.max(
|
||||
100,
|
||||
(int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)
|
||||
);
|
||||
|
||||
WorkerPipeline<TileBatch> encodeBranch, writeBranch = null;
|
||||
if (config.emitTilesInOrder()) {
|
||||
|
@ -148,9 +153,9 @@ public class MbtilesWriter {
|
|||
|
||||
var loggers = ProgressLoggers.create()
|
||||
.addRatePercentCounter("features", features.numFeaturesWritten(), writer.featuresProcessed)
|
||||
.addFileSize(features)
|
||||
.addRateCounter("tiles", writer::tilesEmitted)
|
||||
.addFileSize(fileSize)
|
||||
.add(" features ").addFileSize(features)
|
||||
.newLine()
|
||||
.addProcessStats()
|
||||
.newLine()
|
||||
|
@ -167,30 +172,18 @@ public class MbtilesWriter {
|
|||
timer.stop();
|
||||
}
|
||||
|
||||
private static byte[] gzipCompress(byte[] uncompressedData) throws IOException {
|
||||
var bos = new ByteArrayOutputStream(uncompressedData.length);
|
||||
try (var gzipOS = new GZIPOutputStream(bos)) {
|
||||
gzipOS.write(uncompressedData);
|
||||
}
|
||||
return bos.toByteArray();
|
||||
}
|
||||
|
||||
private String getLastTileLogDetails() {
|
||||
TileCoord lastTile = lastTileWritten.get();
|
||||
String blurb;
|
||||
long minBatch = minBatchLength.getThenReset();
|
||||
long maxBatch = maxBatchLength.getThenReset();
|
||||
String batchSizeRange = (minBatch > 0 && maxBatch < Integer.MAX_VALUE) ? (minBatch + "-" + maxBatch) : "-";
|
||||
if (lastTile == null) {
|
||||
blurb = "n/a";
|
||||
} else {
|
||||
var extentForZoom = config.bounds().tileExtents().getForZoom(lastTile.z());
|
||||
int zMinX = extentForZoom.minX();
|
||||
int zMaxX = extentForZoom.maxX();
|
||||
blurb = "%d/%d/%d (z%d %s%%) batch sizes: %s %s".formatted(
|
||||
blurb = "%d/%d/%d (z%d %s%%) %s".formatted(
|
||||
lastTile.z(), lastTile.x(), lastTile.y(),
|
||||
lastTile.z(), (100 * (lastTile.x() + 1 - zMinX)) / (zMaxX - zMinX),
|
||||
batchSizeRange,
|
||||
lastTile.getDebugUrl()
|
||||
);
|
||||
}
|
||||
|
@ -255,7 +248,7 @@ public class MbtilesWriter {
|
|||
} else {
|
||||
VectorTile en = tileFeatures.getVectorTileEncoder();
|
||||
encoded = en.encode();
|
||||
bytes = gzipCompress(encoded);
|
||||
bytes = gzip(encoded);
|
||||
last = tileFeatures;
|
||||
lastEncoded = encoded;
|
||||
lastBytes = bytes;
|
||||
|
@ -303,7 +296,6 @@ public class MbtilesWriter {
|
|||
while ((batch = tileBatches.get()) != null) {
|
||||
Queue<Mbtiles.TileEntry> tiles = batch.out.get();
|
||||
Mbtiles.TileEntry tile;
|
||||
long batchSize = 0;
|
||||
while ((tile = tiles.poll()) != null) {
|
||||
TileCoord tileCoord = tile.tile();
|
||||
assert lastTile == null || lastTile.compareTo(tileCoord) < 0 : "Tiles out of order %s before %s"
|
||||
|
@ -322,14 +314,15 @@ public class MbtilesWriter {
|
|||
batchedWriter.write(tile.tile(), tile.bytes());
|
||||
stats.wroteTile(z, tile.bytes().length);
|
||||
tilesByZoom[z].inc();
|
||||
batchSize++;
|
||||
}
|
||||
maxBatchLength.accumulate(batchSize);
|
||||
minBatchLength.accumulate(batchSize);
|
||||
lastTileWritten.set(lastTile);
|
||||
}
|
||||
}
|
||||
|
||||
if (time != null) {
|
||||
LOGGER.info("Finished z" + currentZ + " in " + time.stop());
|
||||
}
|
||||
|
||||
if (config.optimizeDb()) {
|
||||
db.vacuumAnalyze();
|
||||
}
|
|
@ -0,0 +1,236 @@
|
|||
package com.onthegomap.flatmap.mbtiles;
|
||||
|
||||
import static com.onthegomap.flatmap.util.Gzip.gunzip;
|
||||
|
||||
import com.onthegomap.flatmap.VectorTile;
|
||||
import com.onthegomap.flatmap.geo.GeometryException;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.locationtech.jts.geom.GeometryCollection;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
|
||||
/**
|
||||
* A utility to verify the contents of an mbtiles file.
|
||||
* <p>
|
||||
* {@link #verify(Mbtiles)} does a basic set of checks that the schema is correct and contains a "name" attribute and at
|
||||
* least one tile. Other classes can add more tests to it.
|
||||
*/
|
||||
public class Verify {
|
||||
|
||||
private static final String GOOD = "\u001B[32m✓\u001B[0m";
|
||||
private static final String BAD = "\u001B[31m✕\u001B[0m";
|
||||
|
||||
private final List<Check> checks = new ArrayList<>();
|
||||
private final Mbtiles mbtiles;
|
||||
|
||||
private Verify(Mbtiles mbtiles) {
|
||||
this.mbtiles = mbtiles;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
try (var mbtiles = Mbtiles.newReadOnlyDatabase(Path.of(args[0]))) {
|
||||
var result = Verify.verify(mbtiles);
|
||||
result.print();
|
||||
result.failIfErrors();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of features in a layer inside a lat/lon bounding box with a geometry type and attributes.
|
||||
*
|
||||
* @param db the mbtiles file
|
||||
* @param layer the layer to check
|
||||
* @param zoom zoom level of tiles to check
|
||||
* @param attrs partial set of attributes to filter features
|
||||
* @param envelope lat/lon bounding box to limit check
|
||||
* @param clazz {@link Geometry} subclass to limit
|
||||
* @return number of features found
|
||||
* @throws IOException if an error occurs reading from the file
|
||||
* @throws GeometryException if an invalid geometry is encountered
|
||||
*/
|
||||
public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map<String, Object> attrs, Envelope envelope,
|
||||
Class<? extends Geometry> clazz) throws IOException, GeometryException {
|
||||
int num = 0;
|
||||
for (var tileCoord : db.getAllTileCoords()) {
|
||||
Envelope tileEnv = new Envelope();
|
||||
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY()));
|
||||
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY()));
|
||||
if (tileCoord.z() == zoom) {
|
||||
byte[] data = db.getTile(tileCoord);
|
||||
for (var feature : decode(data)) {
|
||||
if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) {
|
||||
Geometry geometry = feature.geometry().decode();
|
||||
num += getGeometryCounts(geometry, clazz);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
private static int getGeometryCounts(Geometry geom, Class<? extends Geometry> clazz) {
|
||||
int count = 0;
|
||||
if (geom instanceof GeometryCollection geometryCollection) {
|
||||
for (int i = 0; i < geometryCollection.getNumGeometries(); i++) {
|
||||
count += getGeometryCounts(geometryCollection.getGeometryN(i), clazz);
|
||||
}
|
||||
} else if (clazz.isInstance(geom)) {
|
||||
count = 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static List<VectorTile.Feature> decode(byte[] zipped) {
|
||||
try {
|
||||
return VectorTile.decode(gunzip(zipped));
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a verification result of a basic set of checks on an mbtiles file:
|
||||
* <ul>
|
||||
* <li>has a metadata and tiles table</li>
|
||||
* <li>has a name metadata attribute</li>
|
||||
* <li>has at least one tile</li>
|
||||
* <li>all vector tile geometries are valid</li>
|
||||
* </ul>
|
||||
*/
|
||||
public static Verify verify(Mbtiles mbtiles) {
|
||||
Verify result = new Verify(mbtiles);
|
||||
result.checkBasicStructure();
|
||||
return result;
|
||||
}
|
||||
|
||||
private static boolean isValid(Geometry geom) {
|
||||
if (geom instanceof Polygon polygon) {
|
||||
return polygon.isSimple();
|
||||
} else if (geom instanceof GeometryCollection geometryCollection) {
|
||||
for (int i = 0; i < geometryCollection.getNumGeometries(); i++) {
|
||||
if (!isValid(geom.getGeometryN(i))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a check to this verification result per zoom-level that succeeds if at least {@code minCount} features are
|
||||
* found matching the provided criteria.
|
||||
*
|
||||
* @param bounds lat/lon bounding box to limit check
|
||||
* @param layer the layer to check
|
||||
* @param tags partial set of attributes to filter features
|
||||
* @param minzoom min zoom level of tiles to check
|
||||
* @param maxzoom max zoom level of tiles to check
|
||||
* @param minCount minimum number of required features
|
||||
* @param geometryType {@link Geometry} subclass to limit matches to
|
||||
*/
|
||||
public void checkMinFeatureCount(Envelope bounds, String layer, Map<String, Object> tags, int minzoom, int maxzoom,
|
||||
int minCount, Class<? extends Geometry> geometryType) {
|
||||
for (int z = minzoom; z <= maxzoom; z++) {
|
||||
checkMinFeatureCount(bounds, layer, tags, z, minCount, geometryType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a check to this verification result that succeeds if at least {@code minCount} features are found matching the
|
||||
* provided criteria.
|
||||
*
|
||||
* @param bounds lat/lon bounding box to limit check
|
||||
* @param layer the layer to check
|
||||
* @param tags partial set of attributes to filter features
|
||||
* @param zoom zoom level of tiles to check
|
||||
* @param minCount minimum number of required features
|
||||
* @param geometryType {@link Geometry} subclass to limit matches to
|
||||
*/
|
||||
public void checkMinFeatureCount(Envelope bounds, String layer, Map<String, Object> tags, int zoom, int minCount,
|
||||
Class<? extends Geometry> geometryType) {
|
||||
checkWithMessage("at least %d %s %s features at z%d".formatted(minCount, layer, tags, zoom), () -> {
|
||||
try {
|
||||
int count = getNumFeatures(mbtiles, layer, zoom, tags, bounds, geometryType);
|
||||
return count >= minCount ? Optional.empty() : Optional.of("found " + count);
|
||||
} catch (IOException | GeometryException e) {
|
||||
return Optional.of("error: " + e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Logs verification results. */
|
||||
public void print() {
|
||||
for (Check check : checks) {
|
||||
check.error.ifPresentOrElse(
|
||||
error -> System.out.println(BAD + " " + check.name + ": " + error),
|
||||
() -> System.out.println(GOOD + " " + check.name)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Exits with a nonzero exit code if there were any failures. */
|
||||
public void failIfErrors() {
|
||||
long errors = numErrors();
|
||||
System.out.println(errors + " errors");
|
||||
if (errors > 0) {
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkBasicStructure() {
|
||||
check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name"));
|
||||
check("contains at least one tile", () -> !mbtiles.getAllTileCoords().isEmpty());
|
||||
checkWithMessage("all tiles are valid", () -> {
|
||||
List<String> invalidTiles = mbtiles.getAllTileCoords().stream()
|
||||
.flatMap(coord -> checkValidity(coord, decode(mbtiles.getTile(coord))).stream())
|
||||
.toList();
|
||||
return invalidTiles.isEmpty() ? Optional.empty() :
|
||||
Optional.of(invalidTiles.size() + " invalid tiles: " + invalidTiles.stream().limit(5).toList());
|
||||
});
|
||||
}
|
||||
|
||||
private Optional<String> checkValidity(TileCoord coord, List<VectorTile.Feature> features) {
|
||||
for (var feature : features) {
|
||||
try {
|
||||
Geometry geometry = feature.geometry().decode();
|
||||
if (!isValid(geometry)) {
|
||||
return Optional.of(coord + "/" + feature.layer());
|
||||
}
|
||||
} catch (GeometryException e) {
|
||||
return Optional.of(coord + " error decoding " + feature.layer() + "feature");
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private void checkWithMessage(String name, Supplier<Optional<String>> check) {
|
||||
try {
|
||||
checks.add(new Check(name, check.get()));
|
||||
} catch (Throwable e) {
|
||||
checks.add(new Check(name, Optional.of(e.toString())));
|
||||
}
|
||||
}
|
||||
|
||||
private void check(String name, Supplier<Boolean> check) {
|
||||
checkWithMessage(name, () -> check.get() ? Optional.empty() : Optional.of("false"));
|
||||
}
|
||||
|
||||
public List<Check> results() {
|
||||
return checks;
|
||||
}
|
||||
|
||||
public long numErrors() {
|
||||
return checks.stream().filter(check -> check.error.isPresent()).count();
|
||||
}
|
||||
|
||||
public static record Check(String name, Optional<String> error) {}
|
||||
}
|
|
@ -27,7 +27,7 @@ import org.openstreetmap.osmosis.osmbinary.Osmformat.HeaderBlock;
|
|||
/**
|
||||
* An input file in {@code .osm.pbf} format.
|
||||
*
|
||||
* @see <a href="https://wiki.openstreetmap.org/wiki/PBF_Format>OSM PBF Format</a>
|
||||
* @see <a href="https://wiki.openstreetmap.org/wiki/PBF_Format">OSM PBF Format</a>
|
||||
*/
|
||||
public class OsmInputFile implements Bounds.Provider, OsmSource {
|
||||
|
||||
|
|
|
@ -135,19 +135,18 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
var pipeline = WorkerPipeline.start("osm_pass1", stats)
|
||||
.fromGenerator("pbf", osmInputFile.read("pbfpass1", parseThreads))
|
||||
.addBuffer("reader_queue", 50_000, 10_000)
|
||||
// use only 1 thread since processPass1Element needs to be single-threaded
|
||||
.sinkToConsumer("process", 1, this::processPass1Element);
|
||||
|
||||
var loggers = ProgressLoggers.create()
|
||||
.addRateCounter("nodes", PASS1_NODES, true)
|
||||
.addFileSize(nodeLocationDb)
|
||||
.addFileSizeAndRam(nodeLocationDb)
|
||||
.addRateCounter("ways", PASS1_WAYS, true)
|
||||
.addRateCounter("rels", PASS1_RELATIONS, true)
|
||||
.newLine()
|
||||
.addProcessStats()
|
||||
.addInMemoryObject("hppc", this)
|
||||
.addThreadPoolStats("parse", pbfParsePrefix + "-pool")
|
||||
.newLine()
|
||||
.addThreadPoolStats("parse", pbfParsePrefix + "-pool")
|
||||
.addPipelineStats(pipeline);
|
||||
pipeline.awaitAndLog(loggers, config.logInterval());
|
||||
timer.stop();
|
||||
|
@ -156,6 +155,9 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
void processPass1Element(ReaderElement readerElement) {
|
||||
// only a single thread calls this with elements ordered by ID, so it's safe to manipulate these
|
||||
// shared data structures which are not thread safe
|
||||
if (readerElement.getId() < 0) {
|
||||
throw new IllegalArgumentException("Negative OSM element IDs not supported: " + readerElement);
|
||||
}
|
||||
if (readerElement instanceof ReaderNode node) {
|
||||
PASS1_NODES.inc();
|
||||
// TODO allow limiting node storage to only ones that profile cares about
|
||||
|
@ -173,7 +175,8 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
relationInfoSizes.addAndGet(info.estimateMemoryUsageBytes());
|
||||
for (ReaderRelation.Member member : rel.getMembers()) {
|
||||
int type = member.getType();
|
||||
if (type == ReaderRelation.Member.WAY || type == ReaderRelation.Member.RELATION) {
|
||||
// TODO handle nodes in relations and super-relations
|
||||
if (type == ReaderRelation.Member.WAY) {
|
||||
wayToRelations.put(member.getRef(), encodeRelationMembership(member.getRole(), rel.getId()));
|
||||
}
|
||||
}
|
||||
|
@ -198,8 +201,9 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
*/
|
||||
public void pass2(FeatureGroup writer, FlatmapConfig config) {
|
||||
var timer = stats.startStage("osm_pass2");
|
||||
int readerThreads = Math.max(config.threads() / 4, 1);
|
||||
int processThreads = config.threads() - 1;
|
||||
int threads = config.threads();
|
||||
int readerThreads = Math.max(threads / 4, 1);
|
||||
int processThreads = threads - (threads >= 4 ? 1 : 0);
|
||||
Counter.MultiThreadCounter nodesProcessed = Counter.newMultiThreadCounter();
|
||||
Counter.MultiThreadCounter waysProcessed = Counter.newMultiThreadCounter();
|
||||
Counter.MultiThreadCounter relsProcessed = Counter.newMultiThreadCounter();
|
||||
|
@ -263,7 +267,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
|
||||
var logger = ProgressLoggers.create()
|
||||
.addRatePercentCounter("nodes", PASS1_NODES.get(), nodesProcessed)
|
||||
.addFileSize(nodeLocationDb)
|
||||
.addFileSizeAndRam(nodeLocationDb)
|
||||
.addRatePercentCounter("ways", PASS1_WAYS.get(), waysProcessed)
|
||||
.addRatePercentCounter("rels", PASS1_RELATIONS.get(), relsProcessed)
|
||||
.addRateCounter("features", writer::numFeaturesWritten)
|
||||
|
|
|
@ -111,7 +111,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
|
|||
TileCoord tile = entry.getKey();
|
||||
List<List<CoordinateSequence>> result = entry.getValue();
|
||||
Geometry geom = GeometryCoordinateSequences.reassemblePoints(result);
|
||||
encodeAndEmitFeature(feature, id, attrs, tile, geom, groupInfo);
|
||||
encodeAndEmitFeature(feature, id, attrs, tile, geom, groupInfo, 0);
|
||||
emitted++;
|
||||
}
|
||||
stats.emittedFeatures(zoom, feature.getLayer(), emitted);
|
||||
|
@ -121,13 +121,13 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
|
|||
}
|
||||
|
||||
private void encodeAndEmitFeature(FeatureCollector.Feature feature, long id, Map<String, Object> attrs,
|
||||
TileCoord tile, Geometry geom, RenderedFeature.Group groupInfo) {
|
||||
TileCoord tile, Geometry geom, RenderedFeature.Group groupInfo, int scale) {
|
||||
consumer.accept(new RenderedFeature(
|
||||
tile,
|
||||
new VectorTile.Feature(
|
||||
feature.getLayer(),
|
||||
id,
|
||||
VectorTile.encodeGeometry(geom),
|
||||
VectorTile.encodeGeometry(geom, scale),
|
||||
attrs,
|
||||
groupInfo == null ? VectorTile.Feature.NO_GROUP : groupInfo.group()
|
||||
),
|
||||
|
@ -198,6 +198,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
|
|||
List<List<CoordinateSequence>> geoms = entry.getValue();
|
||||
|
||||
Geometry geom;
|
||||
int scale = 0;
|
||||
if (feature.isPolygon()) {
|
||||
geom = GeometryCoordinateSequences.reassemblePolygons(geoms);
|
||||
/*
|
||||
|
@ -214,10 +215,18 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
|
|||
geom = geom.reverse();
|
||||
} else {
|
||||
geom = GeometryCoordinateSequences.reassembleLineStrings(geoms);
|
||||
// Store lines with extra precision (2^scale) in intermediate feature storage so that
|
||||
// rounding does not introduce artificial endpoint intersections and confuse line merge
|
||||
// post-processing. Features need to be "unscaled" in FeatureGroup after line merging,
|
||||
// and before emitting to output mbtiles.
|
||||
scale = Math.max(config.maxzoom(), 14) - zoom;
|
||||
// need 14 bits to represent tile coordinates (4096 * 2 for buffer * 2 for zig zag encoding)
|
||||
// so cap the scale factor to avoid overflowing 32-bit integer space
|
||||
scale = Math.min(31 - 14, scale);
|
||||
}
|
||||
|
||||
if (!geom.isEmpty()) {
|
||||
encodeAndEmitFeature(feature, id, attrs, tile, geom, null);
|
||||
encodeAndEmitFeature(feature, id, attrs, tile, geom, null, scale);
|
||||
emitted++;
|
||||
}
|
||||
} catch (GeometryException e) {
|
||||
|
|
|
@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
|
|||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.DoubleFunction;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.LongSupplier;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory;
|
|||
/**
|
||||
* Logs the progress of a long-running task (percent complete, queue sizes, CPU and memory usage, etc.)
|
||||
*/
|
||||
@SuppressWarnings("UnusedReturnValue")
|
||||
@SuppressWarnings({"UnusedReturnValue", "unused"})
|
||||
public class ProgressLoggers {
|
||||
|
||||
private static final String COLOR_RESET = "\u001B[0m";
|
||||
|
@ -123,6 +124,23 @@ public class ProgressLoggers {
|
|||
* process.
|
||||
*/
|
||||
public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue) {
|
||||
return addRatePercentCounter(name, total, getValue, n -> Format.formatNumeric(n, true));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds "name: [ numCompleted pctComplete% rate/s ]" to the logger where {@code total} is the total number of bytes to
|
||||
* process.
|
||||
*/
|
||||
public ProgressLoggers addStorageRatePercentCounter(String name, long total, LongSupplier getValue) {
|
||||
return addRatePercentCounter(name, total, getValue, n -> Format.formatStorage(n, true));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds "name: [ numCompleted pctComplete% rate/s ]" to the logger where {@code total} is the total number of items to
|
||||
* process.
|
||||
*/
|
||||
public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue,
|
||||
Function<Number, String> format) {
|
||||
// if there's no total, we can't show progress so fall back to rate logger instead
|
||||
if (total == 0) {
|
||||
return addRateCounter(name, getValue, true);
|
||||
|
@ -140,8 +158,8 @@ public class ProgressLoggers {
|
|||
last.set(valueNow);
|
||||
lastTime.set(now);
|
||||
String result =
|
||||
"[ " + Format.formatNumeric(valueNow, true) + " " + padLeft(formatPercent(1f * valueNow / total), 4)
|
||||
+ " " + Format.formatNumeric(valueDiff / timeDiff, true) + "/s ]";
|
||||
"[ " + format.apply(valueNow) + " " + padLeft(formatPercent(1f * valueNow / total), 4)
|
||||
+ " " + format.apply(valueDiff / timeDiff) + "/s ]";
|
||||
return valueDiff > 0 ? green(result) : result;
|
||||
}));
|
||||
return this;
|
||||
|
@ -203,6 +221,20 @@ public class ProgressLoggers {
|
|||
return add(() -> " " + padRight(formatStorage(longSupplier.diskUsageBytes(), false), 5));
|
||||
}
|
||||
|
||||
/** Adds the total of disk and memory usage of {@code thing}. */
|
||||
public <T extends DiskBacked & MemoryEstimator.HasEstimate> ProgressLoggers addFileSizeAndRam(T thing) {
|
||||
return add(() -> {
|
||||
long bytes = thing.diskUsageBytes() + thing.estimateMemoryUsageBytes();
|
||||
return " " + padRight(formatStorage(bytes, false), 5);
|
||||
});
|
||||
}
|
||||
|
||||
/** Adds the current size of a file on disk. */
|
||||
public ProgressLoggers addFileSize(String name, DiskBacked file) {
|
||||
loggers.add(new ProgressLogger(name, () -> formatStorage(file.diskUsageBytes(), true)));
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the average number of CPUs and % time in GC since last log along with memory usage, total memory, and memory
|
||||
* used after last GC to the output.
|
||||
|
|
|
@ -167,8 +167,8 @@ class PrometheusStats implements Stats {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void monitorFile(String name, Path path) {
|
||||
filesToMonitor.put(name, path);
|
||||
public Map<String, Path> monitoredFiles() {
|
||||
return filesToMonitor;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -2,12 +2,17 @@ package com.onthegomap.flatmap.stats;
|
|||
|
||||
import static io.prometheus.client.Collector.NANOSECONDS_PER_SECOND;
|
||||
|
||||
import com.onthegomap.flatmap.util.FileUtils;
|
||||
import com.onthegomap.flatmap.util.Format;
|
||||
import com.onthegomap.flatmap.util.LogUtil;
|
||||
import com.onthegomap.flatmap.util.MemoryEstimator;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentSkipListMap;
|
||||
import java.util.function.Supplier;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* A utility that collects and reports more detailed statistics about the JVM and running tasks than logs can convey.
|
||||
|
@ -31,9 +36,21 @@ public interface Stats extends AutoCloseable {
|
|||
return PrometheusStats.createAndStartPushing(destination, job, interval);
|
||||
}
|
||||
|
||||
/** Logs top-level stats at the end of a job like the amount of user and CPU time that each task has taken. */
|
||||
/**
|
||||
* Logs top-level stats at the end of a job like the amount of user and CPU time that each task has taken, and size of
|
||||
* each monitored file.
|
||||
*/
|
||||
default void printSummary() {
|
||||
Logger LOGGER = LoggerFactory.getLogger(getClass());
|
||||
LOGGER.info("-".repeat(40));
|
||||
timers().printSummary();
|
||||
LOGGER.info("-".repeat(40));
|
||||
for (var entry : monitoredFiles().entrySet()) {
|
||||
long size = FileUtils.size(entry.getValue());
|
||||
if (size > 0) {
|
||||
LOGGER.info("\t" + entry.getKey() + "\t" + Format.formatStorage(size, false) + "B");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -65,13 +82,18 @@ public interface Stats extends AutoCloseable {
|
|||
/** Returns the timers for all stages started with {@link #startStage(String)}. */
|
||||
Timers timers();
|
||||
|
||||
/** Returns all the files being monitored. */
|
||||
Map<String, Path> monitoredFiles();
|
||||
|
||||
/** Adds a stat that will track the size of a file or directory located at {@code path}. */
|
||||
void monitorFile(String name, Path path);
|
||||
default void monitorFile(String name, Path path) {
|
||||
monitoredFiles().put(name, path);
|
||||
}
|
||||
|
||||
/** Adds a stat that will track the estimated in-memory size of {@code object}. */
|
||||
void monitorInMemoryObject(String name, MemoryEstimator.HasEstimate object);
|
||||
|
||||
/** Tracks a stat with {@code name} that always has a constant {@value}. */
|
||||
/** Tracks a stat with {@code name} that always has a constant {@code value}. */
|
||||
default void gauge(String name, Number value) {
|
||||
gauge(name, () -> value);
|
||||
}
|
||||
|
@ -125,6 +147,7 @@ public interface Stats extends AutoCloseable {
|
|||
}
|
||||
|
||||
private final Timers timers = new Timers();
|
||||
private final Map<String, Path> monitoredFiles = new ConcurrentSkipListMap<>();
|
||||
|
||||
@Override
|
||||
public void wroteTile(int zoom, int bytes) {
|
||||
|
@ -136,7 +159,8 @@ public interface Stats extends AutoCloseable {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void monitorFile(String name, Path path) {
|
||||
public Map<String, Path> monitoredFiles() {
|
||||
return monitoredFiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -17,7 +17,6 @@ public class Timers {
|
|||
private final Map<String, Timer> timers = Collections.synchronizedMap(new LinkedHashMap<>());
|
||||
|
||||
public void printSummary() {
|
||||
LOGGER.info("-".repeat(50));
|
||||
for (var entry : all().entrySet()) {
|
||||
LOGGER.info("\t" + entry.getKey() + "\t" + entry.getValue().elapsed());
|
||||
}
|
||||
|
@ -27,7 +26,7 @@ public class Timers {
|
|||
Timer timer = Timer.start();
|
||||
timers.put(name, timer);
|
||||
LOGGER.info("Starting...");
|
||||
return () -> LOGGER.info("Finished in " + timers.get(name).stop() + "\n");
|
||||
return () -> LOGGER.info("Finished in " + timers.get(name).stop() + System.lineSeparator());
|
||||
}
|
||||
|
||||
/** Returns a snapshot of all timers currently running. Will not reflect timers that start after it's called. */
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Ładowanie…
Reference in New Issue