pull/1/head v0.1.0-pre1
Mike Barry 2021-10-19 21:57:47 -04:00
rodzic 798bc544a6
commit 39b7eca2a1
160 zmienionych plików z 15054 dodań i 744 usunięć

Wyświetl plik

@ -0,0 +1,32 @@
---
name: Bug report
about: Create a report to help improve Flatmap
title: "[BUG] "
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Download data from '...'
2. Run command '....'
3. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem (include tile IDs or latitude/longitude for visual issues with generated maps)
**Environment (please complete the following information):**
- Hardware: [e.g. 2015 Macbook Pro]
- OS: [e.g. MacOS 10.15.7]
- Java version and distribution: [e.g. Eclipse Temurin 17.35]
- Maven version: [e.g. 3.8.1]
**Additional context**
Add any other context about the problem here.

Wyświetl plik

@ -0,0 +1,20 @@
---
name: Feature request
about: Suggest an idea for Flatmap
title: "[FEATURE] "
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

Wyświetl plik

@ -0,0 +1,22 @@
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions
name: 'Cache data/sources'
description: 'Save/restore data/sources cache'
inputs:
basedir:
description: 'Base dir for computing file hash'
required: false
default: ''
runs:
using: 'composite'
steps:
- name: Get Date
id: get-data
run: |
echo "::set-output name=hash::${{ hashFiles('**/FlatmapRunner.java', '**/Downloader.java', '**/Geofabrik.java', '**/BasemapMain.java') }}"
echo "::set-output name=date::$(date -u "+%Y-%m-%d")"
shell: bash
working-directory: ${{ inputs.basedir }}
- uses: actions/cache@v2
with:
path: data/sources
key: data-sources-${{ steps.get-data.outputs.date }}-${{ steps.get-data.outputs.hash }}

33
.github/dependabot.yml vendored 100644
Wyświetl plik

@ -0,0 +1,33 @@
version: 2
updates:
- package-ecosystem: maven
directory: "/"
open-pull-requests-limit: 1
schedule:
interval: daily
time: "04:30"
timezone: America/New_York
labels:
- dependencies
ignore:
- dependency-name: "com.graphhopper:graphhopper-reader-osm"
- package-ecosystem: maven
directory: "/flatmap-examples"
open-pull-requests-limit: 1
schedule:
interval: daily
time: "04:30"
timezone: America/New_York
labels:
- dependencies
ignore:
- dependency-name: "com.onthegomap.flatmap:*"
- package-ecosystem: github-actions
directory: "/"
open-pull-requests-limit: 1
schedule:
interval: daily
time: "04:30"
timezone: America/New_York
labels:
- dependencies

21
.github/workflows/docs.yml vendored 100644
Wyświetl plik

@ -0,0 +1,21 @@
---
name: Docs
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
markdown-link-check:
name: Broken Links
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Run link check
uses: gaurav-nelson/github-action-markdown-link-check@v1
with:
use-quiet-mode: 'no'
use-verbose-mode: 'yes'
config-file: '.github/workflows/docs_mlc_config.json'

Wyświetl plik

@ -0,0 +1,13 @@
{
"ignorePatterns": [
{
"pattern": "^https://github.com/onthegomap/flatmap/.*$"
},
{
"pattern": "^https://onthegomap.github.io/.*$"
},
{
"pattern": "^http://localhost.*$"
}
]
}

Wyświetl plik

@ -1,7 +1,7 @@
# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
name: Java CI with Maven
name: CI
on:
push:
@ -10,23 +10,94 @@ on:
branches: [ main ]
jobs:
# TODO: add format/checkstyle
build:
runs-on: ubuntu-latest
timeout-minutes: 10
name: Java ${{ matrix.jdk }} / ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-latest ]
# os: [ ubuntu-latest, macos-latest, windows-latest ]
jdk: [ 17 ]
# include:
# - os: ubuntu-latest
# jdk: 16
runs-on: ${{ matrix.os }}
timeout-minutes: 5
steps:
- uses: actions/checkout@v2
- name: Set up JDK 16
- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v2
with:
java-version: '16'
distribution: 'adopt'
- name: Cache Maven packages
uses: actions/cache@v2
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'maven'
- name: Build with mvnw (linux/mac)
if: ${{ !contains(matrix.os, 'windows') }}
run: ./mvnw --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
- name: Build with mvnw.cmd (windows)
if: ${{ contains(matrix.os, 'windows') }}
run: mvnw.cmd --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml
shell: cmd
examples:
name: Example project
runs-on: ubuntu-latest
timeout-minutes: 5
continue-on-error: true
steps:
- uses: actions/checkout@v2
- name: Set up JDK 17
uses: actions/setup-java@v2
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
- name: Build with Maven
run: mvn -B package --file pom.xml
java-version: 17
distribution: 'temurin'
# do not cache to ensure we can resolve dependencies
server-id: 'github-flatmap'
server-username: MAVEN_USERNAME
server-password: MAVEN_PASSWORD
- name: Build and test
run: mvn --batch-mode -no-transfer-progress package --file pom.xml
working-directory: flatmap-examples
# workaround for github maven packages not supporting anonymous access (https://github.community/t/download-from-github-package-registry-without-authentication/14407/111)
env:
MAVEN_USERNAME: 'flatmapbot'
MAVEN_PASSWORD: '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'
- name: Find jar
run: mv target/*with-deps.jar ./run.jar
working-directory: flatmap-examples
- name: Run
run: java -jar run.jar --osm-path=../flatmap-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles
working-directory: flatmap-examples
- name: Verify
run: java -cp run.jar com.onthegomap.flatmap.mbtiles.Verify data/out.mbtiles
working-directory: flatmap-examples
run:
name: Build / Run
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v2
- name: Cache data/sources
uses: ./.github/cache-sources-action
- name: Set up JDK
uses: actions/setup-java@v2
with:
java-version: 17
distribution: 'temurin'
cache: 'maven'
- name: Build this branch
run: ./mvnw -DskipTests -Dimage.version=CI_ONLY --batch-mode -no-transfer-progress package jib:dockerBuild --file pom.xml
- name: Download data (java)
run: java -jar flatmap-dist/target/*with-deps.jar --only-download --area=monaco
- name: Download wikidata (java)
run: java -jar flatmap-dist/target/*with-deps.jar --only-fetch-wikidata --area=monaco
- name: Verify build
run: ./scripts/test-release.sh CI_ONLY
env:
SKIP_EXAMPLE_PROJECT: true

Wyświetl plik

@ -0,0 +1,92 @@
# This workflow builds a map using the base and branch commit of a PR and uploads
# the logs as an artifact that update-pr.yml uses to add back as a comment.
name: Performance
on:
pull_request:
branches: [ main ]
env:
# For performance tests, run this branch against main with:
AREA: rhode island
RAM: 4g
# Also pick up a good chunk of the atlantic ocean to catch any regressions in repeated tile performance
# Omit to infer from .osm.pbf bounds
BOUNDS_ARG: "--bounds=-74.07,21.34,-17.84,43.55"
jobs:
performance:
name: Performance Test
runs-on: ubuntu-latest
timeout-minutes: 20
continue-on-error: true
steps:
- name: 'Cancel previous runs'
uses: styfle/cancel-workflow-action@0.9.1
with:
access_token: ${{ github.token }}
- name: 'Checkout branch'
uses: actions/checkout@v2
with:
path: branch
- name: 'Checkout base'
uses: actions/checkout@v2
with:
path: base
ref: ${{ github.event.pull_request.base.sha }}
- name: Cache data/sources
uses: ./branch/.github/cache-sources-action
with:
basedir: branch
- name: Set up JDK
uses: actions/setup-java@v2
with:
java-version: 17
distribution: 'temurin'
cache: 'maven'
- uses: actions/setup-node@v2
with:
node-version: '14'
- run: npm install -g strip-ansi-cli@3.0.2
- name: 'Build branch'
run: ./scripts/build.sh
working-directory: branch
- name: 'Build base'
run: ./scripts/build.sh
working-directory: base
- name: 'Download data'
run: |
set -eo pipefail
cp base/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
cp branch/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
- name: 'Store build info'
run: |
mkdir build-info
echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha
echo "${{ github.sha }}" > build-info/branch_sha
echo "${{ github.event.number }}" > build-info/pull_request_number
- name: 'Run branch'
run: |
rm -f data/out.mbtiles
cp branch/flatmap-dist/target/*with-deps.jar run.jar
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
ls -alh run.jar | tee -a log
cat log | strip-ansi > build-info/branchlogs.txt
- name: 'Run base'
run: |
rm -f data/out.mbtiles
cp base/flatmap-dist/target/*with-deps.jar run.jar
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
ls -alh run.jar | tee -a log
cat log | strip-ansi > build-info/baselogs.txt
- name: 'Upload build-info'
uses: actions/upload-artifact@v2
with:
name: build-info
path: ./build-info

74
.github/workflows/release.yml vendored 100644
Wyświetl plik

@ -0,0 +1,74 @@
name: Publish a Release
on:
workflow_dispatch:
inputs:
version:
description: 'Version without leading "v" (1.0, 2.3.4, 0.2-alpha, 5.9-beta.3)'
required: true
default: ''
image_tags:
description: 'Extra docker image tags ("latest,test")'
required: true
default: 'latest'
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: write
packages: write
steps:
- name: Ensure version does not start with 'v'
uses: actions/github-script@v5
with:
github-token: ${{ github.token }}
script: |
version = context.payload.inputs.version;
if (/^v/.test(version)) throw new Error("Bad version number: " + version)
- uses: actions/checkout@v2
- name: Cache data/sources
uses: ./.github/cache-sources-action
- uses: actions/setup-java@v2
with:
java-version: '17'
distribution: 'temurin'
cache: 'maven'
- name: Check tag does not exist yet
run: if git rev-list "v${{ github.event.inputs.version }}"; then echo "Tag already exists. Aborting the release process."; exit 1; fi
- run: ./scripts/set-versions.sh "${{ github.event.inputs.version }}"
- run: ./scripts/build-release.sh
- run: ./scripts/test-release.sh "${{ github.event.inputs.version }}"
- name: Create tag
uses: actions/github-script@v5
with:
github-token: ${{ github.token }}
script: |
github.rest.git.createRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: "refs/tags/v${{ github.event.inputs.version }}",
sha: context.sha
})
- run: mv flatmap-dist/target/*with-deps.jar flatmap.jar
- name: Log in to the Container Registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create Release
uses: softprops/action-gh-release@v1
with:
fail_on_unmatched_files: true
tag_name: v${{ github.event.inputs.version }}
draft: true
files: |
flatmap.jar
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: ./scripts/push-release.sh ${{ github.event.inputs.version }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAGS: ${{ github.event.inputs.image_tags }}

42
.github/workflows/snapshot.yml vendored 100644
Wyświetl plik

@ -0,0 +1,42 @@
# This workflow builds a map using the base and branch commit of a PR and uploads
# the logs as an artifact that update-pr.yml uses to add back as a comment.
name: Publish a Snapshot
on:
workflow_dispatch:
inputs:
image_tags:
description: 'Extra docker image tags ("latest,test")'
required: true
default: 'latest,snapshot'
jobs:
snapshot:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v2
- name: Cache data/sources
uses: ./.github/cache-sources-action
- name: Set up JDK
uses: actions/setup-java@v2
with:
java-version: 17
distribution: 'temurin'
cache: 'maven'
- run: ./scripts/build-release.sh
- run: ./scripts/test-release.sh
- name: Log in to the Container Registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- run: ./scripts/push-release.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAGS: ${{ github.event.inputs.image_tags }}

89
.github/workflows/update-pr.yml vendored 100644
Wyświetl plik

@ -0,0 +1,89 @@
# This workflow posts the result of a performance test back to the pull request as a comment.
# Needs to be separate from CI because it has elevated privileges so only runs from main branch.
name: Update PR
on:
workflow_run:
workflows: [ "Performance" ]
types:
- completed
jobs:
updatepr:
runs-on: ubuntu-latest
continue-on-error: true
if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }}
timeout-minutes: 5
steps:
# report status back to pull request
- uses: haya14busa/action-workflow_run-status@v1
- uses: actions/checkout@v2
- name: 'Download branch build info'
uses: dawidd6/action-download-artifact@v2
with:
workflow: ${{ github.event.workflow_run.workflow_id }}
run_id: ${{ github.event.workflow_run.id }}
name: build-info
path: build-info
repo: onthegomap/flatmap
- name: 'Get build info'
id: build_info
run: echo "::set-output name=pr_number::$(cat build-info/pull_request_number)"
- name: 'Build comment-body'
run: |
cat build-info/branchlogs.txt | sed -n '/^.*Tile stats/,$p' > branchsummary.txt
cat build-info/branchlogs.txt | sed -n '/^.*Exception in thread/,$p' >> branchsummary.txt
cat build-info/baselogs.txt | sed -n '/^.*Tile stats:/,$p' > basesummary.txt
cat build-info/baselogs.txt | sed -n '/^.*Exception in thread/,$p' >> basesummary.txt
cat << EOF > comment-body.txt
<table>
<thead>
<tr>
<th>Base $(cat build-info/base_sha)</th>
<th>This Branch $(cat build-info/branch_sha)</th>
</tr>
</thead>
<tr>
<td>
\`\`\`
$(cat basesummary.txt)
\`\`\`
</td>
<td>
\`\`\`
$(cat build-info/branchlogs.txt | sed -n '/^.*Tile stats:/,$p')
\`\`\`
</td>
</tr>
</table>
https://github.com/onthegomap/flatmap/actions/runs/${{ github.event.workflow_run.id }}
<details><summary> <strong>Base Logs $(cat build-info/base_sha)</strong></summary>
\`\`\`
$(cat build-info/baselogs.txt)
\`\`\`
</details>
<details><summary> <strong>This Branch Logs $(cat build-info/branch_sha)</strong></summary>
\`\`\`
$(cat build-info/branchlogs.txt)
\`\`\`
</details>
EOF
- name: 'Dump comment body'
run: cat comment-body.txt
- uses: marocchino/sticky-pull-request-comment@v2
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
path: comment-body.txt
header: performance-tests
number: ${{ steps.build_info.outputs.pr_number }}

2
.gitignore vendored
Wyświetl plik

@ -2,8 +2,10 @@
target/
*.jar
!.mvn/wrapper/*.jar
*.log
*/.idea
.idea/*
*.iml
!.idea/codeStyles

Wyświetl plik

@ -105,6 +105,9 @@
<Python>
<option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true" />
</Python>
<ScalaCodeStyleSettings>
<option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
</ScalaCodeStyleSettings>
<TypeScriptCodeStyleSettings>
<option name="INDENT_CHAINED_CALLS" value="false" />
</TypeScriptCodeStyleSettings>
@ -204,6 +207,9 @@
<option name="TAB_SIZE" value="2" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="Markdown">
<option name="RIGHT_MARGIN" value="120" />
</codeStyleSettings>
<codeStyleSettings language="ObjectiveC">
<option name="RIGHT_MARGIN" value="80" />
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />

Wyświetl plik

@ -0,0 +1,117 @@
/*
* Copyright 2007-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
private static final String WRAPPER_VERSION = "0.5.6";
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
String username = System.getenv("MVNW_USERNAME");
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
}
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

BIN
.mvn/wrapper/maven-wrapper.jar vendored 100644

Plik binarny nie jest wyświetlany.

Wyświetl plik

@ -0,0 +1,2 @@
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.2/apache-maven-3.8.2-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

119
ARCHITECTURE.md 100644
Wyświetl plik

@ -0,0 +1,119 @@
# Flatmap Architecture
![Architecture Diagram](diagrams/architecture.png)
Flatmap builds a map in 3 phases:
1. [Process Input Files](#process-input-files) according to
the [Profile](flatmap-core/src/main/java/com/onthegomap/flatmap/Profile.java) and write rendered tile features to
intermediate files on disk
2. [Sort Features](#sort-features) by tile ID
3. [Emit Vector Tiles](#emit-vector-tiles) by iterating through sorted features to group by tile ID, encoding, and
writing to the output MBTiles file
## 1) Process Input Files
First, Flatmap reads [SourceFeatures](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/SourceFeature.java) from
each input source:
- For "simple
sources" [NaturalEarthReader](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java)
or [ShapefileReader](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java) can get the
latitude/longitude geometry directly from each feature
- For OpenStreetMap `.osm.pbf` files,
[OsmReader](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmReader.java)
needs to make 2 passes through the input file to construct feature geometries:
- pass 1:
- nodes: store node latitude/longitude locations in-memory or on disk
using [LongLongMap](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/LongLongMap.java)
- ways: nothing
- relations: call `preprocessOsmRelation` on the profile and store information returned for each relation of
interest, along with relation member IDs in-memory using
a [LongLongMultimap](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/LongLongMultimap.java).
- pass 2:
- nodes: emit a point source feature
- ways:
- lookup the latitude/longitude for each node ID to get the way geometry and relations that the way is contained
in
- emit a source feature with the reconstructed geometry which can either be a line or polygon, depending on
the `area` tag and whether the way is closed
- if this way is part of a multipolygon, also save the way geometry in-memory for later in
a [LongLongMultimap](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/LongLongMultimap.java)
- relations: for any multipolygon relation, fetch the member geometries and attempt to reconstruct the multipolygon
geometry
using [OsmMultipolygon](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmMultipolygon.java), then
emit a polygon source feature with the reconstructed geometry if successfule
Then, for each [SourceFeature](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/SourceFeature.java), render
vector tile features according to the [Profile](flatmap-core/src/main/java/com/onthegomap/flatmap/Profile.java) in a
worker thread (default 1 per core):
- Call `processFeature` method on the profile for each source feature
- For every vector tile feature added to
the [FeatureCollector](flatmap-core/src/main/java/com/onthegomap/flatmap/FeatureCollector.java):
- Call [FeatureRenderer#accept](flatmap-core/src/main/java/com/onthegomap/flatmap/render/FeatureRenderer.java)
which for each zoom level the feature appears in:
- Scale the geometry to that zoom level
- Simplify it in screen pixel coordinates
- Use [TiledGeometry](flatmap-core/src/main/java/com/onthegomap/flatmap/render/TiledGeometry.java)
to slice the geometry into subcomponents that appear in every tile it touches using the stripe clipping algorithm
derived from [geojson-vt](https://github.com/mapbox/geojson-vt):
- `sliceX` splits the geometry into vertical slices for each "column" representing the X coordinate of a vector
tile
- `sliceY` splits each "column" into "rows" representing the Y coordinate of a vector tile
- Uses an [IntRangeSet](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/IntRangeSet.java) to
optimize processing for large filled areas (like oceans)
- If any features wrapped past -180 or 180 degrees, then repeat with a 360 or -360 degree offset
- Reassemble each vector tile geometry and round to tile precision (4096x4096)
- For polygons, [GeoUtils#snapAndFixPolygon](flatmap-core/src/main/java/com/onthegomap/flatmap/geo/GeoUtils.java)
uses [JTS](https://github.com/locationtech/jts) utilities to fix any topology errors (i.e. self-intersections)
introduced by rounding. This is very expensive, but necessary since clients
like [MapLibre GL JS](https://github.com/maplibre/maplibre-gl-js) produce rendering artifacts for invalid
polygons.
- Encode the feature into compact binary format
using [FeatureGroup#newRenderedFeatureEncoder](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/FeatureGroup.java)
consisting of a sortable 64-bit `long` key (zoom, x, y, layer, sort order) and a binary value encoded
using [MessagePack](https://msgpack.org/) (feature group/limit, feature ID, geometry type, tags, geometry)
- Add the encoded feature to a [WorkQueue](flatmap-core/src/main/java/com/onthegomap/flatmap/worker/WorkQueue.java)
Finally, a single-threaded writer reads encoded features off of the work queue and writes them to disk
using [ExternalMergeSort#add](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java)
- Write features to a "chunk" file until that file hits a size limit (i.e. 1GB) then start writing to a new file
## 2) Sort Features
[ExternalMergeSort](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java) sorts all of
the intermediate features using a worker thread per core:
- Read each "chunk" file into memory
- Sort the features it contains by 64-bit `long` key
- Write the chunk back to disk
## 3) Emit Vector Tiles
[MbtilesWriter](flatmap-core/src/main/java/com/onthegomap/flatmap/mbtiles/MbtilesWriter.java) is the main driver. First,
a single-threaded reader reads features from disk:
- [ExternalMergeSort](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/ExternalMergeSort.java) emits sorted
features by doing a k-way merge using a priority queue of iterators reading from each sorted chunk
- [FeatureGroup](flatmap-core/src/main/java/com/onthegomap/flatmap/collection/FeatureGroup.java) collects consecutive
features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit
to limit point label density
- Then [MbtilesWriter](flatmap-core/src/main/java/com/onthegomap/flatmap/mbtiles/MbtilesWriter.java) groups tiles into
variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the
writer thread waits for finished tiles in order)
Then, process tile batches in worker threads (default 1 per core):
- For each tile in the batch, first check to see if it has the same contents as the previous tile to avoid re-encoding
the same thing for large filled areas (i.e. oceans)
- Encode using [VectorTile](flatmap-core/src/main/java/com/onthegomap/flatmap/VectorTile.java)
- gzip each encoded tile
- Pass the batch of encoded vector tiles to the writer thread
Finally, a single-threaded writer writes encoded vector tiles to the output MBTiles file:
- Create the largest prepared statement supported by SQLite (999 parameters)
- Iterate through finished vector tile batches until the prepared statement is full, flush to disk, then repeat
- Then flush any remaining tiles at the end

35
CONTRIBUTING.md 100644
Wyświetl plik

@ -0,0 +1,35 @@
# Contributing to Flatmap
Pull requests are welcome! To set up your development environment:
- Fork the repo
- [Install Java 16 or later](https://adoptium.net/installation.html)
- Build and run the tests ([mvnw](https://github.com/takari/maven-wrapper) automatically downloads maven the first time
you run it):
- on max/linux: `./mvnw clean test`
- on windows: `mvnw.cmd clean test`
- or if you already have maven installed globally on your machine: `mvn clean test`
To edit the code:
- [Install IntelliJ IDE](https://www.jetbrains.com/help/idea/installation-guide.html)
- In IntelliJ, click `Open`, navigate to the the `pom.xml` file in the local copy of this repo, and `Open`
then `Open as Project`
- If IntelliJ asks (and you trust the code) then click `Trust Project`
- If any java source files show "Cannot resolve symbol..." errors for Flatmap classes, you might need to
select: `File -> Invalidate Caches... -> Just Restart`.
- If you see a "Project JDK is not defined" error, then choose `Setup SDK` and point IntelliJ at the Java 16 or later
installed on your system
- To verify everything works correctly, right click on `flatmap-core/src/test/java` folder and click `Run 'All Tests'`
Any pull request should:
- Include at least one unit test to verify the change in behavior
- Include an end-to-end test in [FlatmapTests.java](flatmap-core/src/test/java/com/onthegomap/flatmap/FlatmapTests.java)
to verify any major new user-facing features work
- Use IntelliJ's auto-formatting for modified files (this should get enabled automatically)
- Be free of IntelliJ warnings for modified files
GitHub Workflows will run regression tests on any pull request.
TODO: Set up checkstyle and an auto-formatter to enforce standards, so you can use any IDE.

Wyświetl plik

@ -25,12 +25,10 @@ The `flatmap-core` module includes the following software:
- `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)
- `TiledGeometry` from [geojson-vt](https://github.com/mapbox/geojson-vt) (ISC license)
- `VectorTileEncoder`
from [java-vector-tile](https://github.com/ElectronicChartCentre/java-vector-tile) (Apache
license)
from [java-vector-tile](https://github.com/ElectronicChartCentre/java-vector-tile) (Apache license)
- `Imposm3Parsers` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)
Additionally, the `flatmap-openmaptiles` module is a Java port
of [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
Additionally, the `flatmap-basemap` module is based on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
- Maven Dependencies:
- org.yaml:snakeyaml (Apache license)
@ -41,8 +39,8 @@ of [OpenMapTiles](https://github.com/openmaptiles/openmaptiles):
- `LanguageUtils` ported from OpenMapTiles
- Schema
- The cartography and visual design features of the map tile schema are licensed
under [CC-BY 4.0](http://creativecommons.org/licenses/by/4.0/). Products or services using maps
derived from OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference
under [CC-BY 4.0](http://creativecommons.org/licenses/by/4.0/). Products or services using maps derived from
OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference
"OpenMapTiles" with a link to http://openmaptiles.org/. More
details [here](https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md#design-license-cc-by-40)

142
PLANET.md 100644
Wyświetl plik

@ -0,0 +1,142 @@
# Generating a Map of the World
To generate a map of the world using the built-in [basemap profile](flatmap-basemap) based
on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles), you will need a machine with java 16 or later installed
and at least 10x as much disk space and 1.5x as much RAM as the `planet.osm.pbf` file you start from. All testing has
been done using Digital Ocean droplets with dedicated vCPUs ([referral link](https://m.do.co/c/a947e99aab25)) and
OpenJDK installed through `apt`. Flatmap splits work among cores so the more you have, the less time it takes.
### 1) Choose the Data Source
First decide where to get the `planet.osm.pbf` file:
- One of the [official mirrors](https://wiki.openstreetmap.org/wiki/Planet.osm)
- The [AWS Registry of Open Data](https://registry.opendata.aws/osm/) public S3 mirror (default)
- Or a [Daylight Distribution](https://daylightmap.org/) snapshot from Facebook that lags a bit, but includes extra
quality/consistency checks, and add-ons like ML-detected roads and buildings. NOTE: you need at least
`admin.osc.bz2` then combine and re-number using [osmium-tool](https://osmcode.org/osmium-tool/):
```bash
osmium apply-changes daylight.osm.pbf admin.osc.bz2 <buildings.osc.bz2, ...> -o everything.osm.pbf
osmium renumber everything.osm.pbf -o planet.osm.pbf
```
This takes about 2.5 hours and needs as much RAM as the `planet.osm.pbf` size.
### 2) Run Flatmap
Download the [latest release](https://github.com/onthegomap/flatmap/releases/latest) of `flatmap.jar`.
Then run `java -Xms100g -Xmx100g -jar flatmap.jar` (replacing `100g` with 1.5x the `planet.osm.pbf` size)
with these options:
- `--bounds=world` to set bounding box to the entire planet
- `--nodemap-type=sparsearray` to store node locations in a sparse array instead of a sorted table (sorted table is only
more efficient for extracts)
- `--nodemap-storage=ram` to store all node locations in RAM instead of a memory-mapped file (when using `ram` give the
JVM 1.5x the input file size instead of 0.5x when using `mmap`)
- `--download` to fetch [other data sources](NOTICE.md#data) automatically
- One of these to point flatmap at your data source:
- `--osm-path=path/to/planet.osm.pbf` to point Flatmap at a file you downloaded
- `--osm-url=http://url/of/planet.osm.pbf` to download automatically
- `--area=planet` to use the file in `./data/sources/planet.osm.pbf` or download the latest snapshot from AWS S3
mirror if missing.
Run with `--help` to see all available arguments.
## Example
To generate the tiles shown on https://onthegomap.github.io/flatmap-demo/ I used the `planet-211011.osm.pbf` (64.7GB) S3
snapshot, then ran Flatmap on a Digital Ocean Memory-Optimized droplet with 16 CPUs, 128GB RAM, and 1.17TB disk running
Ubuntu 21.04 x64 in the nyc3 location.
First, I installed java 17 jre and screen:
```bash
apt-get update && apt-get install -y openjdk-17-jre-headless screen
```
Then I added a script `runworld.sh` to run with 100GB of RAM:
```bash
#!/usr/bin/env bash
set -e
java -Xmx100g -Xms100g \
-XX:OnOutOfMemoryError="kill -9 %p" \
-jar flatmap.jar \
`# Download the latest planet.osm.pbf from s3://osm-pds bucket` \
--area=planet --bounds=world --download \
`# Accelerate the download by fetching the 10 1GB chunks at a time in parallel` \
--download-threads=10 --download-chunk-size-mb=1000 \
`# Also download name translations from wikidata` \
--fetch-wikidata \
`# Personal preference overrides to default OpenMapTiles schema` \
--transportation-name-brunnel=false --transportation-z13-paths=true \
--mbtiles=output.mbtiles \
--nodemap-type=sparsearray --nodemap-storage=ram 2>&1 | tee logs.txt
```
Then I ran this in the background using screen, so it would continue if my shell exited:
```bash
screen -d -m "./runworld.sh"
tail -f logs.txt
```
It took 3h21m (including 12 minutes downloading source data) to generate a 99GB `output.mbtiles` file. See
the [full logs](planet-logs/planet-logs.txt) from this run or this summary that it printed at the end. Notice that it
spent almost an hour emitting z13 tiles. That is because the default basemap profile merges nearby building polygons at
z13 which is very expensive. You can disable this behavior by setting `--building-merge-z13=false`.
```
3:21:03 DEB [mbtiles] - Tile stats:
3:21:03 DEB [mbtiles] - z0 avg:71k max:71k
3:21:03 DEB [mbtiles] - z1 avg:171k max:192k
3:21:03 DEB [mbtiles] - z2 avg:258k max:449k
3:21:03 DEB [mbtiles] - z3 avg:117k max:479k
3:21:03 DEB [mbtiles] - z4 avg:51k max:541k
3:21:03 DEB [mbtiles] - z5 avg:23k max:537k
3:21:03 DEB [mbtiles] - z6 avg:14k max:354k
3:21:03 DEB [mbtiles] - z7 avg:11k max:451k
3:21:03 DEB [mbtiles] - z8 avg:6.5k max:356k
3:21:03 DEB [mbtiles] - z9 avg:6k max:485k
3:21:03 DEB [mbtiles] - z10 avg:2.7k max:285k
3:21:03 DEB [mbtiles] - z11 avg:1.3k max:168k
3:21:03 DEB [mbtiles] - z12 avg:741 max:247k
3:21:03 DEB [mbtiles] - z13 avg:388 max:286k
3:21:03 DEB [mbtiles] - z14 avg:340 max:1.7M
3:21:03 DEB [mbtiles] - all avg:395 max:0
3:21:03 DEB [mbtiles] - # features: 2,832,396,934
3:21:03 DEB [mbtiles] - # tiles: 264,204,266
3:21:03 INF [mbtiles] - Finished in 4,668s cpu:66,977s avg:14.3
3:21:03 INF - Finished in 12,064s cpu:156,169s avg:12.9
3:21:03 INF - FINISHED!
3:21:03 INF - ----------------------------------------
3:21:03 INF - overall 12,064s cpu:156,169s avg:12.9
3:21:03 INF - download 169s cpu:1,070s avg:6.3
3:21:03 INF - wikidata 553s cpu:3,825s avg:6.9
3:21:03 INF - lake_centerlines 0.9s cpu:2s avg:1.8
3:21:03 INF - water_polygons 96s cpu:1,150s avg:12
3:21:03 INF - natural_earth 6s cpu:21s avg:3.7
3:21:03 INF - osm_pass1 921s cpu:5,177s avg:5.6
3:21:03 INF - osm_pass2 5,234s cpu:73,527s avg:14
3:21:03 INF - boundaries 14s cpu:18s avg:1.3
3:21:03 INF - sort 407s cpu:4,403s avg:10.8
3:21:03 INF - mbtiles 4,668s cpu:66,977s avg:14.3
3:21:03 INF - ----------------------------------------
3:21:03 INF - features 192GB
3:21:03 INF - mbtiles 99GB
```
Then to generate the extract for [the demo](https://onthegomap.github.io/flatmap-demo/) I ran:
```bash
# install node and tilelive-copy
curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -
apt-get install -y nodejs
npm install -g @mapbox/tilelive @mapbox/mbtiles
# Extract z0-4 for the world
tilelive-copy --minzoom=0 --maxzoom=4 --bounds=-180,-90,180,90 output.mbtiles demo.mbtiles
# Extract z0-14 for just southern New England
tilelive-copy --minzoom=0 --maxzoom=14 --bounds=-73.6346,41.1055,-69.5464,42.9439 output.mbtiles demo.mbtiles
```

226
README.md 100644
Wyświetl plik

@ -0,0 +1,226 @@
# Flatmap
Flatmap is a tool that generates [Mapbox Vector Tiles](https://github.com/mapbox/vector-tile-spec/tree/master/2.1) from
geographic data sources like [OpenStreetMap](https://www.openstreetmap.org/). Flatmap aims to be fast and
memory-efficient so that you can build a map of the world in a few hours on a single machine without any external tools
or database.
Vector tiles contain raw point, line, and polygon geometries that clients like [MapLibre](https://github.com/maplibre)
can use to render custom maps in the browser, native apps, or on a server. Flatmap packages tiles into
an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite) file that can be served using
tools like [TileServer GL](https://github.com/maptiler/tileserver-gl) or even
[queried directly from the browser](https://github.com/phiresky/sql.js-httpvfs).
See [awesome-vector-tiles](https://github.com/mapbox/awesome-vector-tiles) for more projects that work with data in this
format.
Flatmap is named after the "flatmap" operation that it performs: *map* input elements to rendered tile features,
*flatten* them into a big list, and sort by tile ID to group into tiles. The output is also a "flat map" where zoom
level 0 contains the entire planet in a 256x256 px tile, and each zoom level splits parent tiles into 4 quadrants,
revealing more detail.
See [ARCHITECTURE.md](ARCHITECTURE.md) for more details on how Flatmap works.
## Demo
See the [live demo](https://onthegomap.github.io/flatmap-demo/) of vector tiles created by Flatmap.
[![Flatmap Demo Screenshot](./diagrams/demo.png)](https://onthegomap.github.io/flatmap-demo/)
Style [© OpenMapTiles](https://www.openmaptiles.org/)
&middot; Data [© OpenStreetMap contributors](https://www.openstreetmap.org/copyright)
## Usage
To generate a map of an area using the [basemap profile](flatmap-basemap), you will need:
- [Java 16+](https://adoptium.net/installation.html) or [Docker](https://docs.docker.com/get-docker/)
- at least 1GB of free disk space plus 5-10x the size of the `.osm.pbf` file
- at least 1.5x as much free RAM as the input `.osm.pbf` file size
#### To build the map:
Using Java, download `flatmap.jar` from the [latest release](https://github.com/onthegomap/flatmap/releases/latest)
and run it:
```bash
wget https://github.com/onthegomap/flatmap/releases/latest/download/flatmap.jar
java -Xmx1g -jar flatmap.jar --download --area=monaco
```
Or using Docker:
```bash
docker run -e JAVA_TOOL_OPTIONS="-Xmx1g" -v "$(pwd)/data":/data ghcr.io/onthegomap/flatmap:latest --download --area=monaco
```
#### To view tiles locally:
Using [Node.js](https://nodejs.org/en/download/):
```bash
npm install -g tileserver-gl-light
tileserver-gl-light --mbtiles data/output.mbtiles
```
Or using [Docker](https://docs.docker.com/get-docker/):
```bash
docker run --rm -it -v "$(pwd)/data":/data -p 8080:8080 maptiler/tileserver-gl -p 8080
```
Then open http://localhost:8080 to view tiles.
Some common arguments:
- `--download` downloads input sources automatically and `--only-download` exits after downloading
- `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/)
- `--osm-path=path/to/file.osm.pbf` points Flatmap at an existing OSM extract on disk
- `-Xmx1g` controls how much RAM to give the JVM (recommended: 0.5x the input .osm.pbf file size to leave room for
memory-mapped files)
- `--force` overwrites the output file
- `--help` shows all of the options and exits
## Generating a Map of the World
See [PLANET.md](PLANET.md).
## Creating a Custom Map
See the [flatmap-examples](flatmap-examples) project.
## Benchmarks
Some example runtimes (excluding downloading resources):
| Input | Input Size | Profile | Machine | Time | mbtiles size | Logs |
| --- | --- | --- | --- | --- | --- | --- |
| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap | DO 16cpu 128GB RAM | 3h9m (cpu: 42h1m @ 13.3) | 99.6GB | [logs](planet-logs/planet-logs.txt) [VisualVM Profile](planet-logs/planet.nps) |
| [Daylight Distribution v1.6](https://daylightmap.org/2021/09/29/daylight-v16-released.html) with ML buildings and admin boundaries | 68.6GB | Basemap | DO 16cpu 128GB RAM | 3h13m (cpu: 43h40m @ 13.5) | 101.4GB | [logs](planet-logs/logs-daylight.txt) |
| s3://osm-pds/2021/planet-211011.osm.pbf | 64.7GB | Basemap (without z13 building merge) | c5ad.16xlarge (64cpu/128GB RAM) | 59m26s (cpu: 27h6m @ 27.4) | 97.3GB | [logs](planet-logs/planet-logs-c5ad.txt) |
## Alternatives
Some other tools that generate vector tiles from OpenStreetMap data:
- [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) is the reference implementation of
the [OpenMapTiles schema](https://openmaptiles.org/schema/) that the [basemap profile](flatmap-basemap) is based on.
It uses an intermediate postgres database and operates in two modes:
1. Import data into database (~1 day) then serve vector tiles directly from the database. Tile serving is slower and
requires bigger machines, but lets you easily incorporate realtime updates
2. Import data into database (~1 day) then prerender every tile for the planet into an mbtiles file which
takes [over 100 days](https://github.com/openmaptiles/openmaptiles/issues/654#issuecomment-724606293)
or a cluster of machines, but then tiles can be served faster on smaller machines
- [Tilemaker](https://github.com/systemed/tilemaker) uses a similar approach to Flatmap (no intermediate database), is
more mature, and has a convenient lua API for building custom profiles without recompiling the tool, but takes
[about a day](https://github.com/systemed/tilemaker/issues/315#issue-994322040) to generate a map of the world
Some companies that generate and host tiles for you:
- [Mapbox](https://www.mapbox.com/) - data from the pioneer of vector tile technologies
- [Maptiler](https://www.maptiler.com/) - data from the creator of OpenMapTiles schema
- [Stadia Maps](https://stadiamaps.com/) - what [onthegomap.com](https://onthegomap.com/) uses in production
If you want to host tiles yourself but have someone else generate them for you, those companies also offer plans to
download regularly-updated tilesets.
## Features
- Supports [Natural Earth](https://www.naturalearthdata.com/),
OpenStreetMap [.osm.pbf](https://wiki.openstreetmap.org/wiki/PBF_Format),
and [Esri Shapefiles](https://en.wikipedia.org/wiki/Shapefile) data sources
- Java-based [Profile API](flatmap-core/src/main/java/com/onthegomap/flatmap/Profile.java) to customize how source
elements map to vector tile features, and post-process generated tiles
using [JTS geometry utilities](https://github.com/locationtech/jts)
- Automatically fixes self-intersecting polygons
- Built-in basemap profile based on [OpenMapTiles](https://openmaptiles.org/) v3.12.2
- Optionally download additional name translations for elements from Wikidata
- Export real-time stats to a [prometheus push gateway](https://github.com/prometheus/pushgateway) using
`--pushgateway=http://user:password@ip` argument (and a [grafana dashboard](grafana.json) for viewing)
- Automatically downloads region extracts from [Geofabrik](https://download.geofabrik.de/)
using `geofabrik:australia` shortcut as a source URL
- Unit-test profiles to verify mapping logic, or integration-test to verify the actual contents of a generated mbtiles
file ([example](flatmap-examples/src/test/java/com/onthegomap/flatmap/examples/BikeRouteOverlayTest.java))
## Limitations
- It is harder to join and group data than when using database. To join input data sources, profiles must explicitly
store data when processing a feature to use with later features, or apply post-processing to rendered features
immediately before emitting the vector tile.
- Flatmap only does full imports from `.osm.pbf` snapshots, there is no way to incorporate real-time updates.
## Roadmap
- [x] Enough `flatmap-core` functionality to support basemap profile based on OpenMapTiles
- [ ] Basemap profile based on OpenMapTiles v3.12.2
- [x] Port all layers
- [x] Download name translations from wikidata
- [x] Merge buildings at z13
- [x] `adm0_l`/`adm0_r` boundary labels
- [ ] Abbreviate road names to improve visibility
- [ ] Poi layer `agg_stop` tag
- [ ] Get `flatmap-core` into Maven Central
- [ ] Remove geotools dependency for reading shapefiles (not in Maven Central)
- [ ] Remove graphhopper dependency for reading OSM files
- [ ] "Sparse mode" to only store node and relation data for elements used by a profile
- [ ] Support zoom levels higher than 14
- [ ] Handle nodes and relations in relations (only ways handled now)
- [ ] Lake centerline support in `flatmap-core`
- [ ] Improve line merging to combine nearby parallel roads
- [ ] Basemap schema improvements for [onthegomap.com](https://onthegomap.com)
- [ ] Accept other kinds of data sources
- [ ] Extract reusable utilities for complex schemas from `flatmap-basemap` to `flatmap-core`
- [ ] Other schemas
## Contributing
Pull requests are welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
## Support
Have a question or want to share something you've built? Start
a [GitHub discussion](https://github.com/onthegomap/flatmap/discussions).
Found a bug or have a feature request? Open a [GitHub issue](https://github.com/onthegomap/flatmap/issues) to report.
This is a side project, so support is limited. If you have the time and ability, feel free to open a pull request to fix
issues or implement new features.
## Acknowledgement
Flatmap is made possible by these awesome open source projects:
- [OpenMapTiles](https://openmaptiles.org/) for the [schema](https://openmaptiles.org/schema/)
and [reference implementation](https://github.com/openmaptiles/openmaptiles)
that the [basemap profile](flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap/layers)
is based on
- [Graphhopper](https://www.graphhopper.com/) for utilities to process OpenStreetMap data in Java
- [JTS Topology Suite](https://github.com/locationtech/jts) for working with vector geometries
- [Geotools](https://github.com/geotools/geotools) for shapefile processing
- [SQLite JDBC Driver](https://github.com/xerial/sqlite-jdbc) for reading Natural Earth data and writing MBTiles files
- [MessagePack](https://msgpack.org/) for compact binary encoding of intermediate map features
- [geojson-vt](https://github.com/mapbox/geojson-vt) for the basis of
the [stripe clipping algorithm](flatmap-core/src/main/java/com/onthegomap/flatmap/render/TiledGeometry.java)
that flatmap uses to slice geometries into tiles
- [java-vector-tile](https://github.com/ElectronicChartCentre/java-vector-tile) for the basis of
the [vector tile encoder](flatmap-core/src/main/java/com/onthegomap/flatmap/VectorTile.java)
- [imposm3](https://github.com/omniscale/imposm3) for the basis
of [OSM multipolygon processing](flatmap-core/src/main/java/com/onthegomap/flatmap/reader/osm/OsmMultipolygon.java)
and [tag parsing utilities](flatmap-core/src/main/java/com/onthegomap/flatmap/util/Imposm3Parsers.java)
See [NOTICE.md](NOTICE.md) for a full list and license details.
## Author
Flatmap was created by [Michael Barry](https://github.com/msbarry) for future use generating custom basemaps or overlays
for [On The Go Map](https://onthegomap.com).
## License and Attribution
Flatmap source code is licensed under the [Apache 2.0 License](LICENSE), so it can be used and modified in commercial or
other open source projects according to the license guidelines.
Maps built using flatmap do not require any special attribution, but the data or schema used might. Any maps generated
from OpenStreetMap data must [visibly credit OpenStreetMap contributors](https://www.openstreetmap.org/copyright). Any
map generated with the profile based on OpenMapTiles or a derivative
must [visibly credit OpenMapTiles](https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md#design-license-cc-by-40)
as well.

Wyświetl plik

@ -11,8 +11,11 @@
# download=true
# only_download=true
# To override the user-agent header when downloading resources:
# To override request parameters when downloading resources:
# http_user_agent=Flatmap downloader (https://github.com/onthegomap/flatmap)
# http_timeout=10s
# download_threads=10
# download_chunk_size=1000
# Map bounds are inferred from OSM input file bounds, but to override use:
# bounds=W,S,E,N
@ -28,6 +31,12 @@
# Change how often to log progress messages:
# loginterval=1m
# Send stats to a prometheus push gateway:
# pushgateway=https://user:password@ip
# "job" tag to include in every push to prometheus (to separate multiple flatmap instances):
# pushgateway.job=flatmap
# pushgateway.interval=15s
# Override the output location for mbtiles file:
# mbtiles=data/output.mbtiles
# Set the location where temporary files are stored (node map or intermediate features)
@ -71,7 +80,7 @@
# mbtiles_version=0.1
# mbtiles_type=baselayer OR overlay
################# OpenMapTiles profile config #################
################# Basemap profile config #################
# Set the input file name and automatically determine the https://download.geofabrik.de/ download URL for a region by name:
# area=monaco

Plik binarny nie jest wyświetlany.

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 238 KiB

Wyświetl plik

@ -0,0 +1,107 @@
@startuml
!includeurl https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/ce5a3ea16aee1b07487f9960633057d102b626a6/C4_Container.puml
skinparam WrapWidth 80
skinparam ranksep 10
skinparam dpi 200
LAYOUT_LANDSCAPE()
HIDE_STEREOTYPE()
skinparam rectangle {
BorderColor<<diagram>> transparent
BackgroundColor<<diagram>> transparent
}
sprite slicing slicing.png
Boundary(input, "**1) Process Input Files**") {
Boundary(simple, "Read Simple Sources", "Natural Earth & Shapefiles") {
System(sr, "Reader")
}
Boundary(osm, "Read OpenStreetMap Data", ".osm.pbf files") {
System(profile2, "Profile")
Boundary(pass1, "pass 1") {
System(osm1n, "Nodes")
System(osm1w, "Ways")
System(osm1r, "Relations")
' enforce ordering
osm1r -[hidden]> osm1w
osm1w -[hidden]> osm1n
}
BiRel_Up(osm1r, profile2, " ")
SystemDb(osmnl, "Node locations")
SystemDb(osmmem, "Relation Members")
SystemDb(osmmp, "Multipolygon geometries")
SystemDb(osmrl, "Relation Info")
Boundary(pass2, "pass 2") {
System(osm2n, "Nodes")
System(osm2w, "Ways")
System(osm2r, "Relations")
' enforce ordering
osm2r -[hidden]> osm2w
osm2w -[hidden]> osm2n
}
Rel(osm1n, osmnl, " ")
Rel(osm1r, osmmem, " ")
Rel(osm1r, osmrl, " ")
Rel(osmnl, osm2w, " ")
Rel(osmmem, osm2w, " ")
Rel(osmrl, osm2w, " ")
Rel_Left(osm2w, osmmp, " ")
Rel(osmmp, osm2r, " ")
}
Boundary(workers, "Process Workers", "1 per core") {
System(profile, "Profile")
System(render, "Render Vector Tile Features")
}
' slicing diagram
rectangle "<$slicing{scale=0.5}>" as slicing <<diagram>>
slicing -[hidden]> render
System(writer, "Writer")
Rel(sr, profile, " ")
Rel(osm2n, profile, " ")
Rel(osm2w, profile, " ")
Rel(osm2r, profile, " ")
Rel(profile, render, " ")
Rel(render, writer, " ")
}
Boundary(sort, "**2) Sort Features**") {
SystemDb(fc1, "Chunk")
SystemDb(fc2, "Chunk")
SystemDb(fc3, "Chunk")
' enforce ordering
fc1 -[hidden]> fc2
fc2 -[hidden]> fc3
System(sorters, "Sort Workers\n1 per core")
BiRel_Up(fc3, sorters, " ")
}
Rel(writer, fc1, " ")
Rel(writer, fc2, " ")
Rel(writer, fc3, " ")
Boundary(mbtiles, "**3) Emit Vector Tiles**") {
System(mbtread, "Read Features\nand group into tiles")
System(mbtprocess, "Encode & gzip\n1 worker per core")
System(mbtwriter, "Batched Writer")
}
SystemDb(mbtilesdb, "MBTiles", "x,y,z,data")
Rel(fc1, mbtread, " ")
Rel(fc2, mbtread, " ")
Rel(fc3, mbtread, " ")
Rel(mbtread, mbtprocess, " ")
Rel(mbtprocess, mbtwriter, " ")
Rel(mbtwriter, mbtilesdb, " ")
@enduml

BIN
diagrams/demo.png 100644

Plik binarny nie jest wyświetlany.

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 488 KiB

Plik binarny nie jest wyświetlany.

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 8.8 KiB

Wyświetl plik

@ -0,0 +1,48 @@
# Flatmap Basemap Profile
This basemap profile is based on [OpenMapTiles](https://github.com/openmaptiles/openmaptiles) v3.12.2.
See [README.md](../README.md) in the parent directory for instructions on how to run.
## Code Layout
[Generate.java](./src/main/java/com/onthegomap/flatmap/basemap/Generate.java) generates code in
the [generated](./src/main/java/com/onthegomap/flatmap/basemap/generated) package from an OpenMapTiles tag in GitHub:
- [OpenMapTilesSchema](./src/main/java/com/onthegomap/flatmap/basemap/generated/OpenMapTilesSchema.java)
contains an interface for each layer with constants for the name, attributes, and their allowed values
- [Tables](./src/main/java/com/onthegomap/flatmap/basemap/generated/Tables.java)
contains a record for each table that OpenMapTiles [imposm3](https://github.com/omniscale/imposm3) configuration
generates (along with the tag-filtering expression) so layers can listen on instances of those records instead of
doing the tag filtering and parsing themselves
The [layers](./src/main/java/com/onthegomap/flatmap/basemap/layers) package contains a port of the SQL logic to generate
each layer from OpenMapTiles. Layers define how source features (or parsed imposm3 table rows) map to vector tile
features, and logic for post-processing tile geometries.
[BasemapProfile](./src/main/java/com/onthegomap/flatmap/basemap/BasemapProfile.java)
dispatches source features to layers and merges the results.
[BasemapMain](./src/main/java/com/onthegomap/flatmap/basemap/BasemapMain.java) is the main driver that registers source
data and output location.
## Regenerating Code
To run `Generate.java`, use [scripts/regenerate-openmaptiles.sh](../scripts/regenerate-openmaptiles.sh) script with the
OpenMapTiles release tag:
```bash
./scripts/regenerate-openmaptiles.sh v3.12.2
```
Then follow the instructions it prints for reformatting generated code.
## License and Attribution
OpenMapTiles code is licensed under the BSD 3-Clause License, which appears at the top of any file ported from
OpenMapTiles.
The OpenMapTiles schema (or "look and feel") is licensed under [CC-BY 4.0](http://creativecommons.org/licenses/by/4.0/),
so any map derived from that schema
must [visibly credit OpenMapTiles](https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md#design-license-cc-by-40)
. It also uses OpenStreetMap data so you
must [visibly credit OpenStreetMap contributors](https://www.openstreetmap.org/copyright).

Wyświetl plik

@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>flatmap-basemap</artifactId>
<parent>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-parent</artifactId>
<version>0.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-core</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.29</version>
</dependency>
<dependency>
<groupId>org.commonmark</groupId>
<artifactId>commonmark</artifactId>
<version>0.18.0</version>
</dependency>
<dependency>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-core</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>io.github.zlika</groupId>
<artifactId>reproducible-build-maven-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<!-- we don't want to deploy this module -->
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

Wyświetl plik

@ -1,14 +1,14 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import com.onthegomap.flatmap.FlatmapRunner;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.config.Arguments;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import java.nio.file.Path;
/**
* Main entrypoint for generating a map using the OpenMapTiles schema.
* Main entrypoint for generating a map using the basemap schema.
*/
public class OpenMapTilesMain {
public class BasemapMain {
public static void main(String[] args) throws Exception {
run(Arguments.fromArgsOrConfigFile(args));
@ -21,7 +21,7 @@ public class OpenMapTilesMain {
// will be ignored if osm_path or osm_url are set
String area = arguments.getString(
"area",
"name of the geofabrik extract to download if osm_url/osm_path not specified (i.e. 'monaco' 'rhode island' or 'australia')",
"name of the extract to download if osm_url/osm_path not specified (i.e. 'monaco' 'rhode island' 'australia' or 'planet')",
"monaco"
);
@ -29,22 +29,22 @@ public class OpenMapTilesMain {
.setDefaultLanguages(OpenMapTilesSchema.LANGUAGES)
.fetchWikidataNameTranslations(sourcesDir.resolve("wikidata_names.json"))
// defer creation of the profile because it depends on data from the runner
.setProfile(OpenMapTilesProfile::new)
.setProfile(BasemapProfile::new)
// override any of these with arguments: --osm_path=... or --osm_url=...
// or OSM_PATH=... OSM_URL=... environmental argument
// or osm_path=... osm_url=... in a config file
.addShapefileSource("EPSG:3857", OpenMapTilesProfile.LAKE_CENTERLINE_SOURCE,
.addShapefileSource("EPSG:3857", BasemapProfile.LAKE_CENTERLINE_SOURCE,
sourcesDir.resolve("lake_centerline.shp.zip"),
"https://github.com/lukasmartinelli/osm-lakelines/releases/download/v0.9/lake_centerline.shp.zip")
.addShapefileSource(OpenMapTilesProfile.WATER_POLYGON_SOURCE,
.addShapefileSource(BasemapProfile.WATER_POLYGON_SOURCE,
sourcesDir.resolve("water-polygons-split-3857.zip"),
"https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip")
.addNaturalEarthSource(OpenMapTilesProfile.NATURAL_EARTH_SOURCE,
.addNaturalEarthSource(BasemapProfile.NATURAL_EARTH_SOURCE,
sourcesDir.resolve("natural_earth_vector.sqlite.zip"),
"https://naturalearth.s3.amazonaws.com/packages/natural_earth_vector.sqlite.zip") // TODO: go back to "https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip")
.addOsmSource(OpenMapTilesProfile.OSM_SOURCE,
"https://naciscdn.org/naturalearth/packages/natural_earth_vector.sqlite.zip")
.addOsmSource(BasemapProfile.OSM_SOURCE,
sourcesDir.resolve(area.replaceAll("[^a-zA-Z]+", "_") + ".osm.pbf"),
"geofabrik:" + area)
"planet".equalsIgnoreCase(area) ? ("aws:latest") : ("geofabrik:" + area))
// override with --mbtiles=... argument or MBTILES=... env var or mbtiles=... in a config file
.setOutput("mbtiles", dataDir.resolve("output.mbtiles"))
.run();

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import static com.onthegomap.flatmap.geo.GeoUtils.EMPTY_LINE;
import static com.onthegomap.flatmap.geo.GeoUtils.EMPTY_POINT;
@ -8,10 +8,10 @@ import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FlatmapRunner;
import com.onthegomap.flatmap.ForwardingProfile;
import com.onthegomap.flatmap.Profile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.reader.SimpleFeature;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
@ -21,8 +21,7 @@ import java.util.ArrayList;
import java.util.List;
/**
* Delegates the logic for generating a map using OpenMapTiles vector schema to individual implementations in the {@code
* layers} package.
* Delegates the logic for generating a map to individual implementations in the {@code layers} package.
* <p>
* Layer implementations extend these interfaces to subscribe to elements from different sources:
* <ul>
@ -40,7 +39,7 @@ import java.util.List;
* {@link FinishHandler} or post-process features in that layer before rendering the output tile by implementing
* {@link FeaturePostProcessor}.
*/
public class OpenMapTilesProfile extends ForwardingProfile {
public class BasemapProfile extends ForwardingProfile {
// IDs used in stats and logs for each input source, as well as argument/config file overrides to source locations
public static final String LAKE_CENTERLINE_SOURCE = "lake_centerlines";
@ -52,11 +51,11 @@ public class OpenMapTilesProfile extends ForwardingProfile {
/** Index variant that filters out any table only used by layers that implement IgnoreWikidata class. */
private final MultiExpression.Index<Boolean> wikidataMappings;
public OpenMapTilesProfile(FlatmapRunner runner) {
public BasemapProfile(FlatmapRunner runner) {
this(runner.translations(), runner.config(), runner.stats());
}
public OpenMapTilesProfile(Translations translations, FlatmapConfig config, Stats stats) {
public BasemapProfile(Translations translations, FlatmapConfig config, Stats stats) {
List<String> onlyLayers = config.arguments().getList("only_layers", "Include only certain layers", List.of());
List<String> excludeLayers = config.arguments().getList("exclude_layers", "Exclude certain layers", List.of());

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import static com.onthegomap.flatmap.expression.Expression.*;
import static java.util.stream.Collectors.joining;
@ -9,12 +9,13 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.CaseFormat;
import com.onthegomap.flatmap.config.Arguments;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.Expression;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.util.Downloader;
import com.onthegomap.flatmap.util.FileUtils;
import com.onthegomap.flatmap.util.Format;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@ -54,6 +55,7 @@ public class Generate {
private static final Logger LOGGER = LoggerFactory.getLogger(Generate.class);
private static final ObjectMapper mapper = new ObjectMapper();
private static final Yaml yaml;
private static final String LINE_SEPARATOR = System.lineSeparator();
private static final String GENERATED_FILE_HEADER = """
/*
Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors.
@ -102,9 +104,9 @@ public class Generate {
yaml = new Yaml(options);
}
private static <T> T loadAndParseYaml(URL url, Class<T> clazz) throws IOException {
private static <T> T loadAndParseYaml(String url, FlatmapConfig config, Class<T> clazz) throws IOException {
LOGGER.info("reading " + url);
try (var stream = url.openStream()) {
try (var stream = Downloader.openStream(url, config)) {
// Jackson yaml parsing does not handle anchors and references, so first parse the input
// using SnakeYAML, then parse SnakeYAML's output using Jackson to get it into our records.
Map<String, Object> parsed = yaml.load(stream);
@ -125,20 +127,21 @@ public class Generate {
public static void main(String[] args) throws IOException {
Arguments arguments = Arguments.fromArgsOrConfigFile(args);
FlatmapConfig flatmapConfig = FlatmapConfig.from(arguments);
String tag = arguments.getString("tag", "openmaptiles tag to use", "v3.12.2");
String base = "https://raw.githubusercontent.com/openmaptiles/openmaptiles/" + tag + "/";
// start crawling from openmaptiles.yaml
// then crawl schema from each layers/<layer>/<layer>.yaml file that it references
// then crawl table definitions from each layers/<layer>/mapping.yaml file that the layer references
var rootUrl = new URL(base + "openmaptiles.yaml");
OpenmaptilesConfig config = loadAndParseYaml(rootUrl, OpenmaptilesConfig.class);
String rootUrl = base + "openmaptiles.yaml";
OpenmaptilesConfig config = loadAndParseYaml(rootUrl, flatmapConfig, OpenmaptilesConfig.class);
List<LayerConfig> layers = new ArrayList<>();
Set<String> imposm3MappingFiles = new LinkedHashSet<>();
for (String layerFile : config.tileset.layers) {
URL layerURL = new URL(base + layerFile);
LayerConfig layer = loadAndParseYaml(layerURL, LayerConfig.class);
String layerURL = base + layerFile;
LayerConfig layer = loadAndParseYaml(layerURL, flatmapConfig, LayerConfig.class);
layers.add(layer);
for (Datasource datasource : layer.datasources) {
if ("imposm3".equals(datasource.type)) {
@ -152,13 +155,13 @@ public class Generate {
Map<String, Imposm3Table> tables = new LinkedHashMap<>();
for (String uri : imposm3MappingFiles) {
Imposm3Mapping layer = loadAndParseYaml(new URL(uri), Imposm3Mapping.class);
Imposm3Mapping layer = loadAndParseYaml(uri, flatmapConfig, Imposm3Mapping.class);
tables.putAll(layer.tables);
}
String packageName = "com.onthegomap.flatmap.openmaptiles.generated";
String packageName = "com.onthegomap.flatmap.basemap.generated";
String[] packageParts = packageName.split("\\.");
Path output = Path.of("flatmap-openmaptiles", "src", "main", "java")
Path output = Path.of("flatmap-basemap", "src", "main", "java")
.resolve(Path.of(packageParts[0], Arrays.copyOfRange(packageParts, 1, packageParts.length)));
FileUtils.deleteDirectory(output);
@ -183,7 +186,7 @@ public class Generate {
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.Layer;
import com.onthegomap.flatmap.basemap.Layer;
import com.onthegomap.flatmap.util.Translations;
import java.util.List;
import java.util.Map;
@ -220,9 +223,9 @@ public class Generate {
info.languages.stream().map(Format::quote).collect(joining(", ")),
layers.stream()
.map(
l -> "new com.onthegomap.flatmap.openmaptiles.layers.%s(translations, config, stats)"
l -> "new com.onthegomap.flatmap.basemap.layers.%s(translations, config, stats)"
.formatted(lowerUnderscoreToUpperCamel(l.layer.id)))
.collect(joining(",\n"))
.collect(joining("," + LINE_SEPARATOR))
.indent(6).trim()
));
for (var layer : layers) {
@ -263,7 +266,7 @@ public class Generate {
* </ul>
*/
""".stripTrailing().formatted(javadocDescription,
valuesForComment.stream().map(v -> "<li>" + v).collect(joining("\n * "))),
valuesForComment.stream().map(v -> "<li>" + v).collect(joining(LINE_SEPARATOR + " * "))),
name.toUpperCase(Locale.ROOT),
Format.quote(name)
).indent(4));
@ -277,7 +280,7 @@ public class Generate {
.map(v -> "public static final String %s = %s;"
.formatted(name.toUpperCase(Locale.ROOT) + "_" + v.toUpperCase(Locale.ROOT).replace('-', '_'),
Format.quote(v)))
.collect(joining("\n")).indent(2).strip()
.collect(joining(LINE_SEPARATOR)).indent(2).strip()
.indent(4));
fieldValues.append("public static final Set<String> %s = Set.of(%s);".formatted(
name.toUpperCase(Locale.ROOT) + "_VALUES",
@ -287,7 +290,7 @@ public class Generate {
if (valuesNode != null && valuesNode.isObject()) {
MultiExpression<String> mapping = generateFieldMapping(valuesNode);
fieldMappings.append(" public static final MultiExpression<String> %s = %s;\n"
fieldMappings.append(" public static final MultiExpression<String> %s = %s;%n"
.formatted(lowerUnderscoreToUpperCamel(name), generateJavaCode(mapping)));
}
});
@ -477,7 +480,7 @@ public class Generate {
interfaceName,
type,
attrName);
}).collect(joining("\n")).indent(2));
}).collect(joining(LINE_SEPARATOR)).indent(2));
tablesClass.append("""
/** Index to efficiently choose which imposm3 "tables" an element should appear in based on its attributes. */
@ -488,7 +491,7 @@ public class Generate {
classNames.stream().map(
className -> "MultiExpression.entry(new RowClassAndConstructor(%s.class, %s::new), %s.MAPPING)".formatted(
className, className, className))
.collect(joining(",\n")).indent(2).strip()
.collect(joining("," + LINE_SEPARATOR)).indent(2).strip()
).indent(2));
String handlerCondition = classNames.stream().map(className ->
@ -496,7 +499,7 @@ public class Generate {
if (handler instanceof %s.Handler typedHandler) {
result.computeIfAbsent(%s.class, cls -> new ArrayList<>()).add(new RowHandlerAndClass<>(typedHandler.getClass(), typedHandler::process));
}""".formatted(className, className)
).collect(joining("\n"));
).collect(joining(LINE_SEPARATOR));
tablesClass.append("""
/**
* Returns a map from imposm3 "table row" class to the layers that have a handler for it from a list of layer
@ -663,11 +666,11 @@ public class Generate {
/** Renders {@code markdown} as HTML and returns comment text safe to insert in generated javadoc. */
private static String markdownToJavadoc(String markdown) {
return Stream.of(markdown.strip().split("\n\n+"))
return Stream.of(markdown.strip().split("[\r\n][\r\n]+"))
.map(p -> parser.parse(p.strip()))
.map(node -> escapeJavadoc(renderer.render(node)))
.map(p -> p.replaceAll("(^<p>|</p>$)", "").strip())
.collect(joining("\n<p>\n"));
.collect(joining(LINE_SEPARATOR + "<p>" + LINE_SEPARATOR));
}
/** Returns {@code comment} text safe to insert in generated javadoc. */

Wyświetl plik

@ -1,8 +1,8 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import com.onthegomap.flatmap.ForwardingProfile;
/** Interface for all vector tile layer implementations that {@link OpenMapTilesProfile} delegates to. */
/** Interface for all vector tile layer implementations that {@link BasemapProfile} delegates to. */
public interface Layer extends
ForwardingProfile.Handler,
ForwardingProfile.HandlerForLayer {}

Wyświetl plik

@ -35,16 +35,16 @@ See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for deta
*/
// AUTOGENERATED BY Generate.java -- DO NOT MODIFY
package com.onthegomap.flatmap.openmaptiles.generated;
package com.onthegomap.flatmap.basemap.generated;
import static com.onthegomap.flatmap.expression.Expression.FALSE;
import static com.onthegomap.flatmap.expression.Expression.and;
import static com.onthegomap.flatmap.expression.Expression.matchAny;
import static com.onthegomap.flatmap.expression.Expression.or;
import com.onthegomap.flatmap.basemap.Layer;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.Layer;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;
import java.util.List;
@ -53,8 +53,7 @@ import java.util.Set;
/**
* All vector tile layer definitions, attributes, and allowed values generated from the
* <a href="https://github.com/openmaptiles/openmaptiles/blob/v3.12.2/openmaptiles.yaml">OpenMapTiles vector tile
* schema
* v3.12.2</a>.
* schema v3.12.2</a>.
*/
@SuppressWarnings("unused")
public class OpenMapTilesSchema {
@ -72,22 +71,22 @@ public class OpenMapTilesSchema {
/** Returns a list of expected layer implementation instances from the {@code layers} package. */
public static List<Layer> createInstances(Translations translations, FlatmapConfig config, Stats stats) {
return List.of(
new com.onthegomap.flatmap.openmaptiles.layers.Water(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Waterway(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Landcover(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Landuse(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.MountainPeak(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Park(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Boundary(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Aeroway(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Transportation(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Building(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.WaterName(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.TransportationName(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Place(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Housenumber(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.Poi(translations, config, stats),
new com.onthegomap.flatmap.openmaptiles.layers.AerodromeLabel(translations, config, stats)
new com.onthegomap.flatmap.basemap.layers.Water(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Waterway(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Landcover(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Landuse(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.MountainPeak(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Park(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Boundary(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Aeroway(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Transportation(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Building(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.WaterName(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.TransportationName(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Place(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Housenumber(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.Poi(translations, config, stats),
new com.onthegomap.flatmap.basemap.layers.AerodromeLabel(translations, config, stats)
);
}
@ -1492,9 +1491,8 @@ public class OpenMapTilesSchema {
* from population and city class). You can use the <strong>rank</strong> to limit density of labels or improve
* the text hierarchy. The rank value is a combination of the Natural Earth <code>scalerank</code>,
* <code>labelrank</code> and <code>datarank</code> values for countries and states and for cities consists out
* of
* a shifted Natural Earth <code>scalerank</code> combined with a local rank within a grid for cities that do not
* have a Natural Earth <code>scalerank</code>.
* of a shifted Natural Earth <code>scalerank</code> combined with a local rank within a grid for cities that do
* not have a Natural Earth <code>scalerank</code>.
*/
public static final String RANK = "rank";
}

Wyświetl plik

@ -35,7 +35,7 @@ See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for deta
*/
// AUTOGENERATED BY Generate.java -- DO NOT MODIFY
package com.onthegomap.flatmap.openmaptiles.generated;
package com.onthegomap.flatmap.basemap.generated;
import static com.onthegomap.flatmap.expression.Expression.*;

Wyświetl plik

@ -33,17 +33,17 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.basemap.util.Utils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.openmaptiles.util.Utils;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;

Wyświetl plik

@ -33,12 +33,12 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;

Wyświetl plik

@ -33,7 +33,7 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.util.MemoryEstimator.CLASS_HEADER_BYTES;
import static com.onthegomap.flatmap.util.MemoryEstimator.POINTER_BYTES;
@ -46,16 +46,17 @@ import com.graphhopper.coll.GHLongObjectHashMap;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FeatureMerge;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.reader.SimpleFeature;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Format;
import com.onthegomap.flatmap.util.MemoryEstimator;
import com.onthegomap.flatmap.util.Parse;
import com.onthegomap.flatmap.util.Translations;
@ -90,11 +91,11 @@ import org.slf4j.LoggerFactory;
*/
public class Boundary implements
OpenMapTilesSchema.Boundary,
OpenMapTilesProfile.NaturalEarthProcessor,
OpenMapTilesProfile.OsmRelationPreprocessor,
OpenMapTilesProfile.OsmAllProcessor,
OpenMapTilesProfile.FeaturePostProcessor,
OpenMapTilesProfile.FinishHandler {
BasemapProfile.NaturalEarthProcessor,
BasemapProfile.OsmRelationPreprocessor,
BasemapProfile.OsmAllProcessor,
BasemapProfile.FeaturePostProcessor,
BasemapProfile.FinishHandler {
/*
* Uses natural earth at lower zoom levels and OpenStreetMap at higher zoom levels.
@ -292,7 +293,7 @@ public class Boundary implements
@Override
public void finish(String sourceName, FeatureCollector.Factory featureCollectors,
Consumer<FeatureCollector.Feature> emit) {
if (OpenMapTilesProfile.OSM_SOURCE.equals(sourceName)) {
if (BasemapProfile.OSM_SOURCE.equals(sourceName)) {
var timer = stats.startStage("boundaries");
LongObjectMap<PreparedGeometry> countryBoundaries = prepareRegionPolygons();
@ -374,11 +375,10 @@ public class Boundary implements
if (left == null && right == null) {
Coordinate point = GeoUtils.worldToLatLonCoords(GeoUtils.pointAlongOffset(lineString, 0.5, 0)).getCoordinate();
LOGGER.warn("no left or right country for border between OSM country relations: %s around %.5f, %.5f"
LOGGER.warn("no left or right country for border between OSM country relations: %s around %s"
.formatted(
validRegions,
point.getX(),
point.getY()
Format.osmDebugUrl(10, point)
));
}

Wyświetl plik

@ -33,9 +33,9 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.util.MemoryEstimator.CLASS_HEADER_BYTES;
import static com.onthegomap.flatmap.util.Parse.parseDoubleOrNull;
import static java.util.Map.entry;
@ -43,11 +43,11 @@ import static java.util.Map.entry;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FeatureMerge;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
import com.onthegomap.flatmap.stats.Stats;
@ -66,8 +66,8 @@ import java.util.Map;
public class Building implements
OpenMapTilesSchema.Building,
Tables.OsmBuildingPolygon.Handler,
OpenMapTilesProfile.FeaturePostProcessor,
OpenMapTilesProfile.OsmRelationPreprocessor {
BasemapProfile.FeaturePostProcessor,
BasemapProfile.OsmRelationPreprocessor {
/*
* Emit all buildings from OSM data at z14.

Wyświetl plik

@ -33,12 +33,12 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;

Wyświetl plik

@ -33,17 +33,17 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FeatureMerge;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;
@ -62,9 +62,9 @@ import java.util.Set;
*/
public class Landcover implements
OpenMapTilesSchema.Landcover,
OpenMapTilesProfile.NaturalEarthProcessor,
BasemapProfile.NaturalEarthProcessor,
Tables.OsmLandcoverPolygon.Handler,
OpenMapTilesProfile.FeaturePostProcessor {
BasemapProfile.FeaturePostProcessor {
/*
* Large ice areas come from natural earth and the rest come from OpenStreetMap at higher zoom

Wyświetl plik

@ -33,16 +33,16 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Parse;
@ -60,7 +60,7 @@ import java.util.Set;
*/
public class Landuse implements
OpenMapTilesSchema.Landuse,
OpenMapTilesProfile.NaturalEarthProcessor,
BasemapProfile.NaturalEarthProcessor,
Tables.OsmLandusePolygon.Handler {
private static final ZoomFunction<Number> MIN_PIXEL_SIZE_THRESHOLDS = ZoomFunction.fromMaxZoomThresholds(Map.of(

Wyświetl plik

@ -33,21 +33,21 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.elevationTags;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.elevationTags;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import com.carrotsearch.hppc.LongIntHashMap;
import com.carrotsearch.hppc.LongIntMap;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Parse;
import com.onthegomap.flatmap.util.Translations;
@ -65,7 +65,7 @@ import org.locationtech.jts.geom.Point;
public class MountainPeak implements
OpenMapTilesSchema.MountainPeak,
Tables.OsmPeakPoint.Handler,
OpenMapTilesProfile.FeaturePostProcessor {
BasemapProfile.FeaturePostProcessor {
/*
* Mountain peaks come from OpenStreetMap data and are ranked by importance (based on if they

Wyświetl plik

@ -33,24 +33,24 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_BITS;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import com.carrotsearch.hppc.LongIntHashMap;
import com.carrotsearch.hppc.LongIntMap;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.geo.GeometryType;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.SortKey;
import com.onthegomap.flatmap.util.Translations;
@ -67,7 +67,7 @@ import java.util.Locale;
public class Park implements
OpenMapTilesSchema.Park,
Tables.OsmParkPolygon.Handler,
OpenMapTilesProfile.FeaturePostProcessor {
BasemapProfile.FeaturePostProcessor {
// constants for packing the minimum zoom ordering of park labels into the sort-key field
private static final int PARK_NATIONAL_PARK_BOOST = 1 << (SORT_KEY_BITS - 1);

Wyświetl plik

@ -33,26 +33,26 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.nullOrEmpty;
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_BITS;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullOrEmpty;
import com.carrotsearch.hppc.LongIntHashMap;
import com.carrotsearch.hppc.LongIntMap;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.geo.PointIndex;
import com.onthegomap.flatmap.geo.PolygonIndex;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Parse;
@ -81,14 +81,14 @@ import org.locationtech.jts.geom.Point;
*/
public class Place implements
OpenMapTilesSchema.Place,
OpenMapTilesProfile.NaturalEarthProcessor,
BasemapProfile.NaturalEarthProcessor,
Tables.OsmContinentPoint.Handler,
Tables.OsmCountryPoint.Handler,
Tables.OsmStatePoint.Handler,
Tables.OsmIslandPoint.Handler,
Tables.OsmIslandPolygon.Handler,
Tables.OsmCityPoint.Handler,
OpenMapTilesProfile.FeaturePostProcessor {
BasemapProfile.FeaturePostProcessor {
/*
* Place labels locations and names come from OpenStreetMap, but we also join with natural

Wyświetl plik

@ -33,24 +33,24 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIf;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullOrEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIf;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.nullOrEmpty;
import static java.util.Map.entry;
import com.carrotsearch.hppc.LongIntHashMap;
import com.carrotsearch.hppc.LongIntMap;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Parse;
import com.onthegomap.flatmap.util.Translations;
@ -68,7 +68,7 @@ public class Poi implements
OpenMapTilesSchema.Poi,
Tables.OsmPoiPoint.Handler,
Tables.OsmPoiPolygon.Handler,
OpenMapTilesProfile.FeaturePostProcessor {
BasemapProfile.FeaturePostProcessor {
/*
* process() creates the raw POI feature from OSM elements and postProcess()

Wyświetl plik

@ -33,19 +33,19 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.*;
import static com.onthegomap.flatmap.basemap.util.Utils.*;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FeatureMerge;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Parse;
import com.onthegomap.flatmap.util.Translations;
@ -69,8 +69,8 @@ public class Transportation implements
Tables.OsmRailwayLinestring.Handler,
Tables.OsmShipwayLinestring.Handler,
Tables.OsmHighwayPolygon.Handler,
OpenMapTilesProfile.FeaturePostProcessor,
OpenMapTilesProfile.IgnoreWikidata {
BasemapProfile.FeaturePostProcessor,
BasemapProfile.IgnoreWikidata {
/*
* Generates the shape for roads, trails, ferries, railways with detailed
@ -273,6 +273,7 @@ public class Transportation implements
.setAttrWithMinzoom(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()), 10)
.setAttrWithMinzoom(Fields.LAYER, nullIf(element.layer(), 0), 9)
.setSortKey(element.zOrder())
.setMinPixelSize(0) // merge during post-processing, then limit by size
.setMinZoom(minzoom);
}
}
@ -288,6 +289,7 @@ public class Transportation implements
.setAttr(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()))
.setAttr(Fields.LAYER, nullIf(element.layer(), 0))
.setSortKey(element.zOrder())
.setMinPixelSize(0) // merge during post-processing, then limit by size
.setMinZoom(12);
}
@ -302,6 +304,7 @@ public class Transportation implements
.setAttr(Fields.BRUNNEL, brunnel(element.isBridge(), element.isTunnel(), element.isFord()))
.setAttr(Fields.LAYER, nullIf(element.layer(), 0))
.setSortKey(element.zOrder())
.setMinPixelSize(0) // merge during post-processing, then limit by size
.setMinZoom(11);
}

Wyświetl plik

@ -33,15 +33,15 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.layers.Transportation.highwayClass;
import static com.onthegomap.flatmap.openmaptiles.layers.Transportation.highwaySubclass;
import static com.onthegomap.flatmap.openmaptiles.layers.Transportation.isFootwayOrSteps;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.brunnel;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIf;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.layers.Transportation.highwayClass;
import static com.onthegomap.flatmap.basemap.layers.Transportation.highwaySubclass;
import static com.onthegomap.flatmap.basemap.layers.Transportation.isFootwayOrSteps;
import static com.onthegomap.flatmap.basemap.util.Utils.brunnel;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIf;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.util.MemoryEstimator.CLASS_HEADER_BYTES;
import static com.onthegomap.flatmap.util.MemoryEstimator.POINTER_BYTES;
import static com.onthegomap.flatmap.util.MemoryEstimator.estimateSize;
@ -49,13 +49,13 @@ import static com.onthegomap.flatmap.util.MemoryEstimator.estimateSize;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FeatureMerge;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmReader;
@ -88,10 +88,10 @@ import org.slf4j.LoggerFactory;
public class TransportationName implements
OpenMapTilesSchema.TransportationName,
Tables.OsmHighwayLinestring.Handler,
OpenMapTilesProfile.NaturalEarthProcessor,
OpenMapTilesProfile.FeaturePostProcessor,
OpenMapTilesProfile.OsmRelationPreprocessor,
OpenMapTilesProfile.IgnoreWikidata {
BasemapProfile.NaturalEarthProcessor,
BasemapProfile.FeaturePostProcessor,
BasemapProfile.OsmRelationPreprocessor,
BasemapProfile.IgnoreWikidata {
/*
* Generate road names from OSM data. Route network and ref are copied

Wyświetl plik

@ -33,15 +33,15 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.Utils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.Utils;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;
@ -55,8 +55,8 @@ import com.onthegomap.flatmap.util.Translations;
public class Water implements
OpenMapTilesSchema.Water,
Tables.OsmWaterPolygon.Handler,
OpenMapTilesProfile.NaturalEarthProcessor,
OpenMapTilesProfile.OsmWaterPolygonProcessor {
BasemapProfile.NaturalEarthProcessor,
BasemapProfile.OsmWaterPolygonProcessor {
/*
* At low zoom levels, use natural earth for oceans and major lakes, and at high zoom levels

Wyświetl plik

@ -33,20 +33,20 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import com.carrotsearch.hppc.LongObjectMap;
import com.graphhopper.coll.GHLongObjectHashMap;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Parse;
@ -68,8 +68,8 @@ public class WaterName implements
OpenMapTilesSchema.WaterName,
Tables.OsmMarinePoint.Handler,
Tables.OsmWaterPolygon.Handler,
OpenMapTilesProfile.NaturalEarthProcessor,
OpenMapTilesProfile.LakeCenterlineProcessor {
BasemapProfile.NaturalEarthProcessor,
BasemapProfile.LakeCenterlineProcessor {
/*
* Labels for lakes and oceans come primarily from OpenStreetMap data, but we also join

Wyświetl plik

@ -33,19 +33,19 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.FeatureMerge;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.basemap.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.basemap.generated.Tables;
import com.onthegomap.flatmap.basemap.util.LanguageUtils;
import com.onthegomap.flatmap.basemap.util.Utils;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.openmaptiles.generated.OpenMapTilesSchema;
import com.onthegomap.flatmap.openmaptiles.generated.Tables;
import com.onthegomap.flatmap.openmaptiles.util.LanguageUtils;
import com.onthegomap.flatmap.openmaptiles.util.Utils;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.util.Translations;
@ -62,8 +62,8 @@ import java.util.Map;
public class Waterway implements
OpenMapTilesSchema.Waterway,
Tables.OsmWaterwayLinestring.Handler,
OpenMapTilesProfile.FeaturePostProcessor,
OpenMapTilesProfile.NaturalEarthProcessor {
BasemapProfile.FeaturePostProcessor,
BasemapProfile.NaturalEarthProcessor {
/*
* Uses Natural Earth at lower zoom-levels and OpenStreetMap at higher zoom levels.

Wyświetl plik

@ -33,10 +33,10 @@ Design license: CC-BY 4.0
See https://github.com/openmaptiles/openmaptiles/blob/master/LICENSE.md for details on usage
*/
package com.onthegomap.flatmap.openmaptiles.util;
package com.onthegomap.flatmap.basemap.util;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.coalesce;
import static com.onthegomap.flatmap.openmaptiles.util.Utils.nullIfEmpty;
import static com.onthegomap.flatmap.basemap.util.Utils.coalesce;
import static com.onthegomap.flatmap.basemap.util.Utils.nullIfEmpty;
import com.onthegomap.flatmap.util.Translations;
import java.util.HashMap;

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.util;
package com.onthegomap.flatmap.basemap.util;
import com.onthegomap.flatmap.util.Parse;
import java.util.Map;

Wyświetl plik

@ -0,0 +1,44 @@
package com.onthegomap.flatmap.basemap.util;
import com.onthegomap.flatmap.mbtiles.Mbtiles;
import com.onthegomap.flatmap.mbtiles.Verify;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.LineString;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Polygon;
/**
* A utility to check the contents of an mbtiles file generated for Monaco.
*/
public class VerifyMonaco {
public static final Envelope MONACO_BOUNDS = new Envelope(7.40921, 7.44864, 43.72335, 43.75169);
/**
* Returns a verification result with a basic set of checks against an openmaptiles map built from an extract for
* Monaco.
*/
public static Verify verify(Mbtiles mbtiles) {
Verify verify = Verify.verify(mbtiles);
verify.checkMinFeatureCount(MONACO_BOUNDS, "building", Map.of(), 13, 14, 100, Polygon.class);
verify.checkMinFeatureCount(MONACO_BOUNDS, "transportation", Map.of(), 10, 14, 5, LineString.class);
verify.checkMinFeatureCount(MONACO_BOUNDS, "landcover", Map.of(
"class", "grass",
"subclass", "park"
), 14, 10, Polygon.class);
verify.checkMinFeatureCount(MONACO_BOUNDS, "water", Map.of("class", "ocean"), 0, 14, 1, Polygon.class);
verify.checkMinFeatureCount(MONACO_BOUNDS, "place", Map.of("class", "country"), 2, 14, 1, Point.class);
return verify;
}
public static void main(String[] args) throws IOException {
try (var mbtiles = Mbtiles.newReadOnlyDatabase(Path.of(args[0]))) {
var result = verify(mbtiles);
result.print();
result.failIfErrors();
}
}
}

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -11,12 +11,12 @@ import com.onthegomap.flatmap.util.Wikidata;
import java.util.List;
import org.junit.jupiter.api.Test;
public class OpenMapTilesProfileTest {
public class BasemapProfileTest {
private final Wikidata.WikidataTranslations wikidataTranslations = new Wikidata.WikidataTranslations();
private final Translations translations = Translations.defaultProvider(List.of("en", "es", "de"))
.addTranslationProvider(wikidataTranslations);
private final OpenMapTilesProfile profile = new OpenMapTilesProfile(translations, FlatmapConfig.defaults(),
private final BasemapProfile profile = new BasemapProfile(translations, FlatmapConfig.defaults(),
Stats.inMemory());
@Test

Wyświetl plik

@ -1,49 +1,53 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import static com.onthegomap.flatmap.TestUtils.assertContains;
import static com.onthegomap.flatmap.TestUtils.assertFeatureNear;
import static com.onthegomap.flatmap.TestUtils.gunzip;
import static com.onthegomap.flatmap.basemap.util.VerifyMonaco.MONACO_BOUNDS;
import static com.onthegomap.flatmap.util.Gzip.gunzip;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
import com.onthegomap.flatmap.TestUtils;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.util.VerifyMonaco;
import com.onthegomap.flatmap.config.Arguments;
import com.onthegomap.flatmap.mbiles.Mbtiles;
import com.onthegomap.flatmap.mbtiles.Mbtiles;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import org.junit.jupiter.api.io.TempDir;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.LineString;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Polygon;
/**
* End-to-end tests for OpenMapTiles map generation.
* End-to-end tests for basemap generation.
* <p>
* Generates an entire map for the smallest openstreetmap extract available (Monaco) and asserts that expected output
* features exist
*/
public class OpenMapTilesTest {
public class BasemapTest {
@TempDir
static Path tmpDir;
private static Mbtiles mbtiles;
private static final Envelope monacoBounds = new Envelope(7.40921, 7.44864, 43.72335, 43.75169);
@BeforeAll
public static void runFlatmap() throws Exception {
Path dbPath = tmpDir.resolve("output.mbtiles");
OpenMapTilesMain.run(Arguments.of(
BasemapMain.run(Arguments.of(
// Override input source locations
"osm_path", TestUtils.pathToResource("monaco-latest.osm.pbf"),
"natural_earth_path", TestUtils.pathToResource("natural_earth_vector.sqlite"),
"natural_earth_path", TestUtils.pathToResource("natural_earth_vector.sqlite.zip"),
"water_polygons_path", TestUtils.pathToResource("water-polygons-split-3857.zip"),
// no centerlines in monaco - so fake it out with an empty source
"lake_centerlines_path", TestUtils.pathToResource("water-polygons-split-3857.zip"),
@ -206,8 +210,16 @@ public class OpenMapTilesTest {
), 14, 6, LineString.class);
}
@TestFactory
public Stream<DynamicTest> testVerifyChecks() {
return VerifyMonaco.verify(mbtiles).results().stream()
.map(check -> dynamicTest(check.name(), () -> {
check.error().ifPresent(Assertions::fail);
}));
}
private static void assertNumFeatures(String layer, Map<String, Object> attrs, int zoom,
int expected, Class<? extends Geometry> clazz) {
TestUtils.assertNumFeatures(mbtiles, layer, zoom, attrs, monacoBounds, expected, clazz);
TestUtils.assertNumFeatures(mbtiles, layer, zoom, attrs, MONACO_BOUNDS, expected, clazz);
}
}

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.openmaptiles;
package com.onthegomap.flatmap.basemap;
import static com.onthegomap.flatmap.basemap.Generate.parseYaml;
import static com.onthegomap.flatmap.expression.Expression.*;
import static com.onthegomap.flatmap.openmaptiles.Generate.parseYaml;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.DynamicTest.dynamicTest;

Wyświetl plik

@ -1,20 +1,20 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.assertSubmap;
import static com.onthegomap.flatmap.TestUtils.newLineString;
import static com.onthegomap.flatmap.TestUtils.newPoint;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.TestUtils;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.reader.SimpleFeature;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmReader;
@ -35,7 +35,7 @@ public abstract class AbstractLayerTest {
.addTranslationProvider(wikidataTranslations);
final FlatmapConfig params = FlatmapConfig.defaults();
final OpenMapTilesProfile profile = new OpenMapTilesProfile(translations, FlatmapConfig.defaults(),
final BasemapProfile profile = new BasemapProfile(translations, FlatmapConfig.defaults(),
Stats.inMemory());
final Stats stats = Stats.inMemory();
final FeatureCollector.Factory featureCollectorFactory = new FeatureCollector.Factory(params, stats);
@ -61,8 +61,8 @@ public abstract class AbstractLayerTest {
if (vals[i - 1] > vals[i]) {
fail(
Arrays.toString(vals) +
"\nelement at " + (i - 1) + " (" + vals[i - 1] + ") is greater than element at " + i + " (" + vals[i]
+ ")");
System.lineSeparator() + "element at " + (i - 1) + " (" + vals[i - 1] + ") is greater than element at " + i
+ " (" + vals[i] + ")");
}
}
}
@ -91,7 +91,7 @@ public abstract class AbstractLayerTest {
for (int zoom = feature.getMinZoom(); zoom <= feature.getMaxZoom(); zoom++) {
Map<String, Object> map = TestUtils.toMap(feature, zoom);
if (zooms[zoom] != null) {
fail("Multiple features at z" + zoom + ":\n" + zooms[zoom] + "\n" + map);
fail("Multiple features at z" + zoom + ":" + System.lineSeparator() + zooms[zoom] + "\n" + map);
}
zooms[zoom] = map;
}

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import java.util.List;
import java.util.Map;

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import java.util.List;
import java.util.Map;

Wyświetl plik

@ -1,9 +1,9 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.newLineString;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.flatmap.VectorTile;

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import java.util.List;
import java.util.Map;

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.flatmap.VectorTile;

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import com.onthegomap.flatmap.geo.GeoUtils;
import com.onthegomap.flatmap.reader.SimpleFeature;

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.newPoint;
import static org.junit.jupiter.api.Assertions.assertEquals;

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import com.onthegomap.flatmap.geo.GeoUtils;
import java.util.List;

Wyświetl plik

@ -1,12 +1,12 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.newPoint;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.layers.Place.getSortKey;
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_MAX;
import static com.onthegomap.flatmap.collection.FeatureGroup.SORT_KEY_MIN;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.layers.Place.getSortKey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static org.junit.jupiter.api.Assertions.assertEquals;

Wyświetl plik

@ -1,9 +1,9 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.newLineString;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import com.onthegomap.flatmap.FeatureCollector;
import com.onthegomap.flatmap.geo.GeometryException;

Wyświetl plik

@ -1,10 +1,10 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.newLineString;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.LAKE_CENTERLINE_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.LAKE_CENTERLINE_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import com.onthegomap.flatmap.TestUtils;
import com.onthegomap.flatmap.geo.GeoUtils;

Wyświetl plik

@ -1,9 +1,9 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.rectangle;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.WATER_POLYGON_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.OSM_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.WATER_POLYGON_SOURCE;
import com.onthegomap.flatmap.reader.SimpleFeature;
import java.util.List;

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.openmaptiles.layers;
package com.onthegomap.flatmap.basemap.layers;
import static com.onthegomap.flatmap.TestUtils.newLineString;
import static com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile.NATURAL_EARTH_SOURCE;
import static com.onthegomap.flatmap.basemap.BasemapProfile.NATURAL_EARTH_SOURCE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.flatmap.VectorTile;

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.openmaptiles.util;
package com.onthegomap.flatmap.basemap.util;
import static com.onthegomap.flatmap.TestUtils.assertSubmap;
import static com.onthegomap.flatmap.openmaptiles.util.LanguageUtils.containsOnlyLatinCharacters;
import static com.onthegomap.flatmap.basemap.util.LanguageUtils.containsOnlyLatinCharacters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;

Wyświetl plik

@ -0,0 +1,62 @@
package com.onthegomap.flatmap.basemap.util;
import static com.onthegomap.flatmap.geo.GeoUtils.point;
import static com.onthegomap.flatmap.util.Gzip.gzip;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.geo.TileCoord;
import com.onthegomap.flatmap.mbtiles.Mbtiles;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
public class VerifyMonacoTest {
private Mbtiles mbtiles;
@BeforeEach
public void setup() {
mbtiles = Mbtiles.newInMemoryDatabase();
}
@AfterEach
public void teardown() throws IOException {
mbtiles.close();
}
@Test
public void testEmptyFileInvalid() {
assertInvalid(mbtiles);
}
@Test
public void testEmptyTablesInvalid() {
mbtiles.createTables().addTileIndex();
assertInvalid(mbtiles);
}
@Test
public void testStilInvalidWithOneTile() throws IOException {
mbtiles.createTables().addTileIndex();
mbtiles.metadata().setName("name");
try (var writer = mbtiles.newBatchedTileWriter()) {
VectorTile tile = new VectorTile();
tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
"layer",
1,
VectorTile.encodeGeometry(point(0, 0)),
Map.of()
)));
writer.write(TileCoord.ofXYZ(0, 0, 0), gzip(tile.encode()));
}
assertInvalid(mbtiles);
}
private void assertInvalid(Mbtiles mbtiles) {
assertTrue(VerifyMonaco.verify(mbtiles).numErrors() > 0);
}
}

Wyświetl plik

@ -7,20 +7,20 @@
<artifactId>flatmap-benchmarks</artifactId>
<parent>
<groupId>com.onthegomap</groupId>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-parent</artifactId>
<version>0.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.onthegomap</groupId>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-core</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.onthegomap</groupId>
<artifactId>flatmap-openmaptiles</artifactId>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-basemap</artifactId>
<version>${project.parent.version}</version>
</dependency>
</dependencies>
@ -31,10 +31,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<!-- for fatjar assembly descriptor -->
<!-- for with-deps assembly descriptor -->
<dependencies>
<dependency>
<groupId>com.onthegomap</groupId>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-core</artifactId>
<version>${project.parent.version}</version>
</dependency>
@ -47,20 +47,17 @@
</manifestEntries>
</archive>
<descriptorRefs>
<descriptorRef>fatjar</descriptorRef>
<descriptorRef>with-deps</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<!-- we don't want to deploy this module -->
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>

Wyświetl plik

@ -4,9 +4,9 @@ import com.graphhopper.reader.ReaderElementUtils;
import com.graphhopper.reader.ReaderNode;
import com.graphhopper.reader.ReaderRelation;
import com.graphhopper.reader.ReaderWay;
import com.onthegomap.flatmap.basemap.BasemapProfile;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.expression.MultiExpression;
import com.onthegomap.flatmap.openmaptiles.OpenMapTilesProfile;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmInputFile;
import com.onthegomap.flatmap.stats.Stats;
@ -23,10 +23,10 @@ import org.locationtech.jts.geom.Geometry;
* Performance tests for {@link MultiExpression}. Times how long a sample of elements from an OSM input file take to
* match.
*/
public class OpenMapTilesMapping {
public class BasemapMapping {
public static void main(String[] args) throws IOException {
var profile = new OpenMapTilesProfile(Translations.nullProvider(List.of()), FlatmapConfig.defaults(),
var profile = new BasemapProfile(Translations.nullProvider(List.of()), FlatmapConfig.defaults(),
Stats.inMemory());
var random = new Random(0);
var input = new OsmInputFile(Path.of("data", "sources", "north-america_us_massachusetts.pbf"));

Wyświetl plik

@ -6,17 +6,20 @@
<artifactId>flatmap-core</artifactId>
<name>Flatmap Core</name>
<description>Flatmap Core</description>
<parent>
<groupId>com.onthegomap</groupId>
<groupId>com.onthegomap.flatmap</groupId>
<artifactId>flatmap-parent</artifactId>
<version>0.1-SNAPSHOT</version>
</parent>
<properties>
<graphhopper.version>2.3</graphhopper.version>
<geotools.version>25.0</geotools.version>
<graphhopper.version>2.4</graphhopper.version>
<geotools.version>26.0</geotools.version>
<log4j.version>2.14.1</log4j.version>
<prometheus.version>0.11.0</prometheus.version>
<prometheus.version>0.12.0</prometheus.version>
</properties>
<dependencies>
@ -28,7 +31,7 @@
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.18.0</version>
<version>1.18.2</version>
</dependency>
<dependency>
<groupId>org.geotools</groupId>
@ -43,17 +46,17 @@
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.34.0</version>
<version>3.36.0.3</version>
</dependency>
<dependency>
<groupId>org.msgpack</groupId>
<artifactId>msgpack-core</artifactId>
<version>0.8.22</version>
<version>0.9.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.30</version>
<version>1.7.32</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@ -118,7 +121,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>30.1.1-jre</version>
<version>31.0.1-jre</version>
</dependency>
</dependencies>
@ -134,7 +137,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.2</version>
<version>3.2.0</version>
<executions>
<execution>
<goals>
@ -142,8 +145,30 @@
</goals>
</execution>
</executions>
<configuration>
<!-- reduce the size of the sources jar a bit (leave behind monaco.osm.pbf file) -->
<excludes>
<exclude>*.wkb</exclude>
<exclude>*.mbtiles</exclude>
<exclude>*.sqlite</exclude>
<exclude>*.zip</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
<groupId>io.github.zlika</groupId>
<artifactId>reproducible-build-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

Wyświetl plik

@ -5,7 +5,7 @@ import com.onthegomap.flatmap.collection.LongLongMap;
import com.onthegomap.flatmap.config.Arguments;
import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.config.MbtilesMetadata;
import com.onthegomap.flatmap.mbiles.MbtilesWriter;
import com.onthegomap.flatmap.mbtiles.MbtilesWriter;
import com.onthegomap.flatmap.reader.NaturalEarthReader;
import com.onthegomap.flatmap.reader.ShapefileReader;
import com.onthegomap.flatmap.reader.osm.OsmInputFile;
@ -53,7 +53,7 @@ public class FlatmapRunner {
private static final Logger LOGGER = LoggerFactory.getLogger(FlatmapRunner.class);
private final List<Stage> stages = new ArrayList<>();
private final List<ToDownload> toDownload = new ArrayList<>();
private final List<Path> inputPaths = new ArrayList<>();
private final List<InputPath> inputPaths = new ArrayList<>();
private final Timers.Finishable overallTimer;
private final Arguments arguments;
private final Stats stats;
@ -135,7 +135,9 @@ public class FlatmapRunner {
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
* name_url} argument is not set. As a shortcut, can use "geofabrik:monaco" or
* "geofabrik:australia" shorthand to find an extract by name from <a
* href="https://download.geofabrik.de/">Geofabrik download site</a>.
* href="https://download.geofabrik.de/">Geofabrik download site</a> or "aws:latest" to download
* the latest {@code planet.osm.pbf} file from <a href="https://registry.opendata.aws/osm/">AWS
* Open Data Registry</a>.
* @return this runner instance for chaining
* @see OsmInputFile
* @see OsmReader
@ -333,7 +335,6 @@ public class FlatmapRunner {
* As long as {@code use_wikidata} is not set to false, then previously-downloaded wikidata translations will be
* loaded from the cache file so you can run with {@code fetch_wikidata=true} once, then without it each subsequent
* run to only download translations once.
* </ul>
*
* @param defaultWikidataCache Path to store downloaded wikidata name translations to, and to read them from on
* subsequent runs. Overridden by {@code wikidata_cache} argument value.
@ -439,12 +440,14 @@ public class FlatmapRunner {
ran = true;
MbtilesMetadata mbtilesMetadata = new MbtilesMetadata(profile, config.arguments());
if (onlyDownloadSources) {
if (arguments.getBoolean("help", "show arguments then exit", false)) {
System.exit(0);
} else if (onlyDownloadSources) {
// don't check files if not generating map
} else if (overwrite || config.forceOverwrite()) {
FileUtils.deleteFile(output);
} else if (Files.exists(output)) {
throw new IllegalArgumentException(output + " already exists, use force to overwrite.");
throw new IllegalArgumentException(output + " already exists, use the --force argument to overwrite.");
}
LOGGER.info(
@ -556,24 +559,26 @@ public class FlatmapRunner {
toDownload.add(new ToDownload(name, url, path));
}
}
inputPaths.add(path);
inputPaths.add(new InputPath(name, path));
return path;
}
private void download() {
var timer = stats.startStage("download");
Downloader downloader = Downloader.create(config());
Downloader downloader = Downloader.create(config(), stats());
for (ToDownload toDownload : toDownload) {
downloader.add(toDownload.id, toDownload.url, toDownload.path);
if (profile.caresAboutSource(toDownload.id)) {
downloader.add(toDownload.id, toDownload.url, toDownload.path);
}
}
downloader.run();
timer.stop();
}
private void ensureInputFilesExist() {
for (Path path : inputPaths) {
if (!Files.exists(path)) {
throw new IllegalArgumentException(path + " does not exist");
for (InputPath inputPath : inputPaths) {
if (profile.caresAboutSource(inputPath.id) && !Files.exists(inputPath.path)) {
throw new IllegalArgumentException(inputPath.path + " does not exist");
}
}
}
@ -586,4 +591,6 @@ public class FlatmapRunner {
}
private static record ToDownload(String id, String url, Path path) {}
private static record InputPath(String id, Path path) {}
}

Wyświetl plik

@ -90,7 +90,7 @@ public abstract class ForwardingProfile implements Profile {
if (handlers != null) {
for (var handler : handlers) {
handler.processFeature(sourceFeature, features);
// TODO extract common handling for expression-based filtering from openmaptiles to this
// TODO extract common handling for expression-based filtering from basemap to this
// common profile when we have another use-case for it.
}
}

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.mbiles.Mbtiles;
import com.onthegomap.flatmap.mbtiles.Mbtiles;
import com.onthegomap.flatmap.reader.SourceFeature;
import com.onthegomap.flatmap.reader.osm.OsmElement;
import com.onthegomap.flatmap.reader.osm.OsmRelationInfo;
@ -37,11 +37,12 @@ public interface Profile {
* passed along to {@link #processFeature(SourceFeature, FeatureCollector)} for any OSM element in that relation.
* <p>
* The result of this method is stored in memory.
* <p>
* The default implementation returns {@code null} to ignore all relations
*
* @param relation the OSM relation
* @return a list of relation info instances with information extracted from the relation to pass to {@link
* #processFeature(SourceFeature, FeatureCollector)}, or {@code null} to ignore.
* @implNote The default implementation returns {@code null} to ignore all relations
*/
default List<OsmRelationInfo> preprocessOsmRelation(OsmElement.Relation relation) {
return null;
@ -72,6 +73,8 @@ public interface Profile {
* linestrings/polygons.
* <p>
* Many threads invoke this method concurrently so ensure thread-safe access to any shared data structures.
* <p>
* The default implementation passes through input features unaltered
*
* @param layer the output layer name
* @param zoom zoom level of the tile
@ -80,10 +83,9 @@ public interface Profile {
* {@code null} if they should be ignored.
* @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
* the original input features, and continue processing other tiles
* @implSpec The default implementation passes through input features unaltered
*/
default List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
List<VectorTile.Feature> items) throws GeometryException {
List<VectorTile.Feature> items) throws GeometryException {
return items;
}
@ -126,8 +128,9 @@ public interface Profile {
/**
* Returns {@code true} to set {@code type="overlay"} in {@link Mbtiles} metadata otherwise sets {@code
* type="baselayer"}
* <p>
* The default implementation sets {@code type="baselayer"}
*
* @implSpec The default implementation sets {@code type="baselayer"}
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#metadata">MBTiles specification</a>
*/
default boolean isOverlay() {
@ -136,10 +139,11 @@ public interface Profile {
/**
* Defines whether {@link Wikidata} should fetch wikidata translations for the input element.
* <p>
* The default implementation returns {@code true} for all elements
*
* @param elem the input OSM element
* @return {@code true} to fetch wikidata translations for {@code elem}, {@code false} to ignore
* @implSpec the default implementation returns {@code true} for all elements
*/
default boolean caresAboutWikidataTranslation(OsmElement elem) {
return true;
@ -153,15 +157,16 @@ public interface Profile {
* @param next a consumer to pass finished map features to
*/
default void finish(String sourceName, FeatureCollector.Factory featureCollectors,
Consumer<FeatureCollector.Feature> next) {
Consumer<FeatureCollector.Feature> next) {
}
/**
* Returns true if this profile will use any of the elements from an input source.
* <p>
* The default implementation returns true.
*
* @param name the input source name
* @return {@code true} if this profile uses that source, {@code false} if it is safe to ignore
* @implSpec the default implementation returns true
*/
default boolean caresAboutSource(String name) {
return true;
@ -178,7 +183,7 @@ public interface Profile {
@Override
public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
List<VectorTile.Feature> items) {
List<VectorTile.Feature> items) {
return items;
}

Wyświetl plik

@ -75,15 +75,107 @@ public class VectorTile {
// TODO make these configurable
private static final int EXTENT = 4096;
private static final double SIZE = 256d;
private static final double SCALE = ((double) EXTENT) / SIZE;
private final Map<String, Layer> layers = new LinkedHashMap<>();
private static int[] getCommands(Geometry input) {
var encoder = new CommandEncoder();
private static int[] getCommands(Geometry input, int scale) {
var encoder = new CommandEncoder(scale);
encoder.accept(input);
return encoder.result.toArray();
}
/**
* Scales a geometry down by a factor of {@code 2^scale} without materializing an intermediate JTS geometry and
* returns the encoded result.
*/
private static int[] unscale(int[] commands, int scale, GeometryType geomType) {
IntArrayList result = new IntArrayList();
int geometryCount = commands.length;
int length = 0;
int command = 0;
int i = 0;
int inX = 0, inY = 0;
int outX = 0, outY = 0;
int startX = 0, startY = 0;
double scaleFactor = Math.pow(2, -scale);
int lengthIdx = 0;
int moveToIdx = 0;
int pointsInShape = 0;
boolean first = true;
while (i < geometryCount) {
if (length <= 0) {
length = commands[i++];
lengthIdx = result.size();
result.add(length);
command = length & ((1 << 3) - 1);
length = length >> 3;
}
if (length > 0) {
if (command == Command.MOVE_TO.value) {
// degenerate geometry, remove it from output entirely
if (!first && pointsInShape < geomType.minPoints()) {
int prevCommand = result.get(lengthIdx);
result.elementsCount = moveToIdx;
result.add(prevCommand);
// reset deltas
outX = startX;
outY = startY;
}
// keep track of size of next shape...
pointsInShape = 0;
startX = outX;
startY = outY;
moveToIdx = result.size() - 1;
}
first = false;
if (command == Command.CLOSE_PATH.value) {
pointsInShape++;
length--;
continue;
}
int dx = commands[i++];
int dy = commands[i++];
length--;
dx = zigZagDecode(dx);
dy = zigZagDecode(dy);
inX = inX + dx;
inY = inY + dy;
int nextX = (int) Math.round(inX * scaleFactor);
int nextY = (int) Math.round(inY * scaleFactor);
if (nextX == outX && nextY == outY && command == Command.LINE_TO.value) {
int commandLength = result.get(lengthIdx) - 8;
if (commandLength < 8) {
// get rid of lineto section if empty
result.elementsCount = lengthIdx;
} else {
result.set(lengthIdx, commandLength);
}
} else {
pointsInShape++;
int dxOut = nextX - outX;
int dyOut = nextY - outY;
result.add(
zigZagEncode(dxOut),
zigZagEncode(dyOut)
);
outX = nextX;
outY = nextY;
}
}
}
// degenerate geometry, remove it from output entirely
if (pointsInShape < geomType.minPoints()) {
result.elementsCount = moveToIdx;
}
return result.toArray();
}
private static int zigZagEncode(int n) {
// https://developers.google.com/protocol-buffers/docs/encoding#types
return (n << 1) ^ (n >> 31);
@ -94,9 +186,10 @@ public class VectorTile {
return ((n >> 1) ^ (-(n & 1)));
}
private static Geometry decodeCommands(GeometryType geomType, int[] commands) throws GeometryException {
private static Geometry decodeCommands(GeometryType geomType, int[] commands, int scale) throws GeometryException {
try {
GeometryFactory gf = GeoUtils.JTS_FACTORY;
double SCALE = (EXTENT << scale) / SIZE;
int x = 0;
int y = 0;
@ -219,7 +312,7 @@ public class VectorTile {
}
if (geometry == null) {
geometry = gf.createGeometryCollection(new Geometry[0]);
geometry = GeoUtils.EMPTY_GEOMETRY;
}
return geometry;
@ -284,7 +377,7 @@ public class VectorTile {
features.add(new Feature(
layerName,
feature.getId(),
new VectorGeometry(Ints.toArray(feature.getGeometryList()), GeometryType.valueOf(feature.getType())),
new VectorGeometry(Ints.toArray(feature.getGeometryList()), GeometryType.valueOf(feature.getType()), 0),
attrs
));
}
@ -303,7 +396,11 @@ public class VectorTile {
* @return the geometry type and command array for the encoded geometry
*/
public static VectorGeometry encodeGeometry(Geometry geometry) {
return new VectorGeometry(getCommands(geometry), GeometryType.valueOf(geometry));
return encodeGeometry(geometry, 0);
}
public static VectorGeometry encodeGeometry(Geometry geometry, int scale) {
return new VectorGeometry(getCommands(geometry, scale), GeometryType.valueOf(geometry), scale);
}
/**
@ -411,12 +508,28 @@ public class VectorTile {
/**
* A vector tile encoded as a list of commands according to the <a href="https://github.com/mapbox/vector-tile-spec/tree/master/2.1#43-geometry-encoding">vector
* tile specification</a>.
* <p>
* To encode extra precision in intermediate feature geometries, the geometry contained in {@code commands} is scaled
* to a tile extent of {@code EXTENT * 2^scale}, so when the {@code scale == 0} the extent is {@link #EXTENT} and when
* {@code scale == 2} the extent is 4x{@link #EXTENT}. Geometries must be scaled back to 0 using {@link #unscale()}
* before outputting to mbtiles.
*/
public static record VectorGeometry(int[] commands, GeometryType geomType) {
public static record VectorGeometry(int[] commands, GeometryType geomType, int scale) {
public VectorGeometry {
if (scale < 0) {
throw new IllegalArgumentException("scale can not be less than 0, got: " + scale);
}
}
/** Converts an encoded geometry back to a JTS geometry. */
public Geometry decode() throws GeometryException {
return decodeCommands(geomType, commands);
return decodeCommands(geomType, commands, scale);
}
/** Returns this encoded geometry, scaled back to 0, so it is safe to emit to mbtiles output. */
public VectorGeometry unscale() {
return scale == 0 ? this : new VectorGeometry(VectorTile.unscale(commands, scale, geomType), geomType, 0);
}
@Override
@ -491,10 +604,17 @@ public class VectorTile {
* new geometry.
*/
public Feature copyWithNewGeometry(Geometry newGeometry) {
return copyWithNewGeometry(encodeGeometry(newGeometry));
}
/**
* Returns a copy of this feature with {@code geometry} replaced with {@code newGeometry}.
*/
public Feature copyWithNewGeometry(VectorGeometry newGeometry) {
return new Feature(
layer,
id,
encodeGeometry(newGeometry),
newGeometry,
attrs,
group
);
@ -521,10 +641,15 @@ public class VectorTile {
private static class CommandEncoder {
final IntArrayList result = new IntArrayList();
private final double SCALE;
// Initial points use absolute locations, then subsequent points in a geometry use offsets so
// need to keep track of previous x/y location during the encoding.
int x = 0, y = 0;
CommandEncoder(int scale) {
this.SCALE = (EXTENT << scale) / SIZE;
}
static boolean shouldClosePath(Geometry geometry) {
return (geometry instanceof Polygon) || (geometry instanceof LinearRing);
}
@ -536,44 +661,47 @@ public class VectorTile {
void accept(Geometry geometry) {
if (geometry instanceof MultiLineString multiLineString) {
for (int i = 0; i < multiLineString.getNumGeometries(); i++) {
encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false);
encode(((LineString) multiLineString.getGeometryN(i)).getCoordinateSequence(), false, GeometryType.LINE);
}
} else if (geometry instanceof Polygon polygon) {
LineString exteriorRing = polygon.getExteriorRing();
encode(exteriorRing.getCoordinateSequence(), true);
encode(exteriorRing.getCoordinateSequence(), true, GeometryType.POLYGON);
for (int i = 0; i < polygon.getNumInteriorRing(); i++) {
LineString interiorRing = polygon.getInteriorRingN(i);
encode(interiorRing.getCoordinateSequence(), true);
encode(interiorRing.getCoordinateSequence(), true, GeometryType.LINE);
}
} else if (geometry instanceof MultiPolygon multiPolygon) {
for (int i = 0; i < multiPolygon.getNumGeometries(); i++) {
accept(multiPolygon.getGeometryN(i));
}
} else if (geometry instanceof LineString lineString) {
encode(lineString.getCoordinateSequence(), shouldClosePath(geometry));
encode(lineString.getCoordinateSequence(), shouldClosePath(geometry), GeometryType.LINE);
} else if (geometry instanceof Point point) {
encode(point.getCoordinateSequence(), false);
encode(point.getCoordinateSequence(), false, GeometryType.POINT);
} else if (geometry instanceof Puntal) {
encode(new CoordinateArraySequence(geometry.getCoordinates()), shouldClosePath(geometry),
geometry instanceof MultiPoint);
geometry instanceof MultiPoint, GeometryType.POINT);
} else {
LOGGER.warn("Unrecognized geometry type: " + geometry.getGeometryType());
}
}
void encode(CoordinateSequence cs, boolean closePathAtEnd) {
encode(cs, closePathAtEnd, false);
void encode(CoordinateSequence cs, boolean closePathAtEnd, GeometryType geomType) {
encode(cs, closePathAtEnd, false, geomType);
}
void encode(CoordinateSequence cs, boolean closePathAtEnd, boolean multiPoint) {
void encode(CoordinateSequence cs, boolean closePathAtEnd, boolean multiPoint, GeometryType geomType) {
if (cs.size() == 0) {
throw new IllegalArgumentException("empty geometry");
}
int startIdx = result.size();
int numPoints = 0;
int lineToIndex = 0;
int lineToLength = 0;
int startX = x;
int startY = y;
for (int i = 0; i < cs.size(); i++) {
@ -588,7 +716,7 @@ public class VectorTile {
int _y = (int) Math.round(cy * SCALE);
// prevent point equal to the previous
if (i > 0 && _x == x && _y == y) {
if (i > 0 && _x == x && _y == y && !multiPoint) {
lineToLength--;
continue;
}
@ -602,6 +730,7 @@ public class VectorTile {
// delta, then zigzag
result.add(zigZagEncode(_x - x));
result.add(zigZagEncode(_y - y));
numPoints++;
x = _x;
y = _y;
@ -628,6 +757,15 @@ public class VectorTile {
if (closePathAtEnd) {
result.add(commandAndLength(Command.CLOSE_PATH, 1));
numPoints++;
}
// degenerate geometry, skip emitting
if (numPoints < geomType.minPoints()) {
result.elementsCount = startIdx;
// reset deltas
x = startX;
y = startY;
}
}
}

Wyświetl plik

@ -4,18 +4,25 @@ import com.onthegomap.flatmap.util.FileUtils;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* An array of primitives backed by memory-mapped file.
*/
abstract class AppendStoreMmap implements AppendStore {
private static final Logger LOGGER = LoggerFactory.getLogger(AppendStoreMmap.class);
// writes are done using a BufferedOutputStream
final DataOutputStream outputStream;
final int segmentBits;
@ -79,6 +86,30 @@ abstract class AppendStoreMmap implements AppendStore {
channel.close();
}
if (segments != null) {
try {
// attempt to force-unmap the file, so we can delete it later
// https://stackoverflow.com/questions/2972986/how-to-unmap-a-file-from-memory-mapped-using-filechannel-in-java
Class<?> unsafeClass;
try {
unsafeClass = Class.forName("sun.misc.Unsafe");
} catch (Exception ex) {
unsafeClass = Class.forName("jdk.internal.misc.Unsafe");
}
Method clean = unsafeClass.getMethod("invokeCleaner", ByteBuffer.class);
clean.setAccessible(true);
Field theUnsafeField = unsafeClass.getDeclaredField("theUnsafe");
theUnsafeField.setAccessible(true);
Object theUnsafe = theUnsafeField.get(null);
for (int i = 0; i < segments.length; i++) {
var buffer = segments[i];
if (buffer != null) {
clean.invoke(theUnsafe, buffer);
segments[i] = null;
}
}
} catch (Exception e) {
LOGGER.info("Unable to unmap " + path + " " + e);
}
Arrays.fill(segments, null);
}
}

Wyświetl plik

@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import java.util.zip.Deflater;
@ -54,6 +55,8 @@ class ExternalMergeSort implements FeatureSort {
private final List<Chunk> chunks = new ArrayList<>();
private final boolean gzip;
private final FlatmapConfig config;
private final int readerLimit;
private final int writerLimit;
private Chunk currentChunk;
private volatile boolean sorted = false;
@ -83,6 +86,10 @@ class ExternalMergeSort implements FeatureSort {
"Not enough memory to use chunk size " + chunkSizeLimit + " only have " + memory);
}
this.workers = workers;
this.readerLimit = Math.max(1, config.arguments()
.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks", 6));
this.writerLimit = Math.max(1, config.arguments()
.getInteger("sort_max_writers", "maximum number of concurrent write threads to use when sorting chunks", 6));
LOGGER.info("Using merge sort feature map, chunk size=" + (chunkSizeLimit / 1_000_000) + "mb workers=" + workers);
try {
FileUtils.deleteDirectory(dir);
@ -153,6 +160,8 @@ class ExternalMergeSort implements FeatureSort {
}
}
var timer = stats.startStage("sort");
Semaphore readSemaphore = new Semaphore(readerLimit);
Semaphore writeSemaphore = new Semaphore(writerLimit);
AtomicLong reading = new AtomicLong(0);
AtomicLong writing = new AtomicLong(0);
AtomicLong sorting = new AtomicLong(0);
@ -161,10 +170,21 @@ class ExternalMergeSort implements FeatureSort {
var pipeline = WorkerPipeline.start("sort", stats)
.readFromTiny("item_queue", chunks)
.sinkToConsumer("worker", workers, chunk -> {
var toSort = time(reading, chunk::readAll);
time(sorting, toSort::sort);
time(writing, toSort::flush);
doneCounter.incrementAndGet();
try {
readSemaphore.acquire();
var toSort = time(reading, chunk::readAll);
readSemaphore.release();
time(sorting, toSort::sort);
writeSemaphore.acquire();
time(writing, toSort::flush);
writeSemaphore.release();
doneCounter.incrementAndGet();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
});
ProgressLoggers loggers = ProgressLoggers.create()

Wyświetl plik

@ -190,7 +190,7 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
packer.packInt(group.limit());
}
packer.packLong(vectorTileFeature.id());
packer.packByte(vectorTileFeature.geometry().geomType().asByte());
packer.packByte(encodeGeomTypeAndScale(vectorTileFeature.geometry()));
var attrs = vectorTileFeature.attrs();
packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count());
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
@ -238,7 +238,9 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
group = VectorTile.Feature.NO_GROUP;
}
long id = unpacker.unpackLong();
byte geomType = unpacker.unpackByte();
byte geomTypeAndScale = unpacker.unpackByte();
GeometryType geomType = decodeGeomType(geomTypeAndScale);
int scale = decodeScale(geomTypeAndScale);
int mapSize = unpacker.unpackMapHeader();
Map<String, Object> attrs = new HashMap<>(mapSize);
for (int i = 0; i < mapSize; i++) {
@ -263,7 +265,7 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
return new VectorTile.Feature(
layer,
id,
new VectorTile.VectorGeometry(commands, GeometryType.valueOf(geomType)),
new VectorTile.VectorGeometry(commands, geomType, scale),
attrs,
group
);
@ -272,6 +274,20 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
}
}
static GeometryType decodeGeomType(byte geomTypeAndScale) {
return GeometryType.valueOf((byte) (geomTypeAndScale & 0b111));
}
static int decodeScale(byte geomTypeAndScale) {
return (geomTypeAndScale & 0xff) >>> 3;
}
static byte encodeGeomTypeAndScale(VectorTile.VectorGeometry geometry) {
assert geometry.geomType().asByte() >= 0 && geometry.geomType().asByte() <= 8;
assert geometry.scale() >= 0 && geometry.scale() < (1 << 5);
return (byte) (geometry.geomType().asByte() | (geometry.scale() << 3));
}
/** Writes a serialized binary feature to intermediate storage. */
@Override
public void accept(SortableFeature entry) {
@ -412,12 +428,28 @@ public final class FeatureGroup implements Consumer<SortableFeature>, Iterable<F
return encoder;
}
private static void unscale(List<VectorTile.Feature> features) {
for (int i = 0; i < features.size(); i++) {
var feature = features.get(i);
if (feature != null) {
VectorTile.VectorGeometry geometry = feature.geometry();
if (geometry.scale() != 0) {
features.set(i, feature.copyWithNewGeometry(geometry.unscale()));
}
}
}
}
private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
List<VectorTile.Feature> features) {
try {
List<VectorTile.Feature> postProcessed = profile
.postProcessLayerFeatures(layer, tileCoord.z(), features);
features = postProcessed == null ? features : postProcessed;
// lines are stored using a higher precision so that rounding does not
// introduce artificial intersections between endpoints to confuse line merging,
// so we have to reduce the precision here, now that line merging is done.
unscale(features);
} catch (Throwable e) {
// failures in tile post-processing happen very late so err on the side of caution and
// log failures, only throwing when it's a fatal error

Wyświetl plik

@ -177,7 +177,7 @@ public class Arguments {
public Envelope bounds(String key, String description) {
String input = getArg(key);
Envelope result = null;
if ("world".equalsIgnoreCase(input)) {
if ("world".equalsIgnoreCase(input) || "planet".equalsIgnoreCase(input)) {
result = GeoUtils.WORLD_LAT_LON_BOUNDS;
} else if (input != null) {
double[] bounds = Stream.of(input.split("[\\s,]+")).mapToDouble(Double::parseDouble).toArray();
@ -306,4 +306,16 @@ public class Arguments {
logArgValue(key, description, parsed.get(ChronoUnit.SECONDS) + " seconds");
return parsed;
}
/**
* Returns an argument as long.
*
* @throws NumberFormatException if the argument cannot be parsed as an long
*/
public long getLong(String key, String description, long defaultValue) {
String value = getArg(key, Long.toString(defaultValue));
long parsed = Long.parseLong(value);
logArgValue(key, description, parsed);
return parsed;
}
}

Wyświetl plik

@ -20,6 +20,9 @@ public record FlatmapConfig(
String nodeMapType,
String nodeMapStorage,
String httpUserAgent,
Duration httpTimeout,
long downloadChunkSizeMB,
int downloadThreads,
double minFeatureSizeAtMaxZoom,
double minFeatureSizeBelowMaxZoom,
double simplifyToleranceAtMaxZoom,
@ -63,6 +66,9 @@ public record FlatmapConfig(
arguments.getString("nodemap_storage", "storage for location map: mmap or ram", "mmap"),
arguments.getString("http_user_agent", "User-Agent header to set when downloading files over HTTP",
"Flatmap downloader (https://github.com/onthegomap/flatmap)"),
arguments.getDuration("http_timeout", "Timeout to use when downloading files over HTTP", "30s"),
arguments.getLong("download_chunk_size_mb", "Size of file chunks to download in parallel in megabytes", 100),
arguments.getInteger("download_threads", "Number of parallel threads to use when downloading each file", 1),
arguments.getDouble("min_feature_size_at_max_zoom",
"Default value for the minimum size in tile pixels of features to emit at the maximum zoom level to allow for overzooming",
256d / 4096),

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.config;
import com.onthegomap.flatmap.Profile;
import com.onthegomap.flatmap.mbiles.MbtilesWriter;
import com.onthegomap.flatmap.mbtiles.MbtilesWriter;
/** Controls information that {@link MbtilesWriter} will write to the mbtiles metadata table. */
public record MbtilesMetadata(

Wyświetl plik

@ -18,7 +18,7 @@ import java.util.stream.Stream;
* Calling {@code toString()} on any expression will generate code that can be used to recreate an identical copy of the
* original expression, assuming that the generated code includes:
* <pre>{@code
* import static com.onthegomap.flatmap.openmaptiles.expression.Expression.*;
* import static com.onthegomap.flatmap.expression.Expression.*;
* }</pre>
*/
public interface Expression {

Wyświetl plik

@ -7,15 +7,17 @@ import org.locationtech.jts.geom.Puntal;
import vector_tile.VectorTileProto;
public enum GeometryType {
UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN),
POINT(VectorTileProto.Tile.GeomType.POINT),
LINE(VectorTileProto.Tile.GeomType.LINESTRING),
POLYGON(VectorTileProto.Tile.GeomType.POLYGON);
UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0),
POINT(VectorTileProto.Tile.GeomType.POINT, 1),
LINE(VectorTileProto.Tile.GeomType.LINESTRING, 2),
POLYGON(VectorTileProto.Tile.GeomType.POLYGON, 4);
private final VectorTileProto.Tile.GeomType protobufType;
private int minPoints;
GeometryType(VectorTileProto.Tile.GeomType protobufType) {
GeometryType(VectorTileProto.Tile.GeomType protobufType, int minPoints) {
this.protobufType = protobufType;
this.minPoints = minPoints;
}
public static GeometryType valueOf(Geometry geom) {
@ -45,4 +47,8 @@ public enum GeometryType {
public VectorTileProto.Tile.GeomType asProtobufType() {
return protobufType;
}
public int minPoints() {
return minPoints;
}
}

Wyświetl plik

@ -1,7 +1,7 @@
package com.onthegomap.flatmap.geo;
import com.onthegomap.flatmap.mbiles.Mbtiles;
import java.text.NumberFormat;
import com.onthegomap.flatmap.mbtiles.Mbtiles;
import com.onthegomap.flatmap.util.Format;
import javax.annotation.concurrent.Immutable;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.CoordinateXY;
@ -9,7 +9,7 @@ import org.locationtech.jts.geom.CoordinateXY;
/**
* The coordinate of a <a href="https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames">slippy map tile</a>.
* <p>
* In order to encode into a 32-bit integer, only zoom levels <= 14 are supported since we need 4 bits for the
* In order to encode into a 32-bit integer, only zoom levels {@code <= 14} are supported since we need 4 bits for the
* zoom-level, and 14 bits each for the x/y coordinates.
* <p>
* Tiles are ordered by z ascending, x ascending, y descending to match index ordering of {@link Mbtiles} sqlite
@ -19,7 +19,7 @@ import org.locationtech.jts.geom.CoordinateXY;
* @param x x coordinate of the tile where 0 is the western-most tile just to the east the international date line
* and 2^z-1 is the eastern-most tile
* @param y y coordinate of the tile where 0 is the northern-most tile and 2^z-1 is the southern-most tile
* @param z zoom level (<= 14)
* @param z zoom level ({@code <= 14})
*/
@Immutable
public record TileCoord(int encoded, int x, int y, int z) implements Comparable<TileCoord> {
@ -29,11 +29,6 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
// also need to remove hardcoded z14 limits
private static final int XY_MASK = (1 << 14) - 1;
private static final NumberFormat format = NumberFormat.getNumberInstance();
static {
format.setMaximumFractionDigits(5);
}
public TileCoord {
assert z <= 14;
@ -126,7 +121,7 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
/** Returns a URL that displays the openstreetmap data for this tile. */
public String getDebugUrl() {
Coordinate coord = getLatLon();
return "https://www.openstreetmap.org/#map=" + z + "/" + format.format(coord.y) + "/" + format.format(coord.x);
return Format.osmDebugUrl(z, coord);
}
/** Returns the pixel coordinate on this tile of a given latitude/longitude (assuming 256x256 px tiles). */

Wyświetl plik

@ -1,4 +1,4 @@
package com.onthegomap.flatmap.mbiles;
package com.onthegomap.flatmap.mbtiles;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
@ -539,7 +539,8 @@ public final class Mbtiles implements Closeable {
);
}
} catch (SQLException throwables) {
LOGGER.warn("Error retrieving metadata", throwables);
LOGGER.warn("Error retrieving metadata: " + throwables);
LOGGER.trace("Error retrieving metadata details: ", throwables);
}
return result;
}

Wyświetl plik

@ -1,4 +1,6 @@
package com.onthegomap.flatmap.mbiles;
package com.onthegomap.flatmap.mbtiles;
import static com.onthegomap.flatmap.util.Gzip.gzip;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.collection.FeatureGroup;
@ -6,6 +8,7 @@ import com.onthegomap.flatmap.config.FlatmapConfig;
import com.onthegomap.flatmap.config.MbtilesMetadata;
import com.onthegomap.flatmap.geo.TileCoord;
import com.onthegomap.flatmap.stats.Counter;
import com.onthegomap.flatmap.stats.ProcessInfo;
import com.onthegomap.flatmap.stats.ProgressLoggers;
import com.onthegomap.flatmap.stats.Stats;
import com.onthegomap.flatmap.stats.Timer;
@ -15,7 +18,6 @@ import com.onthegomap.flatmap.util.Format;
import com.onthegomap.flatmap.util.LayerStats;
import com.onthegomap.flatmap.worker.WorkQueue;
import com.onthegomap.flatmap.worker.WorkerPipeline;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayDeque;
@ -32,7 +34,6 @@ import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import java.util.zip.GZIPOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,8 +57,6 @@ public class MbtilesWriter {
private final LongAccumulator[] maxTileSizesByZoom;
private final FeatureGroup features;
private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
private final LongAccumulator maxBatchLength = new LongAccumulator(Long::max, 0);
private final LongAccumulator minBatchLength = new LongAccumulator(Long::min, Integer.MAX_VALUE);
private final MbtilesMetadata mbtilesMetadata;
private MbtilesWriter(FeatureGroup features, Mbtiles db, FlatmapConfig config, MbtilesMetadata mbtilesMeatadata,
@ -105,7 +104,13 @@ public class MbtilesWriter {
var pipeline = WorkerPipeline.start("mbtiles", stats);
int queueSize = 5_000;
// a larger tile queue size helps keep cores busy, but needs a lot of RAM
// 5k works fine with 100GB of RAM, so adjust the queue size down from there
// but no less than 100
int queueSize = Math.max(
100,
(int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)
);
WorkerPipeline<TileBatch> encodeBranch, writeBranch = null;
if (config.emitTilesInOrder()) {
@ -148,9 +153,9 @@ public class MbtilesWriter {
var loggers = ProgressLoggers.create()
.addRatePercentCounter("features", features.numFeaturesWritten(), writer.featuresProcessed)
.addFileSize(features)
.addRateCounter("tiles", writer::tilesEmitted)
.addFileSize(fileSize)
.add(" features ").addFileSize(features)
.newLine()
.addProcessStats()
.newLine()
@ -167,30 +172,18 @@ public class MbtilesWriter {
timer.stop();
}
private static byte[] gzipCompress(byte[] uncompressedData) throws IOException {
var bos = new ByteArrayOutputStream(uncompressedData.length);
try (var gzipOS = new GZIPOutputStream(bos)) {
gzipOS.write(uncompressedData);
}
return bos.toByteArray();
}
private String getLastTileLogDetails() {
TileCoord lastTile = lastTileWritten.get();
String blurb;
long minBatch = minBatchLength.getThenReset();
long maxBatch = maxBatchLength.getThenReset();
String batchSizeRange = (minBatch > 0 && maxBatch < Integer.MAX_VALUE) ? (minBatch + "-" + maxBatch) : "-";
if (lastTile == null) {
blurb = "n/a";
} else {
var extentForZoom = config.bounds().tileExtents().getForZoom(lastTile.z());
int zMinX = extentForZoom.minX();
int zMaxX = extentForZoom.maxX();
blurb = "%d/%d/%d (z%d %s%%) batch sizes: %s %s".formatted(
blurb = "%d/%d/%d (z%d %s%%) %s".formatted(
lastTile.z(), lastTile.x(), lastTile.y(),
lastTile.z(), (100 * (lastTile.x() + 1 - zMinX)) / (zMaxX - zMinX),
batchSizeRange,
lastTile.getDebugUrl()
);
}
@ -255,7 +248,7 @@ public class MbtilesWriter {
} else {
VectorTile en = tileFeatures.getVectorTileEncoder();
encoded = en.encode();
bytes = gzipCompress(encoded);
bytes = gzip(encoded);
last = tileFeatures;
lastEncoded = encoded;
lastBytes = bytes;
@ -303,7 +296,6 @@ public class MbtilesWriter {
while ((batch = tileBatches.get()) != null) {
Queue<Mbtiles.TileEntry> tiles = batch.out.get();
Mbtiles.TileEntry tile;
long batchSize = 0;
while ((tile = tiles.poll()) != null) {
TileCoord tileCoord = tile.tile();
assert lastTile == null || lastTile.compareTo(tileCoord) < 0 : "Tiles out of order %s before %s"
@ -322,14 +314,15 @@ public class MbtilesWriter {
batchedWriter.write(tile.tile(), tile.bytes());
stats.wroteTile(z, tile.bytes().length);
tilesByZoom[z].inc();
batchSize++;
}
maxBatchLength.accumulate(batchSize);
minBatchLength.accumulate(batchSize);
lastTileWritten.set(lastTile);
}
}
if (time != null) {
LOGGER.info("Finished z" + currentZ + " in " + time.stop());
}
if (config.optimizeDb()) {
db.vacuumAnalyze();
}

Wyświetl plik

@ -0,0 +1,236 @@
package com.onthegomap.flatmap.mbtiles;
import static com.onthegomap.flatmap.util.Gzip.gunzip;
import com.onthegomap.flatmap.VectorTile;
import com.onthegomap.flatmap.geo.GeometryException;
import com.onthegomap.flatmap.geo.TileCoord;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
import org.locationtech.jts.geom.Polygon;
/**
* A utility to verify the contents of an mbtiles file.
* <p>
* {@link #verify(Mbtiles)} does a basic set of checks that the schema is correct and contains a "name" attribute and at
* least one tile. Other classes can add more tests to it.
*/
public class Verify {
private static final String GOOD = "\u001B[32m✓\u001B[0m";
private static final String BAD = "\u001B[31m✕\u001B[0m";
private final List<Check> checks = new ArrayList<>();
private final Mbtiles mbtiles;
private Verify(Mbtiles mbtiles) {
this.mbtiles = mbtiles;
}
public static void main(String[] args) throws IOException {
try (var mbtiles = Mbtiles.newReadOnlyDatabase(Path.of(args[0]))) {
var result = Verify.verify(mbtiles);
result.print();
result.failIfErrors();
}
}
/**
* Returns the number of features in a layer inside a lat/lon bounding box with a geometry type and attributes.
*
* @param db the mbtiles file
* @param layer the layer to check
* @param zoom zoom level of tiles to check
* @param attrs partial set of attributes to filter features
* @param envelope lat/lon bounding box to limit check
* @param clazz {@link Geometry} subclass to limit
* @return number of features found
* @throws IOException if an error occurs reading from the file
* @throws GeometryException if an invalid geometry is encountered
*/
public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map<String, Object> attrs, Envelope envelope,
Class<? extends Geometry> clazz) throws IOException, GeometryException {
int num = 0;
for (var tileCoord : db.getAllTileCoords()) {
Envelope tileEnv = new Envelope();
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY()));
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY()));
if (tileCoord.z() == zoom) {
byte[] data = db.getTile(tileCoord);
for (var feature : decode(data)) {
if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) {
Geometry geometry = feature.geometry().decode();
num += getGeometryCounts(geometry, clazz);
}
}
}
}
return num;
}
private static int getGeometryCounts(Geometry geom, Class<? extends Geometry> clazz) {
int count = 0;
if (geom instanceof GeometryCollection geometryCollection) {
for (int i = 0; i < geometryCollection.getNumGeometries(); i++) {
count += getGeometryCounts(geometryCollection.getGeometryN(i), clazz);
}
} else if (clazz.isInstance(geom)) {
count = 1;
}
return count;
}
private static List<VectorTile.Feature> decode(byte[] zipped) {
try {
return VectorTile.decode(gunzip(zipped));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
/**
* Returns a verification result of a basic set of checks on an mbtiles file:
* <ul>
* <li>has a metadata and tiles table</li>
* <li>has a name metadata attribute</li>
* <li>has at least one tile</li>
* <li>all vector tile geometries are valid</li>
* </ul>
*/
public static Verify verify(Mbtiles mbtiles) {
Verify result = new Verify(mbtiles);
result.checkBasicStructure();
return result;
}
private static boolean isValid(Geometry geom) {
if (geom instanceof Polygon polygon) {
return polygon.isSimple();
} else if (geom instanceof GeometryCollection geometryCollection) {
for (int i = 0; i < geometryCollection.getNumGeometries(); i++) {
if (!isValid(geom.getGeometryN(i))) {
return false;
}
}
}
return true;
}
/**
* Adds a check to this verification result per zoom-level that succeeds if at least {@code minCount} features are
* found matching the provided criteria.
*
* @param bounds lat/lon bounding box to limit check
* @param layer the layer to check
* @param tags partial set of attributes to filter features
* @param minzoom min zoom level of tiles to check
* @param maxzoom max zoom level of tiles to check
* @param minCount minimum number of required features
* @param geometryType {@link Geometry} subclass to limit matches to
*/
public void checkMinFeatureCount(Envelope bounds, String layer, Map<String, Object> tags, int minzoom, int maxzoom,
int minCount, Class<? extends Geometry> geometryType) {
for (int z = minzoom; z <= maxzoom; z++) {
checkMinFeatureCount(bounds, layer, tags, z, minCount, geometryType);
}
}
/**
* Adds a check to this verification result that succeeds if at least {@code minCount} features are found matching the
* provided criteria.
*
* @param bounds lat/lon bounding box to limit check
* @param layer the layer to check
* @param tags partial set of attributes to filter features
* @param zoom zoom level of tiles to check
* @param minCount minimum number of required features
* @param geometryType {@link Geometry} subclass to limit matches to
*/
public void checkMinFeatureCount(Envelope bounds, String layer, Map<String, Object> tags, int zoom, int minCount,
Class<? extends Geometry> geometryType) {
checkWithMessage("at least %d %s %s features at z%d".formatted(minCount, layer, tags, zoom), () -> {
try {
int count = getNumFeatures(mbtiles, layer, zoom, tags, bounds, geometryType);
return count >= minCount ? Optional.empty() : Optional.of("found " + count);
} catch (IOException | GeometryException e) {
return Optional.of("error: " + e);
}
});
}
/** Logs verification results. */
public void print() {
for (Check check : checks) {
check.error.ifPresentOrElse(
error -> System.out.println(BAD + " " + check.name + ": " + error),
() -> System.out.println(GOOD + " " + check.name)
);
}
}
/** Exits with a nonzero exit code if there were any failures. */
public void failIfErrors() {
long errors = numErrors();
System.out.println(errors + " errors");
if (errors > 0) {
System.exit(1);
}
}
private void checkBasicStructure() {
check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name"));
check("contains at least one tile", () -> !mbtiles.getAllTileCoords().isEmpty());
checkWithMessage("all tiles are valid", () -> {
List<String> invalidTiles = mbtiles.getAllTileCoords().stream()
.flatMap(coord -> checkValidity(coord, decode(mbtiles.getTile(coord))).stream())
.toList();
return invalidTiles.isEmpty() ? Optional.empty() :
Optional.of(invalidTiles.size() + " invalid tiles: " + invalidTiles.stream().limit(5).toList());
});
}
private Optional<String> checkValidity(TileCoord coord, List<VectorTile.Feature> features) {
for (var feature : features) {
try {
Geometry geometry = feature.geometry().decode();
if (!isValid(geometry)) {
return Optional.of(coord + "/" + feature.layer());
}
} catch (GeometryException e) {
return Optional.of(coord + " error decoding " + feature.layer() + "feature");
}
}
return Optional.empty();
}
private void checkWithMessage(String name, Supplier<Optional<String>> check) {
try {
checks.add(new Check(name, check.get()));
} catch (Throwable e) {
checks.add(new Check(name, Optional.of(e.toString())));
}
}
private void check(String name, Supplier<Boolean> check) {
checkWithMessage(name, () -> check.get() ? Optional.empty() : Optional.of("false"));
}
public List<Check> results() {
return checks;
}
public long numErrors() {
return checks.stream().filter(check -> check.error.isPresent()).count();
}
public static record Check(String name, Optional<String> error) {}
}

Wyświetl plik

@ -27,7 +27,7 @@ import org.openstreetmap.osmosis.osmbinary.Osmformat.HeaderBlock;
/**
* An input file in {@code .osm.pbf} format.
*
* @see <a href="https://wiki.openstreetmap.org/wiki/PBF_Format>OSM PBF Format</a>
* @see <a href="https://wiki.openstreetmap.org/wiki/PBF_Format">OSM PBF Format</a>
*/
public class OsmInputFile implements Bounds.Provider, OsmSource {

Wyświetl plik

@ -135,19 +135,18 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
var pipeline = WorkerPipeline.start("osm_pass1", stats)
.fromGenerator("pbf", osmInputFile.read("pbfpass1", parseThreads))
.addBuffer("reader_queue", 50_000, 10_000)
// use only 1 thread since processPass1Element needs to be single-threaded
.sinkToConsumer("process", 1, this::processPass1Element);
var loggers = ProgressLoggers.create()
.addRateCounter("nodes", PASS1_NODES, true)
.addFileSize(nodeLocationDb)
.addFileSizeAndRam(nodeLocationDb)
.addRateCounter("ways", PASS1_WAYS, true)
.addRateCounter("rels", PASS1_RELATIONS, true)
.newLine()
.addProcessStats()
.addInMemoryObject("hppc", this)
.addThreadPoolStats("parse", pbfParsePrefix + "-pool")
.newLine()
.addThreadPoolStats("parse", pbfParsePrefix + "-pool")
.addPipelineStats(pipeline);
pipeline.awaitAndLog(loggers, config.logInterval());
timer.stop();
@ -156,6 +155,9 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
void processPass1Element(ReaderElement readerElement) {
// only a single thread calls this with elements ordered by ID, so it's safe to manipulate these
// shared data structures which are not thread safe
if (readerElement.getId() < 0) {
throw new IllegalArgumentException("Negative OSM element IDs not supported: " + readerElement);
}
if (readerElement instanceof ReaderNode node) {
PASS1_NODES.inc();
// TODO allow limiting node storage to only ones that profile cares about
@ -173,7 +175,8 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
relationInfoSizes.addAndGet(info.estimateMemoryUsageBytes());
for (ReaderRelation.Member member : rel.getMembers()) {
int type = member.getType();
if (type == ReaderRelation.Member.WAY || type == ReaderRelation.Member.RELATION) {
// TODO handle nodes in relations and super-relations
if (type == ReaderRelation.Member.WAY) {
wayToRelations.put(member.getRef(), encodeRelationMembership(member.getRole(), rel.getId()));
}
}
@ -198,8 +201,9 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
*/
public void pass2(FeatureGroup writer, FlatmapConfig config) {
var timer = stats.startStage("osm_pass2");
int readerThreads = Math.max(config.threads() / 4, 1);
int processThreads = config.threads() - 1;
int threads = config.threads();
int readerThreads = Math.max(threads / 4, 1);
int processThreads = threads - (threads >= 4 ? 1 : 0);
Counter.MultiThreadCounter nodesProcessed = Counter.newMultiThreadCounter();
Counter.MultiThreadCounter waysProcessed = Counter.newMultiThreadCounter();
Counter.MultiThreadCounter relsProcessed = Counter.newMultiThreadCounter();
@ -263,7 +267,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
var logger = ProgressLoggers.create()
.addRatePercentCounter("nodes", PASS1_NODES.get(), nodesProcessed)
.addFileSize(nodeLocationDb)
.addFileSizeAndRam(nodeLocationDb)
.addRatePercentCounter("ways", PASS1_WAYS.get(), waysProcessed)
.addRatePercentCounter("rels", PASS1_RELATIONS.get(), relsProcessed)
.addRateCounter("features", writer::numFeaturesWritten)

Wyświetl plik

@ -111,7 +111,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
TileCoord tile = entry.getKey();
List<List<CoordinateSequence>> result = entry.getValue();
Geometry geom = GeometryCoordinateSequences.reassemblePoints(result);
encodeAndEmitFeature(feature, id, attrs, tile, geom, groupInfo);
encodeAndEmitFeature(feature, id, attrs, tile, geom, groupInfo, 0);
emitted++;
}
stats.emittedFeatures(zoom, feature.getLayer(), emitted);
@ -121,13 +121,13 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
}
private void encodeAndEmitFeature(FeatureCollector.Feature feature, long id, Map<String, Object> attrs,
TileCoord tile, Geometry geom, RenderedFeature.Group groupInfo) {
TileCoord tile, Geometry geom, RenderedFeature.Group groupInfo, int scale) {
consumer.accept(new RenderedFeature(
tile,
new VectorTile.Feature(
feature.getLayer(),
id,
VectorTile.encodeGeometry(geom),
VectorTile.encodeGeometry(geom, scale),
attrs,
groupInfo == null ? VectorTile.Feature.NO_GROUP : groupInfo.group()
),
@ -198,6 +198,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
List<List<CoordinateSequence>> geoms = entry.getValue();
Geometry geom;
int scale = 0;
if (feature.isPolygon()) {
geom = GeometryCoordinateSequences.reassemblePolygons(geoms);
/*
@ -214,10 +215,18 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature> {
geom = geom.reverse();
} else {
geom = GeometryCoordinateSequences.reassembleLineStrings(geoms);
// Store lines with extra precision (2^scale) in intermediate feature storage so that
// rounding does not introduce artificial endpoint intersections and confuse line merge
// post-processing. Features need to be "unscaled" in FeatureGroup after line merging,
// and before emitting to output mbtiles.
scale = Math.max(config.maxzoom(), 14) - zoom;
// need 14 bits to represent tile coordinates (4096 * 2 for buffer * 2 for zig zag encoding)
// so cap the scale factor to avoid overflowing 32-bit integer space
scale = Math.min(31 - 14, scale);
}
if (!geom.isEmpty()) {
encodeAndEmitFeature(feature, id, attrs, tile, geom, null);
encodeAndEmitFeature(feature, id, attrs, tile, geom, null, scale);
emitted++;
}
} catch (GeometryException e) {

Wyświetl plik

@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.DoubleFunction;
import java.util.function.Function;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory;
/**
* Logs the progress of a long-running task (percent complete, queue sizes, CPU and memory usage, etc.)
*/
@SuppressWarnings("UnusedReturnValue")
@SuppressWarnings({"UnusedReturnValue", "unused"})
public class ProgressLoggers {
private static final String COLOR_RESET = "\u001B[0m";
@ -123,6 +124,23 @@ public class ProgressLoggers {
* process.
*/
public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue) {
return addRatePercentCounter(name, total, getValue, n -> Format.formatNumeric(n, true));
}
/**
* Adds "name: [ numCompleted pctComplete% rate/s ]" to the logger where {@code total} is the total number of bytes to
* process.
*/
public ProgressLoggers addStorageRatePercentCounter(String name, long total, LongSupplier getValue) {
return addRatePercentCounter(name, total, getValue, n -> Format.formatStorage(n, true));
}
/**
* Adds "name: [ numCompleted pctComplete% rate/s ]" to the logger where {@code total} is the total number of items to
* process.
*/
public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue,
Function<Number, String> format) {
// if there's no total, we can't show progress so fall back to rate logger instead
if (total == 0) {
return addRateCounter(name, getValue, true);
@ -140,8 +158,8 @@ public class ProgressLoggers {
last.set(valueNow);
lastTime.set(now);
String result =
"[ " + Format.formatNumeric(valueNow, true) + " " + padLeft(formatPercent(1f * valueNow / total), 4)
+ " " + Format.formatNumeric(valueDiff / timeDiff, true) + "/s ]";
"[ " + format.apply(valueNow) + " " + padLeft(formatPercent(1f * valueNow / total), 4)
+ " " + format.apply(valueDiff / timeDiff) + "/s ]";
return valueDiff > 0 ? green(result) : result;
}));
return this;
@ -203,6 +221,20 @@ public class ProgressLoggers {
return add(() -> " " + padRight(formatStorage(longSupplier.diskUsageBytes(), false), 5));
}
/** Adds the total of disk and memory usage of {@code thing}. */
public <T extends DiskBacked & MemoryEstimator.HasEstimate> ProgressLoggers addFileSizeAndRam(T thing) {
return add(() -> {
long bytes = thing.diskUsageBytes() + thing.estimateMemoryUsageBytes();
return " " + padRight(formatStorage(bytes, false), 5);
});
}
/** Adds the current size of a file on disk. */
public ProgressLoggers addFileSize(String name, DiskBacked file) {
loggers.add(new ProgressLogger(name, () -> formatStorage(file.diskUsageBytes(), true)));
return this;
}
/**
* Adds the average number of CPUs and % time in GC since last log along with memory usage, total memory, and memory
* used after last GC to the output.

Wyświetl plik

@ -167,8 +167,8 @@ class PrometheusStats implements Stats {
}
@Override
public void monitorFile(String name, Path path) {
filesToMonitor.put(name, path);
public Map<String, Path> monitoredFiles() {
return filesToMonitor;
}
@Override

Wyświetl plik

@ -2,12 +2,17 @@ package com.onthegomap.flatmap.stats;
import static io.prometheus.client.Collector.NANOSECONDS_PER_SECOND;
import com.onthegomap.flatmap.util.FileUtils;
import com.onthegomap.flatmap.util.Format;
import com.onthegomap.flatmap.util.LogUtil;
import com.onthegomap.flatmap.util.MemoryEstimator;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A utility that collects and reports more detailed statistics about the JVM and running tasks than logs can convey.
@ -31,9 +36,21 @@ public interface Stats extends AutoCloseable {
return PrometheusStats.createAndStartPushing(destination, job, interval);
}
/** Logs top-level stats at the end of a job like the amount of user and CPU time that each task has taken. */
/**
* Logs top-level stats at the end of a job like the amount of user and CPU time that each task has taken, and size of
* each monitored file.
*/
default void printSummary() {
Logger LOGGER = LoggerFactory.getLogger(getClass());
LOGGER.info("-".repeat(40));
timers().printSummary();
LOGGER.info("-".repeat(40));
for (var entry : monitoredFiles().entrySet()) {
long size = FileUtils.size(entry.getValue());
if (size > 0) {
LOGGER.info("\t" + entry.getKey() + "\t" + Format.formatStorage(size, false) + "B");
}
}
}
/**
@ -65,13 +82,18 @@ public interface Stats extends AutoCloseable {
/** Returns the timers for all stages started with {@link #startStage(String)}. */
Timers timers();
/** Returns all the files being monitored. */
Map<String, Path> monitoredFiles();
/** Adds a stat that will track the size of a file or directory located at {@code path}. */
void monitorFile(String name, Path path);
default void monitorFile(String name, Path path) {
monitoredFiles().put(name, path);
}
/** Adds a stat that will track the estimated in-memory size of {@code object}. */
void monitorInMemoryObject(String name, MemoryEstimator.HasEstimate object);
/** Tracks a stat with {@code name} that always has a constant {@value}. */
/** Tracks a stat with {@code name} that always has a constant {@code value}. */
default void gauge(String name, Number value) {
gauge(name, () -> value);
}
@ -125,6 +147,7 @@ public interface Stats extends AutoCloseable {
}
private final Timers timers = new Timers();
private final Map<String, Path> monitoredFiles = new ConcurrentSkipListMap<>();
@Override
public void wroteTile(int zoom, int bytes) {
@ -136,7 +159,8 @@ public interface Stats extends AutoCloseable {
}
@Override
public void monitorFile(String name, Path path) {
public Map<String, Path> monitoredFiles() {
return monitoredFiles;
}
@Override

Wyświetl plik

@ -17,7 +17,6 @@ public class Timers {
private final Map<String, Timer> timers = Collections.synchronizedMap(new LinkedHashMap<>());
public void printSummary() {
LOGGER.info("-".repeat(50));
for (var entry : all().entrySet()) {
LOGGER.info("\t" + entry.getKey() + "\t" + entry.getValue().elapsed());
}
@ -27,7 +26,7 @@ public class Timers {
Timer timer = Timer.start();
timers.put(name, timer);
LOGGER.info("Starting...");
return () -> LOGGER.info("Finished in " + timers.get(name).stop() + "\n");
return () -> LOGGER.info("Finished in " + timers.get(name).stop() + System.lineSeparator());
}
/** Returns a snapshot of all timers currently running. Will not reflect timers that start after it's called. */

Some files were not shown because too many files have changed in this diff Show More