kopia lustrzana https://github.com/onthegomap/planetiler
Expose pmtiles writer (#520)
rodzic
9945ad406e
commit
74db638dbc
|
@ -85,7 +85,7 @@ jobs:
|
|||
run: mv target/*with-deps.jar ./run.jar
|
||||
working-directory: planetiler-examples
|
||||
- name: Run
|
||||
run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles
|
||||
run: java -jar run.jar --osm-path=../planetiler-core/src/test/resources/monaco-latest.osm.pbf --output=data/out.mbtiles
|
||||
working-directory: planetiler-examples
|
||||
- name: Verify
|
||||
run: java -cp run.jar com.onthegomap.planetiler.mbtiles.Verify data/out.mbtiles
|
||||
|
|
|
@ -22,73 +22,73 @@ jobs:
|
|||
timeout-minutes: 20
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: 'Cancel previous runs'
|
||||
uses: styfle/cancel-workflow-action@0.11.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- name: 'Checkout branch'
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: branch
|
||||
submodules: true
|
||||
- name: 'Checkout base'
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: base
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
submodules: true
|
||||
- name: Cache data/sources
|
||||
uses: ./branch/.github/cache-sources-action
|
||||
with:
|
||||
basedir: branch
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: '14'
|
||||
- run: npm install -g strip-ansi-cli@3.0.2
|
||||
- name: 'Cancel previous runs'
|
||||
uses: styfle/cancel-workflow-action@0.11.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- name: 'Checkout branch'
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: branch
|
||||
submodules: true
|
||||
- name: 'Checkout base'
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: base
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
submodules: true
|
||||
- name: Cache data/sources
|
||||
uses: ./branch/.github/cache-sources-action
|
||||
with:
|
||||
basedir: branch
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: '14'
|
||||
- run: npm install -g strip-ansi-cli@3.0.2
|
||||
|
||||
- name: 'Build branch'
|
||||
run: ./scripts/build.sh
|
||||
working-directory: branch
|
||||
- name: 'Build base'
|
||||
run: ./scripts/build.sh
|
||||
working-directory: base
|
||||
- name: 'Build branch'
|
||||
run: ./scripts/build.sh
|
||||
working-directory: branch
|
||||
- name: 'Build base'
|
||||
run: ./scripts/build.sh
|
||||
working-directory: base
|
||||
|
||||
- name: 'Download data'
|
||||
run: |
|
||||
set -eo pipefail
|
||||
cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
|
||||
cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
|
||||
- name: 'Download data'
|
||||
run: |
|
||||
set -eo pipefail
|
||||
cp base/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
|
||||
cp branch/planetiler-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}"
|
||||
|
||||
- name: 'Store build info'
|
||||
run: |
|
||||
mkdir build-info
|
||||
echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha
|
||||
echo "${{ github.sha }}" > build-info/branch_sha
|
||||
echo "${{ github.event.number }}" > build-info/pull_request_number
|
||||
- name: 'Store build info'
|
||||
run: |
|
||||
mkdir build-info
|
||||
echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha
|
||||
echo "${{ github.sha }}" > build-info/branch_sha
|
||||
echo "${{ github.event.number }}" > build-info/pull_request_number
|
||||
|
||||
- name: 'Run branch'
|
||||
run: |
|
||||
rm -rf data/out.mbtiles data/tmp
|
||||
cp branch/planetiler-dist/target/*with-deps.jar run.jar
|
||||
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
|
||||
ls -alh run.jar | tee -a log
|
||||
cat log | strip-ansi > build-info/branchlogs.txt
|
||||
- name: 'Run base'
|
||||
run: |
|
||||
rm -rf data/out.mbtiles data/tmp
|
||||
cp base/planetiler-dist/target/*with-deps.jar run.jar
|
||||
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log
|
||||
ls -alh run.jar | tee -a log
|
||||
cat log | strip-ansi > build-info/baselogs.txt
|
||||
- name: 'Run branch'
|
||||
run: |
|
||||
rm -rf data/out.mbtiles data/tmp
|
||||
cp branch/planetiler-dist/target/*with-deps.jar run.jar
|
||||
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --output=data/out.mbtiles 2>&1 | tee log
|
||||
ls -alh run.jar | tee -a log
|
||||
cat log | strip-ansi > build-info/branchlogs.txt
|
||||
- name: 'Run base'
|
||||
run: |
|
||||
rm -rf data/out.mbtiles data/tmp
|
||||
cp base/planetiler-dist/target/*with-deps.jar run.jar
|
||||
java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --output=data/out.mbtiles 2>&1 | tee log
|
||||
ls -alh run.jar | tee -a log
|
||||
cat log | strip-ansi > build-info/baselogs.txt
|
||||
|
||||
- name: 'Upload build-info'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: build-info
|
||||
path: ./build-info
|
||||
- name: 'Upload build-info'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: build-info
|
||||
path: ./build-info
|
||||
|
|
|
@ -50,7 +50,7 @@ java -Xmx110g \
|
|||
--download-threads=10 --download-chunk-size-mb=1000 \
|
||||
`# Also download name translations from wikidata` \
|
||||
--fetch-wikidata \
|
||||
--mbtiles=output.mbtiles \
|
||||
--output=output.mbtiles \
|
||||
`# Store temporary node locations in memory` \
|
||||
--nodemap-type=array --storage=ram
|
||||
```
|
||||
|
@ -67,7 +67,7 @@ java -Xmx20g \
|
|||
--download-threads=10 --download-chunk-size-mb=1000 \
|
||||
`# Also download name translations from wikidata` \
|
||||
--fetch-wikidata \
|
||||
--mbtiles=output.mbtiles \
|
||||
--output=output.mbtiles \
|
||||
`# Store temporary node locations at fixed positions in a memory-mapped file` \
|
||||
--nodemap-type=array --storage=mmap
|
||||
```
|
||||
|
@ -103,7 +103,7 @@ java -Xmx100g \
|
|||
--download-threads=10 --download-chunk-size-mb=1000 \
|
||||
`# Also download name translations from wikidata` \
|
||||
--fetch-wikidata \
|
||||
--mbtiles=output.mbtiles \
|
||||
--output=output.mbtiles \
|
||||
--nodemap-type=sparsearray --nodemap-storage=ram 2>&1 | tee logs.txt
|
||||
```
|
||||
|
||||
|
|
14
README.md
14
README.md
|
@ -8,9 +8,10 @@ or database.
|
|||
|
||||
Vector tiles contain raw point, line, and polygon geometries that clients like [MapLibre](https://github.com/maplibre)
|
||||
can use to render custom maps in the browser, native apps, or on a server. Planetiler packages tiles into
|
||||
an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite) file that can be served using
|
||||
tools like [TileServer GL](https://github.com/maptiler/tileserver-gl) or even
|
||||
[queried directly from the browser](https://github.com/phiresky/sql.js-httpvfs).
|
||||
an [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) (sqlite)
|
||||
or [PMTiles](https://github.com/protomaps/PMTiles) file that can be served using tools
|
||||
like [TileServer GL](https://github.com/maptiler/tileserver-gl) or [Martin](https://github.com/maplibre/martin) or
|
||||
even [queried directly from the browser](https://github.com/protomaps/PMTiles/tree/main/js).
|
||||
See [awesome-vector-tiles](https://github.com/mapbox/awesome-vector-tiles) for more projects that work with data in this
|
||||
format.
|
||||
|
||||
|
@ -87,7 +88,7 @@ Using [Node.js](https://nodejs.org/en/download/):
|
|||
|
||||
```bash
|
||||
npm install -g tileserver-gl-light
|
||||
tileserver-gl-light --mbtiles data/output.mbtiles
|
||||
tileserver-gl-light data/output.mbtiles
|
||||
```
|
||||
|
||||
Or using [Docker](https://docs.docker.com/get-docker/):
|
||||
|
@ -100,6 +101,8 @@ Then open http://localhost:8080 to view tiles.
|
|||
|
||||
Some common arguments:
|
||||
|
||||
- `--output` tells planetiler where to write output to, and what format to write it in. For
|
||||
example `--output=australia.pmtiles` creates a pmtiles archive named `australia.pmtiles`.
|
||||
- `--download` downloads input sources automatically and `--only-download` exits after downloading
|
||||
- `--area=monaco` downloads a `.osm.pbf` extract from [Geofabrik](https://download.geofabrik.de/)
|
||||
- `--osm-path=path/to/file.osm.pbf` points Planetiler at an existing OSM extract on disk
|
||||
|
@ -209,6 +212,8 @@ download regularly-updated tilesets.
|
|||
OpenStreetMap [.osm.pbf](https://wiki.openstreetmap.org/wiki/PBF_Format),
|
||||
[`geopackage`](https://www.geopackage.org/),
|
||||
and [Esri Shapefiles](https://en.wikipedia.org/wiki/Shapefile) data sources
|
||||
- Writes to [MBTiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md) or
|
||||
or [PMTiles](https://github.com/protomaps/PMTiles) output.
|
||||
- Java-based [Profile API](planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java) to customize how source
|
||||
elements map to vector tile features, and post-process generated tiles
|
||||
using [JTS geometry utilities](https://github.com/locationtech/jts)
|
||||
|
@ -328,6 +333,7 @@ Planetiler is made possible by these awesome open source projects:
|
|||
- [cel-java](https://github.com/projectnessie/cel-java) for the Java implementation of
|
||||
Google's [Common Expression Language](https://github.com/google/cel-spec) that powers dynamic expressions embedded in
|
||||
schema config files.
|
||||
- [PMTiles](https://github.com/protomaps/PMTiles) optimized tile storage format
|
||||
|
||||
See [NOTICE.md](NOTICE.md) for a full list and license details.
|
||||
|
||||
|
|
|
@ -66,9 +66,9 @@ public class BenchmarkMbtilesWriter {
|
|||
for (int repetition = 0; repetition < repetitions; repetition++) {
|
||||
|
||||
Path outputPath = getTempOutputPath();
|
||||
try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.compactDb())) {
|
||||
try (var mbtiles = Mbtiles.newWriteToFileDatabase(outputPath, config.arguments())) {
|
||||
|
||||
if (config.skipIndexCreation()) {
|
||||
if (mbtiles.skipIndexCreation()) {
|
||||
mbtiles.createTablesWithoutIndexes();
|
||||
} else {
|
||||
mbtiles.createTablesWithIndexes();
|
||||
|
|
|
@ -1,14 +1,15 @@
|
|||
package com.onthegomap.planetiler;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileArchiveConfig;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.archive.TileArchives;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.collection.LongLongMap;
|
||||
import com.onthegomap.planetiler.collection.LongLongMultimap;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
import com.onthegomap.planetiler.reader.GeoPackageReader;
|
||||
import com.onthegomap.planetiler.reader.NaturalEarthReader;
|
||||
import com.onthegomap.planetiler.reader.ShapefileReader;
|
||||
|
@ -85,7 +86,7 @@ public class Planetiler {
|
|||
private final PlanetilerConfig config;
|
||||
private FeatureGroup featureGroup;
|
||||
private OsmInputFile osmInputFile;
|
||||
private Path output;
|
||||
private TileArchiveConfig output;
|
||||
private boolean overwrite = false;
|
||||
private boolean ran = false;
|
||||
// most common OSM languages
|
||||
|
@ -151,8 +152,8 @@ public class Planetiler {
|
|||
*
|
||||
* @param name string to use in stats and logs to identify this stage
|
||||
* @param defaultPath path to the input file to use if {@code name_path} argument is not set
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set. As a shortcut, can use "geofabrik:monaco" or
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
|
||||
* {@code name_url} argument is not set. As a shortcut, can use "geofabrik:monaco" or
|
||||
* "geofabrik:australia" shorthand to find an extract by name from
|
||||
* <a href="https://download.geofabrik.de/">Geofabrik download site</a> or "aws:latest" to download
|
||||
* the latest {@code planet.osm.pbf} file from <a href="https://registry.opendata.aws/osm/">AWS
|
||||
|
@ -183,10 +184,11 @@ public class Planetiler {
|
|||
),
|
||||
ifSourceUsed(name, () -> {
|
||||
var header = osmInputFile.getHeader();
|
||||
tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant());
|
||||
tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationseq",
|
||||
tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationtime", header.instant());
|
||||
tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationseq",
|
||||
header.osmosisReplicationSequenceNumber());
|
||||
tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl());
|
||||
tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationurl",
|
||||
header.osmosisReplicationBaseUrl());
|
||||
try (
|
||||
var nodeLocations =
|
||||
LongLongMap.from(config.nodeMapType(), config.nodeMapStorage(), nodeDbPath, config.nodeMapMadvise());
|
||||
|
@ -253,8 +255,8 @@ public class Planetiler {
|
|||
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments. Can be a
|
||||
* {@code .shp} file with other shapefile components in the same directory, or a {@code .zip} file
|
||||
* containing the shapefile components.
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
|
||||
* {@code name_url} argument is not set
|
||||
* @return this runner instance for chaining
|
||||
* @see ShapefileReader
|
||||
* @see Downloader
|
||||
|
@ -327,8 +329,8 @@ public class Planetiler {
|
|||
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments. Can be a
|
||||
* {@code .shp} file with other shapefile components in the same directory, or a {@code .zip} file
|
||||
* containing the shapefile components.
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
|
||||
* {@code name_url} argument is not set
|
||||
* @return this runner instance for chaining
|
||||
* @see ShapefileReader
|
||||
* @see Downloader
|
||||
|
@ -362,8 +364,8 @@ public class Planetiler {
|
|||
* {@link org.geotools.referencing.CRS#decode(String)}
|
||||
* @param name string to use in stats and logs to identify this stage
|
||||
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
|
||||
* {@code name_url} argument is not set
|
||||
* @return this runner instance for chaining
|
||||
* @see GeoPackageReader
|
||||
* @see Downloader
|
||||
|
@ -399,8 +401,8 @@ public class Planetiler {
|
|||
*
|
||||
* @param name string to use in stats and logs to identify this stage
|
||||
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
|
||||
* {@code name_url} argument is not set
|
||||
* @return this runner instance for chaining
|
||||
* @see GeoPackageReader
|
||||
* @see Downloader
|
||||
|
@ -415,12 +417,12 @@ public class Planetiler {
|
|||
* To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to
|
||||
* override the download URL set {@code name_url=http://url/of/natural_earth.zip}.
|
||||
*
|
||||
* @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
|
||||
* @param name string to use in stats and logs to identify this stage
|
||||
* @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the
|
||||
* {@code .sqlite} file or a {@code .zip} file containing the sqlite file.
|
||||
* @return this runner instance for chaining
|
||||
* @see NaturalEarthReader
|
||||
* @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
|
||||
*/
|
||||
@Deprecated(forRemoval = true)
|
||||
public Planetiler addNaturalEarthSource(String name, Path defaultPath) {
|
||||
|
@ -436,16 +438,15 @@ public class Planetiler {
|
|||
* To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to
|
||||
* override the download URL set {@code name_url=http://url/of/natural_earth.zip}.
|
||||
*
|
||||
* @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
|
||||
*
|
||||
* @param name string to use in stats and logs to identify this stage
|
||||
* @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the
|
||||
* {@code .sqlite} file or a {@code .zip} file containing the sqlite file.
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
|
||||
* name_url} argument is not set
|
||||
* @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
|
||||
* {@code name_url} argument is not set
|
||||
* @return this runner instance for chaining
|
||||
* @see NaturalEarthReader
|
||||
* @see Downloader
|
||||
* @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
|
||||
*/
|
||||
@Deprecated(forRemoval = true)
|
||||
public Planetiler addNaturalEarthSource(String name, Path defaultPath, String defaultUrl) {
|
||||
|
@ -540,35 +541,69 @@ public class Planetiler {
|
|||
}
|
||||
|
||||
/**
|
||||
* Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists.
|
||||
* <p>
|
||||
* To override the location of the file, set {@code argument=newpath} in the arguments.
|
||||
* Sets the location of the output archive to write rendered tiles to.
|
||||
*
|
||||
* @param argument the argument key to check for an override to {@code fallback}
|
||||
* @param fallback the fallback value if {@code argument} is not set in arguments
|
||||
* @return this runner instance for chaining
|
||||
* @see TileArchiveWriter
|
||||
* @deprecated Use {@link #setOutput(String)} instead
|
||||
*/
|
||||
@Deprecated(forRemoval = true)
|
||||
public Planetiler setOutput(String argument, Path fallback) {
|
||||
this.output = arguments.file(argument, "output tile archive", fallback);
|
||||
this.output =
|
||||
TileArchiveConfig
|
||||
.from(arguments.getString("output|" + argument, "output tile archive path", fallback.toString()));
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists.
|
||||
* Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists.
|
||||
* <p>
|
||||
* To override the location of the file, set {@code argument=newpath} in the arguments.
|
||||
* To override the location of the file, set {@code argument=newpath} in the arguments. To set options for the output
|
||||
* drive add {@code output.mbtiles?arg=value} or add command-line argument {@code mbtiles_arg=value}.
|
||||
*
|
||||
* @param argument the argument key to check for an override to {@code fallback}
|
||||
* @param fallback the fallback value if {@code argument} is not set in arguments
|
||||
* @param defaultOutputUri The default output URI string to write to.
|
||||
* @return this runner instance for chaining
|
||||
* @see TileArchiveWriter
|
||||
* @see TileArchiveConfig For details on URI string formats and options.
|
||||
*/
|
||||
public Planetiler setOutput(String defaultOutputUri) {
|
||||
this.output = TileArchiveConfig.from(arguments.getString("output", "output tile archive URI", defaultOutputUri));
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Alias for {@link #setOutput(String)} which infers the output type based on extension. */
|
||||
public Planetiler setOutput(Path path) {
|
||||
return setOutput(path.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the location of the output archive to write rendered tiles to.
|
||||
*
|
||||
* @deprecated Use {@link #overwriteOutput(String)} instead
|
||||
*/
|
||||
@Deprecated(forRemoval = true)
|
||||
public Planetiler overwriteOutput(String argument, Path fallback) {
|
||||
this.overwrite = true;
|
||||
return setOutput(argument, fallback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the location of the output archive to write rendered tiles to. Overwrites if the archive already exists.
|
||||
* <p>
|
||||
* To override the location of the file, set {@code argument=newpath} in the arguments. To set options for the output
|
||||
* drive add {@code output.mbtiles?arg=value} or add command-line argument {@code mbtiles_arg=value}.
|
||||
*
|
||||
* @param defaultOutputUri The default output URI string to write to.
|
||||
* @return this runner instance for chaining
|
||||
* @see TileArchiveConfig For details on URI string formats and options.
|
||||
*/
|
||||
public Planetiler overwriteOutput(String defaultOutputUri) {
|
||||
this.overwrite = true;
|
||||
return setOutput(defaultOutputUri);
|
||||
}
|
||||
|
||||
/** Alias for {@link #overwriteOutput(String)} which infers the output type based on extension. */
|
||||
public Planetiler overwriteOutput(Path defaultOutput) {
|
||||
return overwriteOutput(defaultOutput.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads all elements from all sourced that have been added, generates map features according to the profile, and
|
||||
* writes the rendered tiles to the output archive.
|
||||
|
@ -600,19 +635,18 @@ public class Planetiler {
|
|||
throw new IllegalArgumentException("Can only run once");
|
||||
}
|
||||
ran = true;
|
||||
tileArchiveMetadata = new TileArchiveMetadata(profile, config.arguments());
|
||||
|
||||
if (arguments.getBoolean("help", "show arguments then exit", false)) {
|
||||
System.exit(0);
|
||||
} else if (onlyDownloadSources) {
|
||||
// don't check files if not generating map
|
||||
} else if (overwrite || config.force()) {
|
||||
FileUtils.deleteFile(output);
|
||||
} else if (Files.exists(output)) {
|
||||
throw new IllegalArgumentException(output + " already exists, use the --force argument to overwrite.");
|
||||
output.delete();
|
||||
} else if (output.exists()) {
|
||||
throw new IllegalArgumentException(output.uri() + " already exists, use the --force argument to overwrite.");
|
||||
}
|
||||
|
||||
LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output);
|
||||
LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri());
|
||||
|
||||
if (!toDownload.isEmpty()) {
|
||||
LOGGER.info(" download: Download sources {}", toDownload.stream().map(d -> d.id).toList());
|
||||
|
@ -635,7 +669,7 @@ public class Planetiler {
|
|||
// in case any temp files are left from a previous run...
|
||||
FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath);
|
||||
Files.createDirectories(tmpDir);
|
||||
FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output);
|
||||
FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalPath());
|
||||
|
||||
if (!toDownload.isEmpty()) {
|
||||
download();
|
||||
|
@ -661,14 +695,16 @@ public class Planetiler {
|
|||
}
|
||||
bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats));
|
||||
}
|
||||
// must construct this after bounds providers are added in order to infer bounds from the input source if not provided
|
||||
tileArchiveMetadata = new TileArchiveMetadata(profile, config);
|
||||
|
||||
try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
|
||||
try (WriteableTileArchive archive = TileArchives.newWriter(output, config)) {
|
||||
featureGroup =
|
||||
FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats);
|
||||
stats.monitorFile("nodes", nodeDbPath);
|
||||
stats.monitorFile("features", featureDbPath);
|
||||
stats.monitorFile("multipolygons", multipolygonPath);
|
||||
stats.monitorFile("archive", output);
|
||||
stats.monitorFile("archive", output.getLocalPath());
|
||||
|
||||
for (Stage stage : stages) {
|
||||
stage.task.run();
|
||||
|
@ -685,9 +721,8 @@ public class Planetiler {
|
|||
|
||||
featureGroup.prepare();
|
||||
|
||||
TileArchiveWriter.writeOutput(featureGroup, archive, () -> FileUtils.fileSize(output), tileArchiveMetadata,
|
||||
config,
|
||||
stats);
|
||||
TileArchiveWriter.writeOutput(featureGroup, archive, output::size, tileArchiveMetadata,
|
||||
config, stats);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("Unable to write to " + output, e);
|
||||
}
|
||||
|
@ -716,7 +751,7 @@ public class Planetiler {
|
|||
readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
|
||||
writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
|
||||
// output only needed during write phase
|
||||
writePhase.addDisk(output, outputSize, "archive output");
|
||||
writePhase.addDisk(output.getLocalPath(), outputSize, "archive output");
|
||||
// if the user opts to remove an input source after reading to free up additional space for the output...
|
||||
for (var input : inputPaths) {
|
||||
if (input.freeAfterReading()) {
|
||||
|
|
|
@ -40,5 +40,10 @@ public interface ReadableTileArchive extends Closeable {
|
|||
*/
|
||||
CloseableIterator<TileCoord> getAllTileCoords();
|
||||
|
||||
/**
|
||||
* Returns the metadata stored in this archive.
|
||||
*/
|
||||
TileArchiveMetadata metadata();
|
||||
|
||||
// TODO access archive metadata
|
||||
}
|
||||
|
|
|
@ -0,0 +1,196 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import static com.onthegomap.planetiler.util.LanguageUtils.nullIfEmpty;
|
||||
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.util.FileUtils;
|
||||
import java.net.URI;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Definition for a tileset, parsed from a URI-like string.
|
||||
* <p>
|
||||
* {@link #from(String)} can accept:
|
||||
* <ul>
|
||||
* <li>A platform-specific absolute or relative path like {@code "./archive.mbtiles"} or
|
||||
* {@code "C:\root\archive.mbtiles"}</li>
|
||||
* <li>A URI pointing at a file, like {@code "file:///root/archive.pmtiles"} or
|
||||
* {@code "file:///C:/root/archive.pmtiles"}</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* Both of these can also have archive-specific options added to the end, for example
|
||||
* {@code "output.mbtiles?compact=false&page_size=16384"}.
|
||||
*
|
||||
* @param format The {@link Format format} of the archive, either inferred from the filename extension or the
|
||||
* {@code ?format=} query parameter
|
||||
* @param scheme Scheme for accessing the archive
|
||||
* @param uri Full URI including scheme, location, and options
|
||||
* @param options Parsed query parameters from the definition string
|
||||
*/
|
||||
public record TileArchiveConfig(
|
||||
Format format,
|
||||
Scheme scheme,
|
||||
URI uri,
|
||||
Map<String, String> options
|
||||
) {
|
||||
|
||||
private static TileArchiveConfig.Scheme getScheme(URI uri) {
|
||||
String scheme = uri.getScheme();
|
||||
if (scheme == null) {
|
||||
return Scheme.FILE;
|
||||
}
|
||||
for (var value : TileArchiveConfig.Scheme.values()) {
|
||||
if (value.id().equals(scheme)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unsupported scheme " + scheme + " from " + uri);
|
||||
}
|
||||
|
||||
private static String getExtension(URI uri) {
|
||||
String path = uri.getPath();
|
||||
if (path != null && (path.contains("."))) {
|
||||
return nullIfEmpty(path.substring(path.lastIndexOf(".") + 1));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Map<String, String> parseQuery(URI uri) {
|
||||
String query = uri.getRawQuery();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
if (query != null) {
|
||||
for (var part : query.split("&")) {
|
||||
var split = part.split("=", 2);
|
||||
result.put(
|
||||
URLDecoder.decode(split[0], StandardCharsets.UTF_8),
|
||||
split.length == 1 ? "true" : URLDecoder.decode(split[1], StandardCharsets.UTF_8)
|
||||
);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static TileArchiveConfig.Format getFormat(URI uri) {
|
||||
String format = parseQuery(uri).get("format");
|
||||
if (format == null) {
|
||||
format = getExtension(uri);
|
||||
}
|
||||
if (format == null) {
|
||||
return TileArchiveConfig.Format.MBTILES;
|
||||
}
|
||||
for (var value : TileArchiveConfig.Format.values()) {
|
||||
if (value.id().equals(format)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unsupported format " + format + " from " + uri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a string definition of a tileset from a URI-like string.
|
||||
*/
|
||||
public static TileArchiveConfig from(String string) {
|
||||
// unix paths parse fine as URIs, but need to explicitly parse windows paths with backslashes
|
||||
if (string.contains("\\")) {
|
||||
String[] parts = string.split("\\?", 2);
|
||||
string = Path.of(parts[0]).toUri().toString();
|
||||
if (parts.length > 1) {
|
||||
string += "?" + parts[1];
|
||||
}
|
||||
}
|
||||
return from(URI.create(string));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a string definition of a tileset from a URI.
|
||||
*/
|
||||
public static TileArchiveConfig from(URI uri) {
|
||||
if (uri.getScheme() == null) {
|
||||
String base = Path.of(uri.getPath()).toAbsolutePath().toUri().normalize().toString();
|
||||
if (uri.getRawQuery() != null) {
|
||||
base += "?" + uri.getRawQuery();
|
||||
}
|
||||
uri = URI.create(base);
|
||||
}
|
||||
return new TileArchiveConfig(
|
||||
getFormat(uri),
|
||||
getScheme(uri),
|
||||
uri,
|
||||
parseQuery(uri)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the local path on disk that this archive reads/writes to, or {@code null} if it is not on disk (ie. an HTTP
|
||||
* repository).
|
||||
*/
|
||||
public Path getLocalPath() {
|
||||
return scheme == Scheme.FILE ? Path.of(URI.create(uri.toString().replaceAll("\\?.*$", ""))) : null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Deletes the archive if possible.
|
||||
*/
|
||||
public void delete() {
|
||||
if (scheme == Scheme.FILE) {
|
||||
FileUtils.delete(getLocalPath());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns {@code true} if the archive already exists, {@code false} otherwise.
|
||||
*/
|
||||
public boolean exists() {
|
||||
return getLocalPath() != null && Files.exists(getLocalPath());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current size of this archive.
|
||||
*/
|
||||
public long size() {
|
||||
return getLocalPath() == null ? 0 : FileUtils.size(getLocalPath());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an {@link Arguments} instance that returns the value for options directly from the query parameters in the
|
||||
* URI, or from {@code arguments} prefixed by {@code "format_"}.
|
||||
*/
|
||||
public Arguments applyFallbacks(Arguments arguments) {
|
||||
return Arguments.of(options).orElse(arguments.withPrefix(format.id));
|
||||
}
|
||||
|
||||
public enum Format {
|
||||
MBTILES("mbtiles"),
|
||||
PMTILES("pmtiles");
|
||||
|
||||
private final String id;
|
||||
|
||||
Format(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String id() {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
public enum Scheme {
|
||||
FILE("file");
|
||||
|
||||
private final String id;
|
||||
|
||||
Scheme(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String id() {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,42 +1,90 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.util.BuildInfo;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
|
||||
import static com.onthegomap.planetiler.util.Format.joinCoordinates;
|
||||
|
||||
/** Controls information that {@link TileArchiveWriter} will write to the archive metadata. */
|
||||
import com.fasterxml.jackson.annotation.JsonAnyGetter;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.util.BuildInfo;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/** Metadata associated with a tile archive. */
|
||||
public record TileArchiveMetadata(
|
||||
String name,
|
||||
String description,
|
||||
String attribution,
|
||||
String version,
|
||||
String type,
|
||||
Map<String, String> planetilerSpecific
|
||||
@JsonProperty(NAME_KEY) String name,
|
||||
@JsonProperty(DESCRIPTION_KEY) String description,
|
||||
@JsonProperty(ATTRIBUTION_KEY) String attribution,
|
||||
@JsonProperty(VERSION_KEY) String version,
|
||||
@JsonProperty(TYPE_KEY) String type,
|
||||
@JsonProperty(FORMAT_KEY) String format,
|
||||
@JsonIgnore Envelope bounds,
|
||||
@JsonIgnore CoordinateXY center,
|
||||
@JsonProperty(ZOOM_KEY) Double zoom,
|
||||
@JsonProperty(MINZOOM_KEY) Integer minzoom,
|
||||
@JsonProperty(MAXZOOM_KEY) Integer maxzoom,
|
||||
@JsonIgnore List<LayerStats.VectorLayer> vectorLayers,
|
||||
@JsonAnyGetter Map<String, String> others
|
||||
) {
|
||||
|
||||
public TileArchiveMetadata(Profile profile) {
|
||||
public static final String NAME_KEY = "name";
|
||||
public static final String DESCRIPTION_KEY = "description";
|
||||
public static final String ATTRIBUTION_KEY = "attribution";
|
||||
public static final String VERSION_KEY = "version";
|
||||
public static final String TYPE_KEY = "type";
|
||||
public static final String FORMAT_KEY = "format";
|
||||
public static final String BOUNDS_KEY = "bounds";
|
||||
public static final String CENTER_KEY = "center";
|
||||
public static final String ZOOM_KEY = "zoom";
|
||||
public static final String MINZOOM_KEY = "minzoom";
|
||||
public static final String MAXZOOM_KEY = "maxzoom";
|
||||
public static final String VECTOR_LAYERS_KEY = "vector_layers";
|
||||
|
||||
public static final String MVT_FORMAT = "pbf";
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TileArchiveMetadata.class);
|
||||
private static final ObjectMapper mapper = new ObjectMapper()
|
||||
.registerModules(new Jdk8Module())
|
||||
.setSerializationInclusion(NON_ABSENT);
|
||||
|
||||
public TileArchiveMetadata(Profile profile, PlanetilerConfig config) {
|
||||
this(profile, config, null);
|
||||
}
|
||||
|
||||
public TileArchiveMetadata(Profile profile, PlanetilerConfig config, List<LayerStats.VectorLayer> vectorLayers) {
|
||||
this(
|
||||
profile.name(),
|
||||
profile.description(),
|
||||
profile.attribution(),
|
||||
profile.version(),
|
||||
profile.isOverlay() ? "overlay" : "baselayer",
|
||||
getString(config, NAME_KEY, profile.name()),
|
||||
getString(config, DESCRIPTION_KEY, profile.description()),
|
||||
getString(config, ATTRIBUTION_KEY, profile.attribution()),
|
||||
getString(config, VERSION_KEY, profile.version()),
|
||||
getString(config, TYPE_KEY, profile.isOverlay() ? "overlay" : "baselayer"),
|
||||
getString(config, FORMAT_KEY, MVT_FORMAT),
|
||||
config.bounds().latLon(),
|
||||
new CoordinateXY(config.bounds().latLon().centre()),
|
||||
GeoUtils.getZoomFromLonLatBounds(config.bounds().latLon()),
|
||||
config.minzoom(),
|
||||
config.maxzoom(),
|
||||
vectorLayers,
|
||||
mapWithBuildInfo()
|
||||
);
|
||||
}
|
||||
|
||||
public TileArchiveMetadata(Profile profile, Arguments args) {
|
||||
this(
|
||||
args.getString("mbtiles_name", "'name' attribute for tileset metadata", profile.name()),
|
||||
args.getString("mbtiles_description", "'description' attribute for tileset metadata", profile.description()),
|
||||
args.getString("mbtiles_attribution", "'attribution' attribute for tileset metadata", profile.attribution()),
|
||||
args.getString("mbtiles_version", "'version' attribute for tileset metadata", profile.version()),
|
||||
args.getString("mbtiles_type", "'type' attribute for tileset metadata",
|
||||
profile.isOverlay() ? "overlay" : "baselayer"),
|
||||
mapWithBuildInfo()
|
||||
);
|
||||
private static String getString(PlanetilerConfig config, String key, String fallback) {
|
||||
return config.arguments()
|
||||
.getString("archive_" + key + "|mbtiles_" + key, "'" + key + "' attribute for tileset metadata", fallback);
|
||||
}
|
||||
|
||||
private static Map<String, String> mapWithBuildInfo() {
|
||||
|
@ -56,20 +104,39 @@ public record TileArchiveMetadata(
|
|||
return result;
|
||||
}
|
||||
|
||||
public TileArchiveMetadata set(String key, Object value) {
|
||||
/** Sets an extra metadata entry in {@link #others}. */
|
||||
public TileArchiveMetadata setExtraMetadata(String key, Object value) {
|
||||
if (key != null && value != null) {
|
||||
planetilerSpecific.put(key, value.toString());
|
||||
others.put(key, value.toString());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public Map<String, String> getAll() {
|
||||
var allKvs = new LinkedHashMap<String, String>(planetilerSpecific);
|
||||
allKvs.put("name", this.name);
|
||||
allKvs.put("description", this.description);
|
||||
allKvs.put("attribution", this.attribution);
|
||||
allKvs.put("version", this.version);
|
||||
allKvs.put("type", this.type);
|
||||
return allKvs;
|
||||
/**
|
||||
* Returns a map with all key-value pairs from this metadata entry, including {@link #others} hoisted to top-level
|
||||
* keys.
|
||||
*/
|
||||
public Map<String, String> toMap() {
|
||||
Map<String, String> result = new LinkedHashMap<>(mapper.convertValue(this, new TypeReference<>() {}));
|
||||
if (bounds != null) {
|
||||
result.put(BOUNDS_KEY, joinCoordinates(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY()));
|
||||
}
|
||||
if (center != null) {
|
||||
result.put(CENTER_KEY, joinCoordinates(center.getX(), center.getY()));
|
||||
}
|
||||
if (vectorLayers != null) {
|
||||
try {
|
||||
result.put(VECTOR_LAYERS_KEY, mapper.writeValueAsString(vectorLayers));
|
||||
} catch (JsonProcessingException e) {
|
||||
LOGGER.warn("Error encoding vector_layers as json", e);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Returns a copy of this instance with {@link #vectorLayers} set to {@code layerStats}. */
|
||||
public TileArchiveMetadata withLayerStats(List<LayerStats.VectorLayer> layerStats) {
|
||||
return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, zoom, minzoom,
|
||||
maxzoom, layerStats, others);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -15,7 +15,6 @@ import com.onthegomap.planetiler.stats.Timer;
|
|||
import com.onthegomap.planetiler.util.DiskBacked;
|
||||
import com.onthegomap.planetiler.util.Format;
|
||||
import com.onthegomap.planetiler.util.Hashing;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import com.onthegomap.planetiler.worker.WorkQueue;
|
||||
import com.onthegomap.planetiler.worker.Worker;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
|
@ -52,7 +51,6 @@ public class TileArchiveWriter {
|
|||
private final WriteableTileArchive archive;
|
||||
private final PlanetilerConfig config;
|
||||
private final Stats stats;
|
||||
private final LayerStats layerStats;
|
||||
private final Counter.Readable[] tilesByZoom;
|
||||
private final Counter.Readable[] totalTileSizesByZoom;
|
||||
private final LongAccumulator[] maxTileSizesByZoom;
|
||||
|
@ -61,14 +59,12 @@ public class TileArchiveWriter {
|
|||
private final TileArchiveMetadata tileArchiveMetadata;
|
||||
|
||||
private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, WriteableTileArchive archive,
|
||||
PlanetilerConfig config,
|
||||
TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) {
|
||||
PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) {
|
||||
this.inputTiles = inputTiles;
|
||||
this.archive = archive;
|
||||
this.config = config;
|
||||
this.tileArchiveMetadata = tileArchiveMetadata;
|
||||
this.stats = stats;
|
||||
this.layerStats = layerStats;
|
||||
tilesByZoom = IntStream.rangeClosed(0, config.maxzoom())
|
||||
.mapToObj(i -> Counter.newSingleThreadCounter())
|
||||
.toArray(Counter.Readable[]::new);
|
||||
|
@ -111,8 +107,9 @@ public class TileArchiveWriter {
|
|||
readWorker = reader.readWorker();
|
||||
}
|
||||
|
||||
TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats,
|
||||
features.layerStats());
|
||||
TileArchiveWriter writer =
|
||||
new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats()
|
||||
.getTileStats()), stats);
|
||||
|
||||
var pipeline = WorkerPipeline.start("archive", stats);
|
||||
|
||||
|
@ -231,7 +228,6 @@ public class TileArchiveWriter {
|
|||
byte[] lastBytes = null, lastEncoded = null;
|
||||
Long lastTileDataHash = null;
|
||||
boolean lastIsFill = false;
|
||||
boolean compactDb = config.compactDb();
|
||||
boolean skipFilled = config.skipFilledTiles();
|
||||
|
||||
for (TileBatch batch : prev) {
|
||||
|
@ -265,7 +261,7 @@ public class TileArchiveWriter {
|
|||
lastEncoded = encoded;
|
||||
lastBytes = bytes;
|
||||
last = tileFeatures;
|
||||
if (compactDb && en.likelyToBeDuplicated() && bytes != null) {
|
||||
if (archive.deduplicates() && en.likelyToBeDuplicated() && bytes != null) {
|
||||
tileDataHash = generateContentHash(bytes);
|
||||
} else {
|
||||
tileDataHash = null;
|
||||
|
@ -292,7 +288,8 @@ public class TileArchiveWriter {
|
|||
|
||||
private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionException, InterruptedException {
|
||||
|
||||
archive.initialize(config, tileArchiveMetadata, layerStats);
|
||||
archive.initialize(tileArchiveMetadata);
|
||||
var order = archive.tileOrder();
|
||||
|
||||
TileCoord lastTile = null;
|
||||
Timer time = null;
|
||||
|
@ -303,8 +300,9 @@ public class TileArchiveWriter {
|
|||
TileEncodingResult encodedTile;
|
||||
while ((encodedTile = encodedTiles.poll()) != null) {
|
||||
TileCoord tileCoord = encodedTile.coord();
|
||||
assert lastTile == null || lastTile.compareTo(tileCoord) < 0 : "Tiles out of order %s before %s"
|
||||
.formatted(lastTile, tileCoord);
|
||||
assert lastTile == null ||
|
||||
order.encode(tileCoord) > order.encode(lastTile) : "Tiles out of order %s before %s"
|
||||
.formatted(lastTile, tileCoord);
|
||||
lastTile = encodedTile.coord();
|
||||
int z = tileCoord.z();
|
||||
if (z != currentZ) {
|
||||
|
@ -331,7 +329,7 @@ public class TileArchiveWriter {
|
|||
}
|
||||
|
||||
|
||||
archive.finish(config);
|
||||
archive.finish(tileArchiveMetadata);
|
||||
}
|
||||
|
||||
private void printTileStats() {
|
||||
|
|
|
@ -0,0 +1,75 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
|
||||
import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */
|
||||
public class TileArchives {
|
||||
private TileArchives() {}
|
||||
|
||||
/**
|
||||
* Returns a new {@link WriteableTileArchive} from the string definition in {@code archive} that will be parsed with
|
||||
* {@link TileArchiveConfig}.
|
||||
*
|
||||
* @throws IOException if an error occurs creating the resource.
|
||||
*/
|
||||
public static WriteableTileArchive newWriter(String archive, PlanetilerConfig config) throws IOException {
|
||||
return newWriter(TileArchiveConfig.from(archive), config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@link ReadableTileArchive} from the string definition in {@code archive} that will be parsed with
|
||||
* {@link TileArchiveConfig}.
|
||||
*
|
||||
* @throws IOException if an error occurs opening the resource.
|
||||
*/
|
||||
public static ReadableTileArchive newReader(String archive, PlanetilerConfig config) throws IOException {
|
||||
return newReader(TileArchiveConfig.from(archive), config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}.
|
||||
*
|
||||
* @throws IOException if an error occurs creating the resource.
|
||||
*/
|
||||
public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
|
||||
throws IOException {
|
||||
var options = archive.applyFallbacks(config.arguments());
|
||||
return switch (archive.format()) {
|
||||
case MBTILES ->
|
||||
// pass-through legacy arguments for fallback
|
||||
Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments()
|
||||
.subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION)));
|
||||
case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath());
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}.
|
||||
*
|
||||
* @throws IOException if an error occurs opening the resource.
|
||||
*/
|
||||
public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
|
||||
throws IOException {
|
||||
var options = archive.applyFallbacks(config.arguments());
|
||||
return switch (archive.format()) {
|
||||
case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options);
|
||||
case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath());
|
||||
};
|
||||
}
|
||||
|
||||
/** Alias for {@link #newReader(String, PlanetilerConfig)}. */
|
||||
public static ReadableTileArchive newReader(Path path, PlanetilerConfig config) throws IOException {
|
||||
return newReader(path.toString(), config);
|
||||
}
|
||||
|
||||
/** Alias for {@link #newWriter(String, PlanetilerConfig)}. */
|
||||
public static WriteableTileArchive newWriter(Path path, PlanetilerConfig config) throws IOException {
|
||||
return newWriter(path.toString(), config);
|
||||
}
|
||||
|
||||
}
|
|
@ -2,7 +2,6 @@ package com.onthegomap.planetiler.archive;
|
|||
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import java.io.Closeable;
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
|
@ -15,6 +14,36 @@ import javax.annotation.concurrent.NotThreadSafe;
|
|||
@NotThreadSafe
|
||||
public interface WriteableTileArchive extends Closeable {
|
||||
|
||||
/**
|
||||
* Returns true if this tile archive deduplicates tiles with the same content.
|
||||
* <p>
|
||||
* If false, then {@link TileWriter} will skip computing tile hashes.
|
||||
*/
|
||||
boolean deduplicates();
|
||||
|
||||
/**
|
||||
* Specify the preferred insertion order for this archive, e.g. {@link TileOrder#TMS} or {@link TileOrder#HILBERT}.
|
||||
*/
|
||||
TileOrder tileOrder();
|
||||
|
||||
/**
|
||||
* Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
|
||||
* required state here.
|
||||
*/
|
||||
default void initialize(TileArchiveMetadata metadata) {}
|
||||
|
||||
/**
|
||||
* Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might
|
||||
* depend on {@link PlanetilerConfig}.
|
||||
*/
|
||||
TileWriter newTileWriter();
|
||||
|
||||
/**
|
||||
* Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on
|
||||
* disk.
|
||||
*/
|
||||
default void finish(TileArchiveMetadata tileArchiveMetadata) {}
|
||||
|
||||
interface TileWriter extends Closeable {
|
||||
|
||||
void write(TileEncodingResult encodingResult);
|
||||
|
@ -30,29 +59,5 @@ public interface WriteableTileArchive extends Closeable {
|
|||
default void printStats() {}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Specify the preferred insertion order for this archive, e.g. {@link TileOrder#TMS} or {@link TileOrder#HILBERT}.
|
||||
*/
|
||||
TileOrder tileOrder();
|
||||
|
||||
/**
|
||||
* Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
|
||||
* required state here.
|
||||
*/
|
||||
void initialize(PlanetilerConfig config, TileArchiveMetadata metadata, LayerStats layerStats);
|
||||
|
||||
/**
|
||||
* Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might
|
||||
* depend on {@link PlanetilerConfig}.
|
||||
*/
|
||||
TileWriter newTileWriter();
|
||||
|
||||
/**
|
||||
* Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on
|
||||
* disk.
|
||||
*/
|
||||
void finish(PlanetilerConfig config);
|
||||
|
||||
// TODO update archive metadata
|
||||
}
|
||||
|
|
|
@ -12,6 +12,8 @@ import java.time.format.DateTimeParseException;
|
|||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
@ -20,6 +22,7 @@ import java.util.Set;
|
|||
import java.util.TreeMap;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.function.UnaryOperator;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
|
@ -28,6 +31,12 @@ import org.slf4j.LoggerFactory;
|
|||
/**
|
||||
* Lightweight abstraction over ways to provide key/value pair arguments to a program like jvm properties, environmental
|
||||
* variables, or a config file.
|
||||
* <p>
|
||||
* When looking up a key, tries to find a case-and-separator-insensitive match, for example {@code "CONFIG_OPTION"} will
|
||||
* match {@code "config-option"} and {@code "config_option"}.
|
||||
* <p>
|
||||
* If you replace an option with a new value, you can read a value from the new option and fall back to old one by using
|
||||
* {@code "new_flag|old_flag"} as the key.
|
||||
*/
|
||||
public class Arguments {
|
||||
|
||||
|
@ -42,15 +51,6 @@ public class Arguments {
|
|||
this.keys = keys;
|
||||
}
|
||||
|
||||
private static Arguments from(UnaryOperator<String> provider, Supplier<? extends Collection<String>> rawKeys,
|
||||
UnaryOperator<String> forward, UnaryOperator<String> reverse) {
|
||||
Supplier<List<String>> keys = () -> rawKeys.get().stream().flatMap(key -> {
|
||||
String reversed = reverse.apply(key);
|
||||
return key.equalsIgnoreCase(reversed) ? Stream.empty() : Stream.of(reversed);
|
||||
}).toList();
|
||||
return new Arguments(key -> provider.apply(forward.apply(key)), keys);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns arguments from JVM system properties prefixed with {@code planetiler.}
|
||||
* <p>
|
||||
|
@ -64,10 +64,7 @@ public class Arguments {
|
|||
}
|
||||
|
||||
static Arguments fromJvmProperties(UnaryOperator<String> getter, Supplier<? extends Collection<String>> keys) {
|
||||
return from(getter, keys,
|
||||
key -> "planetiler." + key.toLowerCase(Locale.ROOT),
|
||||
key -> key.replaceFirst("^planetiler\\.", "").toLowerCase(Locale.ROOT)
|
||||
);
|
||||
return fromPrefixed(getter, keys, "planetiler", ".", false);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -83,10 +80,7 @@ public class Arguments {
|
|||
}
|
||||
|
||||
static Arguments fromEnvironment(UnaryOperator<String> getter, Supplier<Set<String>> keys) {
|
||||
return from(getter, keys,
|
||||
key -> "PLANETILER_" + key.toUpperCase(Locale.ROOT),
|
||||
key -> key.replaceFirst("^PLANETILER_", "").toLowerCase(Locale.ROOT)
|
||||
);
|
||||
return fromPrefixed(getter, keys, "PLANETILER", "_", true);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -191,8 +185,21 @@ public class Arguments {
|
|||
.orElse(fromEnvironment());
|
||||
}
|
||||
|
||||
private static String normalize(String key, String separator, boolean upperCase) {
|
||||
String result = key.replaceAll("[._-]", separator);
|
||||
return upperCase ? result.toUpperCase(Locale.ROOT) : result.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private static String normalize(String key) {
|
||||
return normalize(key, "_", false);
|
||||
}
|
||||
|
||||
public static Arguments of(Map<String, String> map) {
|
||||
return new Arguments(map::get, map::keySet);
|
||||
Map<String, String> updated = new LinkedHashMap<>();
|
||||
for (var entry : map.entrySet()) {
|
||||
updated.put(normalize(entry.getKey()), entry.getValue());
|
||||
}
|
||||
return new Arguments(updated::get, updated::keySet);
|
||||
}
|
||||
|
||||
/** Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs. */
|
||||
|
@ -204,12 +211,36 @@ public class Arguments {
|
|||
return of(map);
|
||||
}
|
||||
|
||||
private static Arguments from(UnaryOperator<String> provider, Supplier<? extends Collection<String>> rawKeys,
|
||||
UnaryOperator<String> forward, UnaryOperator<String> reverse) {
|
||||
Supplier<List<String>> keys = () -> rawKeys.get().stream().flatMap(key -> {
|
||||
String reversed = reverse.apply(key);
|
||||
return normalize(key).equals(normalize(reversed)) ? Stream.empty() : Stream.of(reversed);
|
||||
}).toList();
|
||||
return new Arguments(key -> provider.apply(forward.apply(key)), keys);
|
||||
}
|
||||
|
||||
private static Arguments fromPrefixed(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys,
|
||||
String prefix, String separator, boolean uppperCase) {
|
||||
var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)),
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
return from(provider, keys,
|
||||
key -> normalize(prefix + separator + key, separator, uppperCase),
|
||||
key -> normalize(prefixRegex.matcher(key).replaceFirst(""))
|
||||
);
|
||||
}
|
||||
|
||||
private String get(String key) {
|
||||
String value = provider.apply(key);
|
||||
if (value == null) {
|
||||
value = provider.apply(key.replace('-', '_'));
|
||||
if (value == null) {
|
||||
value = provider.apply(key.replace('_', '-'));
|
||||
String[] options = key.split("\\|");
|
||||
String value = null;
|
||||
for (int i = 0; i < options.length; i++) {
|
||||
String option = options[i].strip();
|
||||
value = provider.apply(normalize(option));
|
||||
if (value != null) {
|
||||
if (i != 0) {
|
||||
LOGGER.warn("Argument '{}' is deprecated", option);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
|
@ -274,8 +305,8 @@ public class Arguments {
|
|||
}
|
||||
|
||||
protected void logArgValue(String key, String description, Object result) {
|
||||
if (!silent) {
|
||||
LOGGER.debug("argument: {}={} ({})", key, result, description);
|
||||
if (!silent && LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -460,7 +491,7 @@ public class Arguments {
|
|||
public Map<String, String> toMap() {
|
||||
Map<String, String> result = new HashMap<>();
|
||||
for (var key : keys.get()) {
|
||||
result.put(key, get(key));
|
||||
result.put(normalize(key), get(key));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
@ -484,4 +515,27 @@ public class Arguments {
|
|||
public boolean silenced() {
|
||||
return silent;
|
||||
}
|
||||
|
||||
public Arguments copy() {
|
||||
return new Arguments(provider, keys);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}.
|
||||
*/
|
||||
public Arguments withPrefix(String prefix) {
|
||||
return fromPrefixed(provider, keys, prefix, "_", false);
|
||||
}
|
||||
|
||||
/** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */
|
||||
public Arguments subset(String... allowedKeys) {
|
||||
Set<String> allowed = new HashSet<>();
|
||||
for (String key : allowedKeys) {
|
||||
allowed.add(normalize(key));
|
||||
}
|
||||
return new Arguments(
|
||||
key -> allowed.contains(normalize(key)) ? provider.apply(key) : null,
|
||||
() -> keys.get().stream().filter(key -> allowed.contains(normalize(key))).toList()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -24,8 +24,6 @@ public record PlanetilerConfig(
|
|||
int minzoom,
|
||||
int maxzoom,
|
||||
int maxzoomForRendering,
|
||||
boolean skipIndexCreation,
|
||||
boolean optimizeDb,
|
||||
boolean force,
|
||||
boolean gzipTempStorage,
|
||||
boolean mmapTempStorage,
|
||||
|
@ -47,7 +45,6 @@ public record PlanetilerConfig(
|
|||
double simplifyToleranceAtMaxZoom,
|
||||
double simplifyToleranceBelowMaxZoom,
|
||||
boolean osmLazyReads,
|
||||
boolean compactDb,
|
||||
boolean skipFilledTiles,
|
||||
int tileWarningSizeBytes,
|
||||
Boolean color
|
||||
|
@ -125,8 +122,6 @@ public record PlanetilerConfig(
|
|||
minzoom,
|
||||
maxzoom,
|
||||
renderMaxzoom,
|
||||
arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false),
|
||||
arguments.getBoolean("optimize_db", "Vacuum analyze mbtiles after writing", false),
|
||||
arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
|
||||
arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),
|
||||
arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true),
|
||||
|
@ -168,9 +163,6 @@ public record PlanetilerConfig(
|
|||
arguments.getBoolean("osm_lazy_reads",
|
||||
"Read OSM blocks from disk in worker threads",
|
||||
true),
|
||||
arguments.getBoolean("compact_db",
|
||||
"Reduce the DB size by separating and deduping the tile data",
|
||||
true),
|
||||
arguments.getBoolean("skip_filled_tiles",
|
||||
"Skip writing tiles containing only polygon fills to the output",
|
||||
false),
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package com.onthegomap.planetiler.mbtiles;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
|
||||
import static com.onthegomap.planetiler.util.Format.joinCoordinates;
|
||||
|
||||
import com.carrotsearch.hppc.LongIntHashMap;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
@ -11,14 +12,14 @@ import com.onthegomap.planetiler.archive.ReadableTileArchive;
|
|||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.reader.FileFormatException;
|
||||
import com.onthegomap.planetiler.util.CloseableIterator;
|
||||
import com.onthegomap.planetiler.util.Format;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import com.onthegomap.planetiler.util.Parse;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
|
@ -27,21 +28,20 @@ import java.sql.PreparedStatement;
|
|||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.text.NumberFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.TreeMap;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.DoubleStream;
|
||||
import java.util.stream.IntStream;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
@ -54,6 +54,17 @@ import org.sqlite.SQLiteConfig;
|
|||
*/
|
||||
public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive {
|
||||
|
||||
// Options that can be set through "file.mbtiles?compact=true" query parameters
|
||||
// or "file.mbtiles" with "--mbtiles-compact=true" command-line flag
|
||||
public static final String COMPACT_DB = "compact";
|
||||
public static final String SKIP_INDEX_CREATION = "no_index";
|
||||
public static final String VACUUM_ANALYZE = "vacuum_analyze";
|
||||
|
||||
public static final String LEGACY_COMPACT_DB = "compact_db";
|
||||
public static final String LEGACY_SKIP_INDEX_CREATION = "skip_mbtiles_index_creation";
|
||||
public static final String LEGACY_VACUUM_ANALYZE = "optimize_db";
|
||||
|
||||
|
||||
// https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
|
||||
private static final int MBTILES_APPLICATION_ID = 0x4d504258;
|
||||
|
||||
|
@ -93,72 +104,115 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
|
||||
private final Connection connection;
|
||||
private final boolean compactDb;
|
||||
private final boolean skipIndexCreation;
|
||||
private final boolean vacuumAnalyze;
|
||||
private PreparedStatement getTileStatement = null;
|
||||
|
||||
private Mbtiles(Connection connection, boolean compactDb) {
|
||||
private Mbtiles(Connection connection, Arguments arguments) {
|
||||
this.connection = connection;
|
||||
this.compactDb = compactDb;
|
||||
this.compactDb = arguments.getBoolean(
|
||||
COMPACT_DB + "|" + LEGACY_COMPACT_DB,
|
||||
"mbtiles: reduce the DB size by separating and deduping the tile data",
|
||||
true
|
||||
);
|
||||
this.skipIndexCreation = arguments.getBoolean(
|
||||
SKIP_INDEX_CREATION + "|" + LEGACY_SKIP_INDEX_CREATION,
|
||||
"mbtiles: skip adding index to sqlite DB",
|
||||
false
|
||||
);
|
||||
this.vacuumAnalyze = arguments.getBoolean(
|
||||
VACUUM_ANALYZE + "|" + LEGACY_VACUUM_ANALYZE,
|
||||
"mbtiles: vacuum analyze sqlite DB after writing",
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns a new mbtiles file that won't get written to disk. Useful for toy use-cases like unit tests. */
|
||||
public static Mbtiles newInMemoryDatabase(boolean compactDb) {
|
||||
try {
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setApplicationId(MBTILES_APPLICATION_ID);
|
||||
return new Mbtiles(DriverManager.getConnection("jdbc:sqlite::memory:", config.toProperties()), compactDb);
|
||||
} catch (SQLException throwables) {
|
||||
throw new IllegalStateException("Unable to create in-memory database", throwables);
|
||||
}
|
||||
return newInMemoryDatabase(Arguments.of(COMPACT_DB, compactDb ? "true" : "false"));
|
||||
}
|
||||
|
||||
/** @see {@link #newInMemoryDatabase(boolean)} */
|
||||
/** Returns an in-memory database with extra mbtiles and pragma options set from {@code options}. */
|
||||
public static Mbtiles newInMemoryDatabase(Arguments options) {
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setApplicationId(MBTILES_APPLICATION_ID);
|
||||
return new Mbtiles(newConnection("jdbc:sqlite::memory:", config, options), options);
|
||||
}
|
||||
|
||||
/** Alias for {@link #newInMemoryDatabase(boolean)} */
|
||||
public static Mbtiles newInMemoryDatabase() {
|
||||
return newInMemoryDatabase(true);
|
||||
}
|
||||
|
||||
/** Returns a new connection to an mbtiles file optimized for fast bulk writes. */
|
||||
public static Mbtiles newWriteToFileDatabase(Path path, boolean compactDb) {
|
||||
try {
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setJournalMode(SQLiteConfig.JournalMode.OFF);
|
||||
config.setSynchronous(SQLiteConfig.SynchronousMode.OFF);
|
||||
config.setCacheSize(1_000_000); // 1GB
|
||||
config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
|
||||
config.setTempStore(SQLiteConfig.TempStore.MEMORY);
|
||||
config.setApplicationId(MBTILES_APPLICATION_ID);
|
||||
return new Mbtiles(DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties()),
|
||||
compactDb);
|
||||
} catch (SQLException throwables) {
|
||||
throw new IllegalArgumentException("Unable to open " + path, throwables);
|
||||
}
|
||||
/**
|
||||
* Returns a new connection to an mbtiles file optimized for fast bulk writes with extra mbtiles and pragma options
|
||||
* set from {@code options}.
|
||||
*/
|
||||
public static Mbtiles newWriteToFileDatabase(Path path, Arguments options) {
|
||||
Objects.requireNonNull(path);
|
||||
SQLiteConfig sqliteConfig = new SQLiteConfig();
|
||||
sqliteConfig.setJournalMode(SQLiteConfig.JournalMode.OFF);
|
||||
sqliteConfig.setSynchronous(SQLiteConfig.SynchronousMode.OFF);
|
||||
sqliteConfig.setCacheSize(1_000_000); // 1GB
|
||||
sqliteConfig.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
|
||||
sqliteConfig.setTempStore(SQLiteConfig.TempStore.MEMORY);
|
||||
sqliteConfig.setApplicationId(MBTILES_APPLICATION_ID);
|
||||
var connection = newConnection("jdbc:sqlite:" + path.toAbsolutePath(), sqliteConfig, options);
|
||||
return new Mbtiles(connection, options);
|
||||
}
|
||||
|
||||
/** Returns a new connection to an mbtiles file optimized for reads. */
|
||||
public static Mbtiles newReadOnlyDatabase(Path path) {
|
||||
return newReadOnlyDatabase(path, Arguments.of());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new connection to an mbtiles file optimized for reads with extra mbtiles and pragma options set from
|
||||
* {@code options}.
|
||||
*/
|
||||
public static Mbtiles newReadOnlyDatabase(Path path, Arguments options) {
|
||||
Objects.requireNonNull(path);
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setReadOnly(true);
|
||||
config.setCacheSize(100_000);
|
||||
config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
|
||||
config.setPageSize(32_768);
|
||||
// helps with 3 or more threads concurrently accessing:
|
||||
// config.setOpenMode(SQLiteOpenMode.NOMUTEX);
|
||||
Connection connection = newConnection("jdbc:sqlite:" + path.toAbsolutePath(), config, options);
|
||||
return new Mbtiles(connection, options);
|
||||
}
|
||||
|
||||
private static Connection newConnection(String url, SQLiteConfig defaults, Arguments args) {
|
||||
try {
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setReadOnly(true);
|
||||
config.setCacheSize(100_000);
|
||||
config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
|
||||
config.setPageSize(32_768);
|
||||
// helps with 3 or more threads concurrently accessing:
|
||||
// config.setOpenMode(SQLiteOpenMode.NOMUTEX);
|
||||
Connection connection = DriverManager
|
||||
.getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties());
|
||||
return new Mbtiles(connection, false /* in read-only mode, it's irrelevant if compact or not */);
|
||||
args = args.copy().silence();
|
||||
var config = new SQLiteConfig(defaults.toProperties());
|
||||
for (var pragma : SQLiteConfig.Pragma.values()) {
|
||||
var value = args.getString(pragma.getPragmaName(), pragma.getPragmaName(), null);
|
||||
if (value != null) {
|
||||
LOGGER.info("Setting custom mbtiles sqlite pragma {}={}", pragma.getPragmaName(), value);
|
||||
config.setPragma(pragma, value);
|
||||
}
|
||||
}
|
||||
return DriverManager.getConnection(url, config.toProperties());
|
||||
} catch (SQLException throwables) {
|
||||
throw new IllegalArgumentException("Unable to open " + path, throwables);
|
||||
throw new IllegalArgumentException("Unable to open " + url, throwables);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deduplicates() {
|
||||
return compactDb;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileOrder tileOrder() {
|
||||
return TileOrder.TMS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
|
||||
if (config.skipIndexCreation()) {
|
||||
public void initialize(TileArchiveMetadata tileArchiveMetadata) {
|
||||
if (skipIndexCreation) {
|
||||
createTablesWithoutIndexes();
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
LOGGER.info("Skipping index creation. Add later by executing: {}",
|
||||
|
@ -168,26 +222,12 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
createTablesWithIndexes();
|
||||
}
|
||||
|
||||
var metadata = metadata()
|
||||
.setName(tileArchiveMetadata.name())
|
||||
.setFormat("pbf")
|
||||
.setDescription(tileArchiveMetadata.description())
|
||||
.setAttribution(tileArchiveMetadata.attribution())
|
||||
.setVersion(tileArchiveMetadata.version())
|
||||
.setType(tileArchiveMetadata.type())
|
||||
.setBoundsAndCenter(config.bounds().latLon())
|
||||
.setMinzoom(config.minzoom())
|
||||
.setMaxzoom(config.maxzoom())
|
||||
.setJson(new MetadataJson(layerStats.getTileStats()));
|
||||
|
||||
for (var entry : tileArchiveMetadata.planetilerSpecific().entrySet()) {
|
||||
metadata.setMetadata(entry.getKey(), entry.getValue());
|
||||
}
|
||||
metadataTable().set(tileArchiveMetadata);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finish(PlanetilerConfig config) {
|
||||
if (config.optimizeDb()) {
|
||||
public void finish(TileArchiveMetadata tileArchiveMetadata) {
|
||||
if (vacuumAnalyze) {
|
||||
vacuumAnalyze();
|
||||
}
|
||||
}
|
||||
|
@ -341,9 +381,9 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
return newTileWriter();
|
||||
}
|
||||
|
||||
/** Returns the contents of the metadata table. */
|
||||
public Metadata metadata() {
|
||||
return new Metadata();
|
||||
@Override
|
||||
public TileArchiveMetadata metadata() {
|
||||
return new Metadata().get();
|
||||
}
|
||||
|
||||
/** Returns the contents of the metadata table. */
|
||||
|
@ -389,12 +429,21 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
return connection;
|
||||
}
|
||||
|
||||
public boolean skipIndexCreation() {
|
||||
return skipIndexCreation;
|
||||
}
|
||||
|
||||
public boolean compactDb() {
|
||||
return compactDb;
|
||||
}
|
||||
|
||||
/**
|
||||
* Data contained in the {@code json} row of the metadata table
|
||||
*
|
||||
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#vector-tileset-metadata">MBtiles
|
||||
* schema</a>
|
||||
*/
|
||||
// TODO add tilestats
|
||||
public record MetadataJson(
|
||||
@JsonProperty("vector_layers") List<LayerStats.VectorLayer> vectorLayers
|
||||
) {
|
||||
|
@ -405,7 +454,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
|
||||
public static MetadataJson fromJson(String json) {
|
||||
try {
|
||||
return objectMapper.readValue(json, MetadataJson.class);
|
||||
return json == null ? null : objectMapper.readValue(json, MetadataJson.class);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalStateException("Invalid metadata json: " + json, e);
|
||||
}
|
||||
|
@ -466,6 +515,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
|
||||
/** Contents of a row of the tiles_data table. */
|
||||
private record TileDataEntry(int tileDataId, byte[] tileData) {
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TileDataEntry [tileDataId=" + tileDataId + ", tileData=" + Arrays.toString(tileData) + "]";
|
||||
|
@ -494,6 +544,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
|
||||
/** Iterates through tile coordinates one at a time without materializing the entire list in memory. */
|
||||
private class TileCoordIterator implements CloseableIterator<TileCoord> {
|
||||
|
||||
private final Statement statement;
|
||||
private final ResultSet rs;
|
||||
private boolean hasNext = false;
|
||||
|
@ -568,7 +619,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
insertStmtTableName = tableName;
|
||||
insertStmtInsertIgnore = insertIgnore;
|
||||
insertStmtValuesPlaceHolder = columns.stream().map(c -> "?").collect(Collectors.joining(",", "(", ")"));
|
||||
insertStmtColumnsCsv = columns.stream().collect(Collectors.joining(","));
|
||||
insertStmtColumnsCsv = String.join(",", columns);
|
||||
batchStatement = createBatchInsertPreparedStatement(batchLimit);
|
||||
}
|
||||
|
||||
|
@ -779,16 +830,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
/** Data contained in the metadata table. */
|
||||
public class Metadata {
|
||||
|
||||
private static final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
|
||||
|
||||
static {
|
||||
nf.setMaximumFractionDigits(5);
|
||||
}
|
||||
|
||||
private static String join(double... items) {
|
||||
return DoubleStream.of(items).mapToObj(nf::format).collect(Collectors.joining(","));
|
||||
}
|
||||
|
||||
/** Inserts a row into the metadata table that sets {@code name=value}. */
|
||||
public Metadata setMetadata(String name, Object value) {
|
||||
if (value != null) {
|
||||
String stringValue = value.toString();
|
||||
|
@ -810,79 +852,15 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
return this;
|
||||
}
|
||||
|
||||
public Metadata setName(String value) {
|
||||
return setMetadata("name", value);
|
||||
}
|
||||
|
||||
/** Format of the tile data, should always be pbf {@code pbf}. */
|
||||
public Metadata setFormat(String format) {
|
||||
return setMetadata("format", format);
|
||||
}
|
||||
|
||||
public Metadata setBounds(double left, double bottom, double right, double top) {
|
||||
return setMetadata("bounds", join(left, bottom, right, top));
|
||||
}
|
||||
|
||||
public Metadata setBounds(Envelope envelope) {
|
||||
return setBounds(envelope.getMinX(), envelope.getMinY(), envelope.getMaxX(), envelope.getMaxY());
|
||||
}
|
||||
|
||||
public Metadata setCenter(double longitude, double latitude, double zoom) {
|
||||
return setMetadata("center", join(longitude, latitude, zoom));
|
||||
}
|
||||
|
||||
public Metadata setBoundsAndCenter(Envelope envelope) {
|
||||
return setBounds(envelope).setCenter(envelope);
|
||||
}
|
||||
|
||||
/** Estimate a reasonable center for the map to fit an envelope. */
|
||||
public Metadata setCenter(Envelope envelope) {
|
||||
Coordinate center = envelope.centre();
|
||||
double zoom = Math.ceil(GeoUtils.getZoomFromLonLatBounds(envelope));
|
||||
return setCenter(center.x, center.y, zoom);
|
||||
}
|
||||
|
||||
public Metadata setMinzoom(int value) {
|
||||
return setMetadata("minzoom", value);
|
||||
}
|
||||
|
||||
public Metadata setMaxzoom(int maxZoom) {
|
||||
return setMetadata("maxzoom", maxZoom);
|
||||
}
|
||||
|
||||
public Metadata setAttribution(String value) {
|
||||
return setMetadata("attribution", value);
|
||||
}
|
||||
|
||||
public Metadata setDescription(String value) {
|
||||
return setMetadata("description", value);
|
||||
}
|
||||
|
||||
/** {@code overlay} or {@code baselayer}. */
|
||||
public Metadata setType(String value) {
|
||||
return setMetadata("type", value);
|
||||
}
|
||||
|
||||
public Metadata setTypeIsOverlay() {
|
||||
return setType("overlay");
|
||||
}
|
||||
|
||||
public Metadata setTypeIsBaselayer() {
|
||||
return setType("baselayer");
|
||||
}
|
||||
|
||||
public Metadata setVersion(String value) {
|
||||
return setMetadata("version", value);
|
||||
}
|
||||
|
||||
public Metadata setJson(String value) {
|
||||
return setMetadata("json", value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts a row into the metadata table that sets the value for {@code "json"} key to {@code value} serialized as a
|
||||
* string.
|
||||
*/
|
||||
public Metadata setJson(MetadataJson value) {
|
||||
return value == null ? this : setJson(value.toJson());
|
||||
return value == null ? this : setMetadata("json", value.toJson());
|
||||
}
|
||||
|
||||
/** Returns all key-value pairs from the metadata table. */
|
||||
public Map<String, String> getAll() {
|
||||
TreeMap<String, String> result = new TreeMap<>();
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
|
@ -895,10 +873,86 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
|
|||
);
|
||||
}
|
||||
} catch (SQLException throwables) {
|
||||
LOGGER.warn("Error retrieving metadata: " + throwables);
|
||||
LOGGER.warn("Error retrieving metadata: {}", throwables.toString());
|
||||
LOGGER.trace("Error retrieving metadata details: ", throwables);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts rows into the metadata table that set all of the well-known metadata keys from
|
||||
* {@code tileArchiveMetadata} and passes through the raw values of any options not explicitly called out in the
|
||||
* MBTiles specification.
|
||||
*
|
||||
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#content">MBTiles 1.3
|
||||
* specification</a>
|
||||
*/
|
||||
public Metadata set(TileArchiveMetadata tileArchiveMetadata) {
|
||||
var map = new LinkedHashMap<>(tileArchiveMetadata.toMap());
|
||||
|
||||
setMetadata(TileArchiveMetadata.FORMAT_KEY, tileArchiveMetadata.format());
|
||||
var center = tileArchiveMetadata.center();
|
||||
var zoom = tileArchiveMetadata.zoom();
|
||||
if (center != null) {
|
||||
if (zoom != null) {
|
||||
setMetadata(TileArchiveMetadata.CENTER_KEY, joinCoordinates(center.x, center.y, Math.ceil(zoom)));
|
||||
} else {
|
||||
setMetadata(TileArchiveMetadata.CENTER_KEY, joinCoordinates(center.x, center.y));
|
||||
}
|
||||
}
|
||||
var bounds = tileArchiveMetadata.bounds();
|
||||
if (bounds != null) {
|
||||
setMetadata(TileArchiveMetadata.BOUNDS_KEY,
|
||||
joinCoordinates(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY()));
|
||||
}
|
||||
setJson(new MetadataJson(tileArchiveMetadata.vectorLayers()));
|
||||
|
||||
map.remove(TileArchiveMetadata.FORMAT_KEY);
|
||||
map.remove(TileArchiveMetadata.CENTER_KEY);
|
||||
map.remove(TileArchiveMetadata.ZOOM_KEY);
|
||||
map.remove(TileArchiveMetadata.BOUNDS_KEY);
|
||||
map.remove(TileArchiveMetadata.VECTOR_LAYERS_KEY);
|
||||
|
||||
for (var entry : map.entrySet()) {
|
||||
setMetadata(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link TileArchiveMetadata} instance parsed from all the rows in the metadata table.
|
||||
*/
|
||||
public TileArchiveMetadata get() {
|
||||
Map<String, String> map = new HashMap<>(getAll());
|
||||
String[] bounds = map.containsKey(TileArchiveMetadata.BOUNDS_KEY) ?
|
||||
map.remove(TileArchiveMetadata.BOUNDS_KEY).split(",") : null;
|
||||
String[] center = map.containsKey(TileArchiveMetadata.CENTER_KEY) ?
|
||||
map.remove(TileArchiveMetadata.CENTER_KEY).split(",") : null;
|
||||
var metadataJson = MetadataJson.fromJson(map.remove("json"));
|
||||
return new TileArchiveMetadata(
|
||||
map.remove(TileArchiveMetadata.NAME_KEY),
|
||||
map.remove(TileArchiveMetadata.DESCRIPTION_KEY),
|
||||
map.remove(TileArchiveMetadata.ATTRIBUTION_KEY),
|
||||
map.remove(TileArchiveMetadata.VERSION_KEY),
|
||||
map.remove(TileArchiveMetadata.TYPE_KEY),
|
||||
map.remove(TileArchiveMetadata.FORMAT_KEY),
|
||||
bounds == null || bounds.length < 4 ? null : new Envelope(
|
||||
Double.parseDouble(bounds[0]),
|
||||
Double.parseDouble(bounds[2]),
|
||||
Double.parseDouble(bounds[1]),
|
||||
Double.parseDouble(bounds[3])
|
||||
),
|
||||
center == null || center.length < 2 ? null : new CoordinateXY(
|
||||
Double.parseDouble(center[0]),
|
||||
Double.parseDouble(center[1])
|
||||
),
|
||||
center == null || center.length < 3 ? null : Double.parseDouble(center[2]),
|
||||
Parse.parseIntOrNull(map.remove(TileArchiveMetadata.MINZOOM_KEY)),
|
||||
Parse.parseIntOrNull(map.remove(TileArchiveMetadata.MAXZOOM_KEY)),
|
||||
metadataJson == null ? null : metadataJson.vectorLayers,
|
||||
// any left-overs:
|
||||
map
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -189,7 +189,7 @@ public class Verify {
|
|||
}
|
||||
|
||||
private void checkBasicStructure() {
|
||||
check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name"));
|
||||
check("contains name attribute", () -> mbtiles.metadata().toMap().containsKey("name"));
|
||||
check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent());
|
||||
checkWithMessage("all tiles are valid", () -> {
|
||||
List<String> invalidTiles = mbtiles.getAllTileCoords().stream()
|
||||
|
|
|
@ -23,6 +23,8 @@ import java.util.HashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
||||
public class Pmtiles {
|
||||
public enum Compression {
|
||||
|
@ -201,6 +203,22 @@ public class Pmtiles {
|
|||
throw new FileFormatException("Failed to read enough bytes for PMTiles header.");
|
||||
}
|
||||
}
|
||||
|
||||
public Envelope bounds() {
|
||||
return new Envelope(
|
||||
minLonE7 / 1e7,
|
||||
maxLonE7 / 1e7,
|
||||
minLatE7 / 1e7,
|
||||
maxLatE7 / 1e7
|
||||
);
|
||||
}
|
||||
|
||||
public CoordinateXY center() {
|
||||
return new CoordinateXY(
|
||||
centerLonE7 / 1e7,
|
||||
centerLatE7 / 1e7
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class Entry implements Comparable<Entry> {
|
||||
|
@ -366,7 +384,7 @@ public class Pmtiles {
|
|||
try {
|
||||
return objectMapper.readValue(bytes, JsonMetadata.class);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("Invalid metadata json: " + bytes, e);
|
||||
throw new IllegalStateException("Invalid metadata json: " + new String(bytes, StandardCharsets.UTF_8), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,13 +1,19 @@
|
|||
package com.onthegomap.planetiler.pmtiles;
|
||||
|
||||
import com.onthegomap.planetiler.archive.ReadableTileArchive;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.util.CloseableIterator;
|
||||
import com.onthegomap.planetiler.util.Gzip;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
|
@ -22,6 +28,10 @@ public class ReadablePmtiles implements ReadableTileArchive {
|
|||
this.header = Pmtiles.Header.fromBytes(getBytes(0, Pmtiles.HEADER_LEN));
|
||||
}
|
||||
|
||||
public static ReadableTileArchive newReadFromFile(Path path) throws IOException {
|
||||
return new ReadablePmtiles(FileChannel.open(path, StandardOpenOption.READ));
|
||||
}
|
||||
|
||||
private synchronized byte[] getBytes(long start, int length) throws IOException {
|
||||
channel.position(start);
|
||||
var buf = ByteBuffer.allocate(length);
|
||||
|
@ -103,6 +113,34 @@ public class ReadablePmtiles implements ReadableTileArchive {
|
|||
return Pmtiles.JsonMetadata.fromBytes(buf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileArchiveMetadata metadata() {
|
||||
try {
|
||||
var jsonMetadata = getJsonMetadata();
|
||||
var map = new LinkedHashMap<>(jsonMetadata.otherMetadata());
|
||||
return new TileArchiveMetadata(
|
||||
map.remove(TileArchiveMetadata.NAME_KEY),
|
||||
map.remove(TileArchiveMetadata.DESCRIPTION_KEY),
|
||||
map.remove(TileArchiveMetadata.ATTRIBUTION_KEY),
|
||||
map.remove(TileArchiveMetadata.VERSION_KEY),
|
||||
map.remove(TileArchiveMetadata.TYPE_KEY),
|
||||
switch (header.tileType()) {
|
||||
case MVT -> TileArchiveMetadata.MVT_FORMAT;
|
||||
default -> null;
|
||||
},
|
||||
header.bounds(),
|
||||
header.center(),
|
||||
(double) header.centerZoom(),
|
||||
(int) header.minZoom(),
|
||||
(int) header.maxZoom(),
|
||||
jsonMetadata.vectorLayers(),
|
||||
map
|
||||
);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static class TileCoordIterator implements CloseableIterator<TileCoord> {
|
||||
private final Stream<TileCoord> stream;
|
||||
private final Iterator<TileCoord> iterator;
|
||||
|
|
|
@ -12,7 +12,6 @@ import com.onthegomap.planetiler.geo.TileCoord;
|
|||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.util.Format;
|
||||
import com.onthegomap.planetiler.util.Gzip;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import com.onthegomap.planetiler.util.SeekableInMemoryByteChannel;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
|
@ -24,10 +23,10 @@ import java.nio.file.StandardOpenOption;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalLong;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
@ -46,8 +45,6 @@ public final class WriteablePmtiles implements WriteableTileArchive {
|
|||
private long currentOffset = 0;
|
||||
private long numUnhashedTiles = 0;
|
||||
private long numAddressedTiles = 0;
|
||||
private LayerStats layerStats;
|
||||
private TileArchiveMetadata tileArchiveMetadata;
|
||||
private boolean isClustered = true;
|
||||
|
||||
private WriteablePmtiles(SeekableByteChannel channel) throws IOException {
|
||||
|
@ -91,7 +88,7 @@ public final class WriteablePmtiles implements WriteableTileArchive {
|
|||
* @return byte arrays of the root and all leaf directories, and the # of leaves.
|
||||
* @throws IOException if compression fails
|
||||
*/
|
||||
protected static Directories makeDirectories(List<Pmtiles.Entry> entries) throws IOException {
|
||||
static Directories makeDirectories(List<Pmtiles.Entry> entries) throws IOException {
|
||||
int maxEntriesRootOnly = 16384;
|
||||
int attemptNum = 1;
|
||||
if (entries.size() < maxEntriesRootOnly) {
|
||||
|
@ -124,19 +121,18 @@ public final class WriteablePmtiles implements WriteableTileArchive {
|
|||
return new WriteablePmtiles(bytes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deduplicates() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileOrder tileOrder() {
|
||||
return TileOrder.HILBERT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
|
||||
this.layerStats = layerStats;
|
||||
this.tileArchiveMetadata = tileArchiveMetadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finish(PlanetilerConfig config) {
|
||||
public void finish(TileArchiveMetadata tileArchiveMetadata) {
|
||||
if (!isClustered) {
|
||||
LOGGER.info("Tile data was not written in order, sorting entries...");
|
||||
Collections.sort(entries);
|
||||
|
@ -144,10 +140,32 @@ public final class WriteablePmtiles implements WriteableTileArchive {
|
|||
}
|
||||
try {
|
||||
Directories directories = makeDirectories(entries);
|
||||
byte[] jsonBytes = new Pmtiles.JsonMetadata(layerStats.getTileStats(), tileArchiveMetadata.getAll()).toBytes();
|
||||
var otherMetadata = new LinkedHashMap<>(tileArchiveMetadata.toMap());
|
||||
|
||||
// exclude keys included in top-level header
|
||||
otherMetadata.remove(TileArchiveMetadata.CENTER_KEY);
|
||||
otherMetadata.remove(TileArchiveMetadata.ZOOM_KEY);
|
||||
otherMetadata.remove(TileArchiveMetadata.BOUNDS_KEY);
|
||||
otherMetadata.remove(TileArchiveMetadata.FORMAT_KEY);
|
||||
otherMetadata.remove(TileArchiveMetadata.MINZOOM_KEY);
|
||||
otherMetadata.remove(TileArchiveMetadata.MAXZOOM_KEY);
|
||||
otherMetadata.remove(TileArchiveMetadata.VECTOR_LAYERS_KEY);
|
||||
|
||||
byte[] jsonBytes =
|
||||
new Pmtiles.JsonMetadata(tileArchiveMetadata.vectorLayers(), otherMetadata).toBytes();
|
||||
jsonBytes = Gzip.gzip(jsonBytes);
|
||||
|
||||
Envelope envelope = config.bounds().latLon();
|
||||
String formatString = tileArchiveMetadata.format();
|
||||
var outputFormat =
|
||||
TileArchiveMetadata.MVT_FORMAT.equals(formatString) ? Pmtiles.TileType.MVT : Pmtiles.TileType.UNKNOWN;
|
||||
|
||||
var bounds = tileArchiveMetadata.bounds() == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : tileArchiveMetadata.bounds();
|
||||
var center = tileArchiveMetadata.center() == null ? bounds.centre() : tileArchiveMetadata.center();
|
||||
int zoom = (int) Math.ceil(tileArchiveMetadata.zoom() == null ? GeoUtils.getZoomFromLonLatBounds(bounds) :
|
||||
tileArchiveMetadata.zoom());
|
||||
int minzoom = tileArchiveMetadata.minzoom() == null ? 0 : tileArchiveMetadata.minzoom();
|
||||
int maxzoom =
|
||||
tileArchiveMetadata.maxzoom() == null ? PlanetilerConfig.MAX_MAXZOOM : tileArchiveMetadata.maxzoom();
|
||||
|
||||
Pmtiles.Header header = new Pmtiles.Header(
|
||||
(byte) 3,
|
||||
|
@ -165,16 +183,16 @@ public final class WriteablePmtiles implements WriteableTileArchive {
|
|||
isClustered,
|
||||
Pmtiles.Compression.GZIP,
|
||||
Pmtiles.Compression.GZIP,
|
||||
Pmtiles.TileType.MVT,
|
||||
(byte) config.minzoom(),
|
||||
(byte) config.maxzoom(),
|
||||
(int) (envelope.getMinX() * 10_000_000),
|
||||
(int) (envelope.getMinY() * 10_000_000),
|
||||
(int) (envelope.getMaxX() * 10_000_000),
|
||||
(int) (envelope.getMaxY() * 10_000_000),
|
||||
(byte) Math.ceil(GeoUtils.getZoomFromLonLatBounds(envelope)),
|
||||
(int) ((envelope.getMinX() + envelope.getMaxX()) / 2 * 10_000_000),
|
||||
(int) ((envelope.getMinY() + envelope.getMaxY()) / 2 * 10_000_000)
|
||||
outputFormat,
|
||||
(byte) minzoom,
|
||||
(byte) maxzoom,
|
||||
(int) (bounds.getMinX() * 10_000_000),
|
||||
(int) (bounds.getMinY() * 10_000_000),
|
||||
(int) (bounds.getMaxX() * 10_000_000),
|
||||
(int) (bounds.getMaxY() * 10_000_000),
|
||||
(byte) zoom,
|
||||
(int) center.x * 10_000_000,
|
||||
(int) center.y * 10_000_000
|
||||
);
|
||||
|
||||
LOGGER.info("Writing metadata and leaf directories...");
|
||||
|
|
|
@ -43,15 +43,17 @@ public interface Stats extends AutoCloseable {
|
|||
*/
|
||||
default void printSummary() {
|
||||
Format format = Format.defaultInstance();
|
||||
Logger LOGGER = LoggerFactory.getLogger(getClass());
|
||||
LOGGER.info("");
|
||||
LOGGER.info("-".repeat(40));
|
||||
timers().printSummary();
|
||||
LOGGER.info("-".repeat(40));
|
||||
for (var entry : monitoredFiles().entrySet()) {
|
||||
long size = FileUtils.size(entry.getValue());
|
||||
if (size > 0) {
|
||||
LOGGER.info("\t" + entry.getKey() + "\t" + format.storage(size, false) + "B");
|
||||
Logger logger = LoggerFactory.getLogger(getClass());
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("");
|
||||
logger.info("-".repeat(40));
|
||||
timers().printSummary();
|
||||
logger.info("-".repeat(40));
|
||||
for (var entry : monitoredFiles().entrySet()) {
|
||||
long size = FileUtils.size(entry.getValue());
|
||||
if (size > 0) {
|
||||
logger.info("\t{}\t{}B", entry.getKey(), format.storage(size, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -110,7 +112,9 @@ public interface Stats extends AutoCloseable {
|
|||
|
||||
/** Adds a stat that will track the size of a file or directory located at {@code path}. */
|
||||
default void monitorFile(String name, Path path) {
|
||||
monitoredFiles().put(name, path);
|
||||
if (path != null) {
|
||||
monitoredFiles().put(name, path);
|
||||
}
|
||||
}
|
||||
|
||||
/** Adds a stat that will track the estimated in-memory size of {@code object}. */
|
||||
|
|
|
@ -217,17 +217,19 @@ public class FileUtils {
|
|||
*/
|
||||
public static void createParentDirectories(Path... paths) {
|
||||
for (var path : paths) {
|
||||
try {
|
||||
if (Files.isDirectory(path) && !Files.exists(path)) {
|
||||
Files.createDirectories(path);
|
||||
} else {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null && !Files.exists(parent)) {
|
||||
Files.createDirectories(parent);
|
||||
if (path != null) {
|
||||
try {
|
||||
if (Files.isDirectory(path) && !Files.exists(path)) {
|
||||
Files.createDirectories(path);
|
||||
} else {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null && !Files.exists(parent)) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("Unable to create parent directories " + path, e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("Unable to create parent directories " + path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,6 +8,8 @@ import java.util.NavigableMap;
|
|||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.DoubleStream;
|
||||
import org.apache.commons.text.StringEscapeUtils;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
|
||||
|
@ -19,6 +21,26 @@ public class Format {
|
|||
public static final Locale DEFAULT_LOCALE = Locale.getDefault(Locale.Category.FORMAT);
|
||||
|
||||
private static final ConcurrentMap<Locale, Format> instances = new ConcurrentHashMap<>();
|
||||
@SuppressWarnings("java:S5164")
|
||||
private static final NumberFormat latLonNF = NumberFormat.getNumberInstance(Locale.US);
|
||||
private static final NavigableMap<Long, String> STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries(
|
||||
Map.entry(1_000L, "k"),
|
||||
Map.entry(1_000_000L, "M"),
|
||||
Map.entry(1_000_000_000L, "G"),
|
||||
Map.entry(1_000_000_000_000L, "T"),
|
||||
Map.entry(1_000_000_000_000_000L, "P")
|
||||
));
|
||||
private static final NavigableMap<Long, String> NUMERIC_SUFFIXES = new TreeMap<>(Map.ofEntries(
|
||||
Map.entry(1_000L, "k"),
|
||||
Map.entry(1_000_000L, "M"),
|
||||
Map.entry(1_000_000_000L, "B"),
|
||||
Map.entry(1_000_000_000_000L, "T"),
|
||||
Map.entry(1_000_000_000_000_000L, "Q")
|
||||
));
|
||||
|
||||
static {
|
||||
latLonNF.setMaximumFractionDigits(5);
|
||||
}
|
||||
|
||||
// `NumberFormat` instances are not thread safe, so we need to wrap them inside a `ThreadLocal`.
|
||||
//
|
||||
|
@ -49,6 +71,11 @@ public class Format {
|
|||
});
|
||||
}
|
||||
|
||||
/** Returns a string with {@code items} rounded to 5 decimals and joined with a comma. */
|
||||
public static synchronized String joinCoordinates(double... items) {
|
||||
return DoubleStream.of(items).mapToObj(latLonNF::format).collect(Collectors.joining(","));
|
||||
}
|
||||
|
||||
public static Format forLocale(Locale locale) {
|
||||
return instances.computeIfAbsent(locale, Format::new);
|
||||
}
|
||||
|
@ -57,21 +84,6 @@ public class Format {
|
|||
return forLocale(DEFAULT_LOCALE);
|
||||
}
|
||||
|
||||
private static final NavigableMap<Long, String> STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries(
|
||||
Map.entry(1_000L, "k"),
|
||||
Map.entry(1_000_000L, "M"),
|
||||
Map.entry(1_000_000_000L, "G"),
|
||||
Map.entry(1_000_000_000_000L, "T"),
|
||||
Map.entry(1_000_000_000_000_000L, "P")
|
||||
));
|
||||
private static final NavigableMap<Long, String> NUMERIC_SUFFIXES = new TreeMap<>(Map.ofEntries(
|
||||
Map.entry(1_000L, "k"),
|
||||
Map.entry(1_000_000L, "M"),
|
||||
Map.entry(1_000_000_000L, "B"),
|
||||
Map.entry(1_000_000_000_000L, "T"),
|
||||
Map.entry(1_000_000_000_000_000L, "Q")
|
||||
));
|
||||
|
||||
public static String padRight(String str, int size) {
|
||||
StringBuilder strBuilder = new StringBuilder(str);
|
||||
while (strBuilder.length() < size) {
|
||||
|
@ -88,6 +100,23 @@ public class Format {
|
|||
return strBuilder.toString();
|
||||
}
|
||||
|
||||
/** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */
|
||||
public static String quote(Object string) {
|
||||
if (string == null) {
|
||||
return "null";
|
||||
}
|
||||
return '"' + StringEscapeUtils.escapeJava(string.toString()) + '"';
|
||||
}
|
||||
|
||||
/** Returns an openstreetmap.org map link for a lat/lon */
|
||||
public static String osmDebugUrl(int zoom, Coordinate coord) {
|
||||
return "https://www.openstreetmap.org/#map=%d/%.5f/%.5f".formatted(
|
||||
zoom,
|
||||
coord.y,
|
||||
coord.x
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns a number of bytes formatted like "123" "1.2k" "240M", etc. */
|
||||
public String storage(Number num, boolean pad) {
|
||||
return format(num, pad, STORAGE_SUFFIXES);
|
||||
|
@ -161,21 +190,4 @@ public class Format {
|
|||
}
|
||||
return simplified.toString().replace("PT", "").toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
/** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */
|
||||
public static String quote(Object string) {
|
||||
if (string == null) {
|
||||
return "null";
|
||||
}
|
||||
return '"' + StringEscapeUtils.escapeJava(string.toString()) + '"';
|
||||
}
|
||||
|
||||
/** Returns an openstreetmap.org map link for a lat/lon */
|
||||
public static String osmDebugUrl(int zoom, Coordinate coord) {
|
||||
return "https://www.openstreetmap.org/#map=%d/%.5f/%.5f".formatted(
|
||||
zoom,
|
||||
coord.y,
|
||||
coord.x
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -70,7 +70,7 @@ public class ResourceUsage {
|
|||
|
||||
/** Requests {@code amount} bytes on the file system that contains {@code path}. */
|
||||
public ResourceUsage addDisk(Path path, long amount, String description) {
|
||||
return add(new DiskUsage(path), amount, description);
|
||||
return path == null ? this : add(new DiskUsage(path), amount, description);
|
||||
}
|
||||
|
||||
/** Requests {@code amount} bytes of RAM in the JVM heap. */
|
||||
|
|
|
@ -15,6 +15,7 @@ import com.onthegomap.planetiler.geo.GeometryException;
|
|||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
|
||||
import com.onthegomap.planetiler.reader.SimpleFeature;
|
||||
import com.onthegomap.planetiler.reader.SimpleReader;
|
||||
import com.onthegomap.planetiler.reader.SourceFeature;
|
||||
|
@ -141,14 +142,14 @@ class PlanetilerTests {
|
|||
FeatureGroup featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, stats);
|
||||
runner.run(featureGroup, profile, config);
|
||||
featureGroup.prepare();
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.compactDb())) {
|
||||
TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config.arguments()),
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.arguments())) {
|
||||
TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config),
|
||||
config,
|
||||
stats);
|
||||
var tileMap = TestUtils.getTileMap(db);
|
||||
tileMap.values().forEach(fs -> fs.forEach(f -> f.geometry().validate()));
|
||||
int tileDataCount = config.compactDb() ? TestUtils.getTilesDataCount(db) : 0;
|
||||
return new PlanetilerResults(tileMap, db.metadata().getAll(), tileDataCount);
|
||||
int tileDataCount = db.compactDb() ? TestUtils.getTilesDataCount(db) : 0;
|
||||
return new PlanetilerResults(tileMap, db.metadata().toMap(), tileDataCount);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -248,20 +249,15 @@ class PlanetilerTests {
|
|||
"format", "pbf",
|
||||
"minzoom", "0",
|
||||
"maxzoom", "14",
|
||||
"center", "0,0,0",
|
||||
"center", "0,0",
|
||||
"bounds", "-180,-85.05113,180,85.05113"
|
||||
), results.metadata);
|
||||
assertSubmap(Map.of(
|
||||
"planetiler:version", BuildInfo.get().version()
|
||||
), results.metadata);
|
||||
assertSameJson(
|
||||
"""
|
||||
{
|
||||
"vector_layers": [
|
||||
]
|
||||
}
|
||||
""",
|
||||
results.metadata.get("json")
|
||||
"[]",
|
||||
results.metadata.get("vector_layers")
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -269,11 +265,11 @@ class PlanetilerTests {
|
|||
void testOverrideMetadata() throws Exception {
|
||||
var results = runWithReaderFeatures(
|
||||
Map.of(
|
||||
"mbtiles_name", "override_name",
|
||||
"mbtiles_description", "override_description",
|
||||
"mbtiles_attribution", "override_attribution",
|
||||
"mbtiles_version", "override_version",
|
||||
"mbtiles_type", "override_type"
|
||||
"archive_name", "override_name",
|
||||
"archive_description", "override_description",
|
||||
"archive_attribution", "override_attribution",
|
||||
"archive_version", "override_version",
|
||||
"archive_type", "override_type"
|
||||
),
|
||||
List.of(),
|
||||
(sourceFeature, features) -> {
|
||||
|
@ -331,13 +327,11 @@ class PlanetilerTests {
|
|||
), results.tiles);
|
||||
assertSameJson(
|
||||
"""
|
||||
{
|
||||
"vector_layers": [
|
||||
{"id": "layer", "fields": {"name": "String", "attr": "String"}, "minzoom": 13, "maxzoom": 15}
|
||||
]
|
||||
}
|
||||
[
|
||||
{"id": "layer", "fields": {"name": "String", "attr": "String"}, "minzoom": 13, "maxzoom": 15}
|
||||
]
|
||||
""",
|
||||
results.metadata.get("json")
|
||||
results.metadata.get("vector_layers")
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -1686,13 +1680,14 @@ class PlanetilerTests {
|
|||
@ValueSource(strings = {
|
||||
"",
|
||||
"--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4",
|
||||
"--emit-tiles-in-order=false",
|
||||
"--free-osm-after-read",
|
||||
"--osm-parse-node-bounds",
|
||||
"--output-format=pmtiles"
|
||||
})
|
||||
void testPlanetilerRunner(String args) throws Exception {
|
||||
boolean pmtiles = args.contains("pmtiles");
|
||||
Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf");
|
||||
Path mbtiles = tempDir.resolve("output.mbtiles");
|
||||
Path output = tempDir.resolve(pmtiles ? "output.pmtiles" : "output.mbtiles");
|
||||
Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf");
|
||||
Files.copy(originalOsm, tempOsm);
|
||||
Planetiler.create(Arguments.fromArgs(
|
||||
|
@ -1710,7 +1705,7 @@ class PlanetilerTests {
|
|||
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
|
||||
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
|
||||
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null)
|
||||
.setOutput("mbtiles", mbtiles)
|
||||
.setOutput(output)
|
||||
.run();
|
||||
|
||||
// make sure it got deleted after write
|
||||
|
@ -1718,7 +1713,9 @@ class PlanetilerTests {
|
|||
assertFalse(Files.exists(tempOsm));
|
||||
}
|
||||
|
||||
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
|
||||
try (
|
||||
var db = pmtiles ? ReadablePmtiles.newReadFromFile(output) : Mbtiles.newReadOnlyDatabase(output)
|
||||
) {
|
||||
int features = 0;
|
||||
var tileMap = TestUtils.getTileMap(db);
|
||||
for (var tile : tileMap.values()) {
|
||||
|
@ -1735,7 +1732,7 @@ class PlanetilerTests {
|
|||
"planetiler:osm:osmosisreplicationtime", "2021-04-21T20:21:46Z",
|
||||
"planetiler:osm:osmosisreplicationseq", "2947",
|
||||
"planetiler:osm:osmosisreplicationurl", "http://download.geofabrik.de/europe/monaco-updates"
|
||||
), db.metadata().getAll());
|
||||
), db.metadata().toMap());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1760,7 +1757,7 @@ class PlanetilerTests {
|
|||
.addShapefileGlobSource("shapefile-glob-zip", resourceDir.resolve("shapefile.zip"), "*.shp")
|
||||
// Match *.shp within shapefile.zip
|
||||
.addShapefileSource("shapefile", resourceDir.resolve("shapefile.zip"))
|
||||
.setOutput("mbtiles", mbtiles)
|
||||
.setOutput(mbtiles)
|
||||
.run();
|
||||
|
||||
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
|
||||
|
@ -1806,7 +1803,7 @@ class PlanetilerTests {
|
|||
}
|
||||
})
|
||||
.addGeoPackageSource("geopackage", TestUtils.pathToResource(inputFile), null)
|
||||
.setOutput("mbtiles", mbtiles)
|
||||
.setOutput(mbtiles)
|
||||
.run();
|
||||
|
||||
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
|
||||
|
@ -1834,7 +1831,7 @@ class PlanetilerTests {
|
|||
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
|
||||
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
|
||||
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null)
|
||||
.setOutput("mbtiles", tempDir.resolve("output.mbtiles"))
|
||||
.setOutput(tempDir.resolve("output.mbtiles"))
|
||||
.run();
|
||||
}
|
||||
|
||||
|
@ -1909,9 +1906,8 @@ class PlanetilerTests {
|
|||
|
||||
|
||||
private PlanetilerResults runForCompactTest(boolean compactDbEnabled) throws Exception {
|
||||
|
||||
return runWithReaderFeatures(
|
||||
Map.of("threads", "1", "compact-db", Boolean.toString(compactDbEnabled)),
|
||||
Map.of("threads", "1", "mbtiles-compact", Boolean.toString(compactDbEnabled)),
|
||||
List.of(
|
||||
newReaderFeature(WORLD_POLYGON, Map.of())
|
||||
),
|
||||
|
|
|
@ -15,6 +15,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
|
|||
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
|
||||
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.onthegomap.planetiler.archive.ReadableTileArchive;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.geo.GeometryException;
|
||||
|
@ -199,7 +200,8 @@ public class TestUtils {
|
|||
return round(input, 1e5);
|
||||
}
|
||||
|
||||
public static Map<TileCoord, List<ComparableFeature>> getTileMap(Mbtiles db) throws SQLException, IOException {
|
||||
public static Map<TileCoord, List<ComparableFeature>> getTileMap(ReadableTileArchive db)
|
||||
throws IOException {
|
||||
Map<TileCoord, List<ComparableFeature>> tiles = new TreeMap<>();
|
||||
for (var tile : getAllTiles(db)) {
|
||||
var bytes = gunzip(tile.bytes());
|
||||
|
@ -218,21 +220,10 @@ public class TestUtils {
|
|||
}
|
||||
}
|
||||
|
||||
public static Set<Mbtiles.TileEntry> getAllTiles(Mbtiles db) throws SQLException {
|
||||
Set<Mbtiles.TileEntry> result = new HashSet<>();
|
||||
try (Statement statement = db.connection().createStatement()) {
|
||||
ResultSet rs = statement.executeQuery("select zoom_level, tile_column, tile_row, tile_data from tiles");
|
||||
while (rs.next()) {
|
||||
int z = rs.getInt("zoom_level");
|
||||
int rawy = rs.getInt("tile_row");
|
||||
int x = rs.getInt("tile_column");
|
||||
result.add(new Mbtiles.TileEntry(
|
||||
TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z),
|
||||
rs.getBytes("tile_data")
|
||||
));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
public static Set<Mbtiles.TileEntry> getAllTiles(ReadableTileArchive db) {
|
||||
return db.getAllTileCoords().stream()
|
||||
.map(coord -> new Mbtiles.TileEntry(coord, db.getTile(coord)))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public static int getTilesDataCount(Mbtiles db) throws SQLException {
|
||||
|
|
|
@ -0,0 +1,36 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class TileArchiveConfigTest {
|
||||
|
||||
@Test
|
||||
void testMbtiles() {
|
||||
var config = TileArchiveConfig.from("output.mbtiles");
|
||||
assertEquals(TileArchiveConfig.Format.MBTILES, config.format());
|
||||
assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme());
|
||||
assertEquals(Map.of(), config.options());
|
||||
assertEquals(Path.of("output.mbtiles").toAbsolutePath(), config.getLocalPath());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMbtilesWithOptions() {
|
||||
var config = TileArchiveConfig.from("output.mbtiles?compact=true");
|
||||
assertEquals(TileArchiveConfig.Format.MBTILES, config.format());
|
||||
assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme());
|
||||
assertEquals(Map.of("compact", "true"), config.options());
|
||||
assertEquals(Path.of("output.mbtiles").toAbsolutePath(), config.getLocalPath());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPmtiles() {
|
||||
assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("output.pmtiles").format());
|
||||
assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("output.mbtiles?format=pmtiles").format());
|
||||
assertEquals(TileArchiveConfig.Format.PMTILES,
|
||||
TileArchiveConfig.from("file:///output.mbtiles?format=pmtiles").format());
|
||||
}
|
||||
}
|
|
@ -0,0 +1,67 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
||||
class TileArchiveMetadataTest {
|
||||
|
||||
@Test
|
||||
void testAddMetadataWorldBounds() {
|
||||
var bounds = GeoUtils.WORLD_LAT_LON_BOUNDS;
|
||||
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), PlanetilerConfig.from(Arguments.of(Map.of(
|
||||
"bounds", bounds.getMinX() + "," + bounds.getMinY() + "," + bounds.getMaxX() + "," + bounds.getMaxY()
|
||||
))));
|
||||
assertEquals(bounds, metadata.bounds());
|
||||
assertEquals(new CoordinateXY(0, 0), metadata.center());
|
||||
assertEquals(0d, metadata.zoom().doubleValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAddMetadataSmallBounds() {
|
||||
var bounds = new Envelope(-73.6632, -69.7598, 41.1274, 43.0185);
|
||||
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), PlanetilerConfig.from(Arguments.of(Map.of(
|
||||
"bounds", "-73.6632,41.1274,-69.7598,43.0185"
|
||||
))));
|
||||
assertEquals(bounds, metadata.bounds());
|
||||
assertEquals(-71.7115, metadata.center().x, 1e-5);
|
||||
assertEquals(42.07295, metadata.center().y, 1e-5);
|
||||
assertEquals(7, Math.ceil(metadata.zoom()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testToMap() {
|
||||
var bounds = "-73.6632,41.1274,-69.7598,43.0185";
|
||||
var metadata = new TileArchiveMetadata(
|
||||
new Profile.NullProfile(),
|
||||
PlanetilerConfig.from(Arguments.of(Map.of(
|
||||
"bounds", bounds
|
||||
))));
|
||||
var map = new TreeMap<>(metadata.toMap());
|
||||
assertNotNull(map.remove("planetiler:version"));
|
||||
assertNotNull(map.remove("planetiler:githash"));
|
||||
assertNotNull(map.remove("planetiler:buildtime"));
|
||||
assertEquals(
|
||||
new TreeMap<String, String>(Map.of(
|
||||
"name", "Null",
|
||||
"type", "baselayer",
|
||||
"format", "pbf",
|
||||
"zoom", "6.5271217861412305",
|
||||
"minzoom", "0",
|
||||
"maxzoom", "14",
|
||||
"bounds", "-73.6632,41.1274,-69.7598,43.0185",
|
||||
"center", "-71.7115,42.07295"
|
||||
)),
|
||||
map
|
||||
);
|
||||
}
|
||||
}
|
|
@ -9,6 +9,7 @@ import java.nio.file.Path;
|
|||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
||||
|
@ -293,4 +294,59 @@ class ArgumentsTest {
|
|||
assertEquals(false, args.getBooleanObject("BOOL_FALSE", "test"));
|
||||
assertEquals(false, args.getBooleanObject("BOOL_NO", "test"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDeprecatedArgs() {
|
||||
assertEquals("newvalue",
|
||||
Arguments.of("oldkey", "oldvalue", "newkey", "newvalue")
|
||||
.getString("newkey|oldkey", "key", "fallback"));
|
||||
assertEquals("oldvalue",
|
||||
Arguments.of("oldkey", "oldvalue")
|
||||
.getString("newkey|oldkey", "key", "fallback"));
|
||||
assertEquals("fallback",
|
||||
Arguments.of()
|
||||
.getString("newkey|oldkey", "key", "fallback"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWithPrefix() {
|
||||
var args = Arguments.of("prefix_a", "a_val", "prefix-b", "b_val", "other", "other_val").withPrefix("prefix");
|
||||
assertEquals("a_val", args.getArg("a"));
|
||||
assertEquals("b_val", args.getArg("b"));
|
||||
assertNull(args.getArg("other"));
|
||||
assertNull(args.getArg("prefix_a"));
|
||||
assertNull(args.getArg("prefix_b"));
|
||||
assertNull(args.getArg("prefix_other"));
|
||||
assertEquals(Set.of("a", "b"), args.toMap().keySet());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPrefixFromEnvironment() {
|
||||
Map<String, String> env = Map.of(
|
||||
"OTHER", "value",
|
||||
"PLANETILEROTHER", "VALUE",
|
||||
"PLANETILER_MBTILES_KEY1", "value1",
|
||||
"PLANETILER_PMTILES_KEY2", "value2"
|
||||
);
|
||||
Arguments args = Arguments.fromEnvironment(env::get, env::keySet).withPrefix("mbtiles");
|
||||
assertEquals(Map.of(
|
||||
"key1", "value1"
|
||||
), args.toMap());
|
||||
assertEquals("value1", args.getArg("key1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSubset() {
|
||||
var args = Arguments.of(Map.of(
|
||||
"key_1", "val_1",
|
||||
"key-2", "val_2",
|
||||
"key-3", "val_3"
|
||||
)).subset("key-1", "key-2");
|
||||
assertEquals(Map.of(
|
||||
"key_1", "val_1",
|
||||
"key_2", "val_2"
|
||||
), args.toMap());
|
||||
assertEquals("val_1", args.getArg("key-1"));
|
||||
assertNull(args.getArg("key-3"));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,24 +5,27 @@ import static org.junit.jupiter.api.Assertions.*;
|
|||
|
||||
import com.google.common.math.IntMath;
|
||||
import com.onthegomap.planetiler.TestUtils;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import java.io.IOException;
|
||||
import java.math.RoundingMode;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
import java.util.stream.Collectors;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
||||
class MbtilesTest {
|
||||
|
@ -33,11 +36,12 @@ class MbtilesTest {
|
|||
private static final int TILES_DATA_BATCH = MAX_PARAMETERS_IN_PREPARED_STATEMENT / 2;
|
||||
|
||||
|
||||
private static final
|
||||
|
||||
void testWriteTiles(int howMany, boolean skipIndexCreation, boolean optimize, boolean compactDb)
|
||||
throws IOException, SQLException {
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase(compactDb)) {
|
||||
private static void testWriteTiles(Path path, int howMany, boolean skipIndexCreation, boolean optimize,
|
||||
boolean compactDb) throws IOException, SQLException {
|
||||
var options = Arguments.of("compact", Boolean.toString(compactDb));
|
||||
try (
|
||||
Mbtiles db = path == null ? Mbtiles.newInMemoryDatabase(options) : Mbtiles.newWriteToFileDatabase(path, options)
|
||||
) {
|
||||
if (skipIndexCreation) {
|
||||
db.createTablesWithoutIndexes();
|
||||
} else {
|
||||
|
@ -84,24 +88,42 @@ class MbtilesTest {
|
|||
@ParameterizedTest
|
||||
@ValueSource(ints = {0, 1, TILES_BATCH, TILES_BATCH + 1, 2 * TILES_BATCH, 2 * TILES_BATCH + 1})
|
||||
void testWriteTilesDifferentSizeInNonCompactMode(int howMany) throws IOException, SQLException {
|
||||
testWriteTiles(howMany, false, false, false);
|
||||
testWriteTiles(null, howMany, false, false, false);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = {0, 1, TILES_DATA_BATCH, TILES_DATA_BATCH + 1, 2 * TILES_DATA_BATCH, 2 * TILES_DATA_BATCH + 1,
|
||||
TILES_SHALLOW_BATCH, TILES_SHALLOW_BATCH + 1, 2 * TILES_SHALLOW_BATCH, 2 * TILES_SHALLOW_BATCH + 1})
|
||||
void testWriteTilesDifferentSizeInCompactMode(int howMany) throws IOException, SQLException {
|
||||
testWriteTiles(howMany, false, false, true);
|
||||
testWriteTiles(null, howMany, false, false, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSkipIndexCreation() throws IOException, SQLException {
|
||||
testWriteTiles(10, true, false, false);
|
||||
testWriteTiles(null, 10, true, false, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testVacuumAnalyze() throws IOException, SQLException {
|
||||
testWriteTiles(10, false, true, false);
|
||||
testWriteTiles(null, 10, false, true, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWriteToFile(@TempDir Path tmpDir) throws IOException, SQLException {
|
||||
testWriteTiles(tmpDir.resolve("archive.mbtiles"), 10, false, false, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCustomPragma() throws IOException, SQLException {
|
||||
try (
|
||||
Mbtiles db = Mbtiles.newInMemoryDatabase(Arguments.of(
|
||||
"cache-size", "123",
|
||||
"garbage", "456"
|
||||
));
|
||||
) {
|
||||
int result = db.connection().createStatement().executeQuery("pragma cache_size").getInt(1);
|
||||
assertEquals(123, result);
|
||||
}
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
|
@ -121,71 +143,47 @@ class MbtilesTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
void testAddMetadata() throws IOException {
|
||||
Map<String, String> expected = new TreeMap<>();
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
|
||||
var metadata = db.createTablesWithoutIndexes().metadata();
|
||||
metadata.setName("name value");
|
||||
expected.put("name", "name value");
|
||||
|
||||
metadata.setFormat("pbf");
|
||||
expected.put("format", "pbf");
|
||||
|
||||
metadata.setAttribution("attribution value");
|
||||
expected.put("attribution", "attribution value");
|
||||
|
||||
metadata.setBoundsAndCenter(GeoUtils.toLatLonBoundsBounds(new Envelope(0.25, 0.75, 0.25, 0.75)));
|
||||
expected.put("bounds", "-90,-66.51326,90,66.51326");
|
||||
expected.put("center", "0,0,1");
|
||||
|
||||
metadata.setDescription("description value");
|
||||
expected.put("description", "description value");
|
||||
|
||||
metadata.setMinzoom(1);
|
||||
expected.put("minzoom", "1");
|
||||
|
||||
metadata.setMaxzoom(13);
|
||||
expected.put("maxzoom", "13");
|
||||
|
||||
metadata.setVersion("1.2.3");
|
||||
expected.put("version", "1.2.3");
|
||||
|
||||
metadata.setTypeIsBaselayer();
|
||||
expected.put("type", "baselayer");
|
||||
|
||||
assertEquals(expected, metadata.getAll());
|
||||
}
|
||||
void testRoundTripMetadata() throws IOException {
|
||||
roundTripMetadata(new TileArchiveMetadata(
|
||||
"MyName",
|
||||
"MyDescription",
|
||||
"MyAttribution",
|
||||
"MyVersion",
|
||||
"baselayer",
|
||||
TileArchiveMetadata.MVT_FORMAT,
|
||||
new Envelope(1, 2, 3, 4),
|
||||
new CoordinateXY(5, 6),
|
||||
7d,
|
||||
8,
|
||||
9,
|
||||
List.of(new LayerStats.VectorLayer("MyLayer", Map.of())),
|
||||
Map.of("other key", "other value")
|
||||
));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAddMetadataWorldBounds() throws IOException {
|
||||
Map<String, String> expected = new TreeMap<>();
|
||||
void testRoundTripMinimalMetadata() throws IOException {
|
||||
var empty =
|
||||
new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, Map.of());
|
||||
roundTripMetadata(empty);
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
|
||||
var metadata = db.createTablesWithoutIndexes().metadata();
|
||||
metadata.setBoundsAndCenter(GeoUtils.WORLD_LAT_LON_BOUNDS);
|
||||
expected.put("bounds", "-180,-85.05113,180,85.05113");
|
||||
expected.put("center", "0,0,0");
|
||||
|
||||
assertEquals(expected, metadata.getAll());
|
||||
db.createTablesWithoutIndexes();
|
||||
assertEquals(empty, db.metadata());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAddMetadataSmallBounds() throws IOException {
|
||||
Map<String, String> expected = new TreeMap<>();
|
||||
private static void roundTripMetadata(TileArchiveMetadata metadata) throws IOException {
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
|
||||
var metadata = db.createTablesWithoutIndexes().metadata();
|
||||
metadata.setBoundsAndCenter(new Envelope(-73.6632, -69.7598, 41.1274, 43.0185));
|
||||
expected.put("bounds", "-73.6632,41.1274,-69.7598,43.0185");
|
||||
expected.put("center", "-71.7115,42.07295,7");
|
||||
|
||||
assertEquals(expected, metadata.getAll());
|
||||
db.createTablesWithoutIndexes();
|
||||
var metadataTable = db.metadataTable();
|
||||
metadataTable.set(metadata);
|
||||
assertEquals(metadata, metadataTable.get());
|
||||
}
|
||||
}
|
||||
|
||||
private void testMetadataJson(Mbtiles.MetadataJson object, String expected) throws IOException {
|
||||
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
|
||||
var metadata = db.createTablesWithoutIndexes().metadata();
|
||||
var metadata = db.createTablesWithoutIndexes().metadataTable();
|
||||
metadata.setJson(object);
|
||||
var actual = metadata.getAll().get("json");
|
||||
assertSameJson(expected, actual);
|
||||
|
|
|
@ -45,7 +45,7 @@ class VerifyTest {
|
|||
@Test
|
||||
void testValidWithNameAndOneTile() throws IOException {
|
||||
mbtiles.createTablesWithIndexes();
|
||||
mbtiles.metadata().setName("name");
|
||||
mbtiles.metadataTable().setMetadata("name", "name");
|
||||
try (var writer = mbtiles.newTileWriter()) {
|
||||
VectorTile tile = new VectorTile();
|
||||
tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
|
||||
|
@ -62,7 +62,7 @@ class VerifyTest {
|
|||
@Test
|
||||
void testInvalidGeometry() throws IOException {
|
||||
mbtiles.createTablesWithIndexes();
|
||||
mbtiles.metadata().setName("name");
|
||||
mbtiles.metadataTable().setMetadata("name", "name");
|
||||
try (var writer = mbtiles.newTileWriter()) {
|
||||
VectorTile tile = new VectorTile();
|
||||
tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
|
||||
|
|
|
@ -15,15 +15,16 @@ import com.onthegomap.planetiler.util.SeekableInMemoryByteChannel;
|
|||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
||||
class PmtilesTest {
|
||||
|
||||
|
@ -181,11 +182,12 @@ class PmtilesTest {
|
|||
var in = WriteablePmtiles.newWriteToMemory(bytes);
|
||||
|
||||
var config = PlanetilerConfig.defaults();
|
||||
in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
|
||||
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
|
||||
in.initialize(metadata);
|
||||
var writer = in.newTileWriter();
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.empty()));
|
||||
|
||||
in.finish(config);
|
||||
in.finish(metadata);
|
||||
var reader = new ReadablePmtiles(bytes);
|
||||
var header = reader.getHeader();
|
||||
assertEquals(1, header.numAddressedTiles());
|
||||
|
@ -200,28 +202,59 @@ class PmtilesTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
void testWritePmtilesToFileWithMetadata(@TempDir Path tempDir) throws IOException {
|
||||
void testRoundtripMetadata() throws IOException {
|
||||
roundTripMetadata(new TileArchiveMetadata(
|
||||
"MyName",
|
||||
"MyDescription",
|
||||
"MyAttribution",
|
||||
"MyVersion",
|
||||
"baselayer",
|
||||
TileArchiveMetadata.MVT_FORMAT,
|
||||
new Envelope(1, 2, 3, 4),
|
||||
new CoordinateXY(5, 6),
|
||||
7d,
|
||||
8,
|
||||
9,
|
||||
List.of(new LayerStats.VectorLayer("MyLayer", Map.of())),
|
||||
Map.of("other key", "other value")
|
||||
));
|
||||
}
|
||||
|
||||
try (var in = WriteablePmtiles.newWriteToFile(tempDir.resolve("tmp.pmtiles"))) {
|
||||
var config = PlanetilerConfig.defaults();
|
||||
in.initialize(config,
|
||||
new TileArchiveMetadata("MyName", "MyDescription", "MyAttribution", "MyVersion", "baselayer", new HashMap<>()),
|
||||
new LayerStats());
|
||||
@Test
|
||||
void testRoundtripMetadataMinimal() throws IOException {
|
||||
roundTripMetadata(
|
||||
new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, Map.of()),
|
||||
new TileArchiveMetadata(null, null, null, null, null, null,
|
||||
new Envelope(-180, 180, -85.0511287, 85.0511287),
|
||||
new CoordinateXY(0, 0),
|
||||
0d,
|
||||
0,
|
||||
15,
|
||||
null,
|
||||
Map.of()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private static void roundTripMetadata(TileArchiveMetadata metadata) throws IOException {
|
||||
roundTripMetadata(metadata, metadata);
|
||||
}
|
||||
|
||||
private static void roundTripMetadata(TileArchiveMetadata input, TileArchiveMetadata output) throws IOException {
|
||||
try (
|
||||
var channel = new SeekableInMemoryByteChannel(0);
|
||||
var in = WriteablePmtiles.newWriteToMemory(channel)
|
||||
) {
|
||||
in.initialize(input);
|
||||
var writer = in.newTileWriter();
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.empty()));
|
||||
|
||||
in.finish(config);
|
||||
in.finish(input);
|
||||
var reader = new ReadablePmtiles(channel);
|
||||
assertArrayEquals(new byte[]{0xa, 0x2}, reader.getTile(0, 0, 0));
|
||||
|
||||
assertEquals(output, reader.metadata());
|
||||
}
|
||||
|
||||
var reader = new ReadablePmtiles(FileChannel.open(tempDir.resolve("tmp.pmtiles")));
|
||||
assertArrayEquals(new byte[]{0xa, 0x2}, reader.getTile(0, 0, 0));
|
||||
|
||||
var metadata = reader.getJsonMetadata();
|
||||
assertEquals("MyName", metadata.otherMetadata().get("name"));
|
||||
assertEquals("MyDescription", metadata.otherMetadata().get("description"));
|
||||
assertEquals("MyAttribution", metadata.otherMetadata().get("attribution"));
|
||||
assertEquals("MyVersion", metadata.otherMetadata().get("version"));
|
||||
assertEquals("baselayer", metadata.otherMetadata().get("type"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -250,13 +283,14 @@ class PmtilesTest {
|
|||
var in = WriteablePmtiles.newWriteToMemory(bytes);
|
||||
|
||||
var config = PlanetilerConfig.defaults();
|
||||
in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
|
||||
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
|
||||
in.initialize(metadata);
|
||||
var writer = in.newTileWriter();
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 2), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
|
||||
|
||||
in.finish(config);
|
||||
in.finish(metadata);
|
||||
var reader = new ReadablePmtiles(bytes);
|
||||
var header = reader.getHeader();
|
||||
assertEquals(3, header.numAddressedTiles());
|
||||
|
@ -276,12 +310,13 @@ class PmtilesTest {
|
|||
var in = WriteablePmtiles.newWriteToMemory(bytes);
|
||||
|
||||
var config = PlanetilerConfig.defaults();
|
||||
in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
|
||||
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
|
||||
in.initialize(metadata);
|
||||
var writer = in.newTileWriter();
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
|
||||
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
|
||||
|
||||
in.finish(config);
|
||||
in.finish(metadata);
|
||||
var reader = new ReadablePmtiles(bytes);
|
||||
var header = reader.getHeader();
|
||||
assertEquals(2, header.numAddressedTiles());
|
||||
|
@ -301,7 +336,8 @@ class PmtilesTest {
|
|||
var in = WriteablePmtiles.newWriteToMemory(bytes);
|
||||
|
||||
var config = PlanetilerConfig.defaults();
|
||||
in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
|
||||
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
|
||||
in.initialize(metadata);
|
||||
var writer = in.newTileWriter();
|
||||
|
||||
int ENTRIES = 20000;
|
||||
|
@ -311,7 +347,7 @@ class PmtilesTest {
|
|||
OptionalLong.empty()));
|
||||
}
|
||||
|
||||
in.finish(config);
|
||||
in.finish(metadata);
|
||||
var reader = new ReadablePmtiles(bytes);
|
||||
var header = reader.getHeader();
|
||||
assertEquals(ENTRIES, header.numAddressedTiles());
|
||||
|
|
|
@ -151,8 +151,6 @@ cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.proper
|
|||
- `minzoom` - Minimum tile zoom level to emit
|
||||
- `maxzoom` - Maximum tile zoom level to emit
|
||||
- `render_maxzoom` - Maximum rendering zoom level up to
|
||||
- `skip_mbtiles_index_creation` - Skip adding index to mbtiles file
|
||||
- `optimize_db` - Vacuum analyze mbtiles file after writing
|
||||
- `force` - Overwriting output file and ignore warnings
|
||||
- `gzip_temp` - Gzip temporary feature storage (uses more CPU, but less disk space)
|
||||
- `mmap_temp` - Use memory-mapped IO for temp feature files
|
||||
|
@ -175,7 +173,6 @@ cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.proper
|
|||
maximum zoom level to allow for overzooming
|
||||
- `simplify_tolerance` - Default value for the tile pixel tolerance to use when simplifying features below the maximum
|
||||
zoom level
|
||||
- `compact_db` - Reduce the DB size by separating and deduping the tile data
|
||||
- `skip_filled_tiles` - Skip writing tiles containing only polygon fills to the output
|
||||
- `tile_warning_size_mb` - Maximum size in megabytes of a tile to emit a warning about
|
||||
|
||||
|
|
|
@ -138,28 +138,6 @@
|
|||
"render_maxzoom": {
|
||||
"description": "Maximum rendering zoom level up to"
|
||||
},
|
||||
"skip_mbtiles_index_creation": {
|
||||
"description": "Skip adding index to mbtiles file",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
}
|
||||
]
|
||||
},
|
||||
"optimize_db": {
|
||||
"description": "Vacuum analyze mbtiles file after writing",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
}
|
||||
]
|
||||
},
|
||||
"force": {
|
||||
"description": "Overwriting output file and ignore warnings",
|
||||
"anyOf": [
|
||||
|
@ -294,17 +272,6 @@
|
|||
"simplify_tolerance": {
|
||||
"description": "Default value for the tile pixel tolerance to use when simplifying features below the maximum zoom level"
|
||||
},
|
||||
"compact_db": {
|
||||
"description": "Reduce the DB size by separating and deduping the tile data",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
}
|
||||
]
|
||||
},
|
||||
"skip_filled_tiles": {
|
||||
"description": "Skip writing tiles containing only polygon fills to the output",
|
||||
"anyOf": [
|
||||
|
|
|
@ -10,7 +10,7 @@ import java.nio.file.Path;
|
|||
|
||||
/**
|
||||
* Main driver to create maps configured by a YAML file.
|
||||
*
|
||||
* <p>
|
||||
* Parses the config file into a {@link ConfiguredProfile}, loads sources into {@link Planetiler} runner and kicks off
|
||||
* the map generation process.
|
||||
*/
|
||||
|
@ -54,7 +54,7 @@ public class ConfiguredMapMain {
|
|||
configureSource(planetiler, sourcesDir, source);
|
||||
}
|
||||
|
||||
planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")).run();
|
||||
planetiler.overwriteOutput(Path.of("data", "output.mbtiles")).run();
|
||||
}
|
||||
|
||||
private static void configureSource(Planetiler planetiler, Path sourcesDir, Source source) {
|
||||
|
|
|
@ -187,8 +187,6 @@ public class Contexts {
|
|||
argumentValues.put("minzoom", config.minzoom());
|
||||
argumentValues.put("maxzoom", config.maxzoom());
|
||||
argumentValues.put("render_maxzoom", config.maxzoomForRendering());
|
||||
argumentValues.put("skip_mbtiles_index_creation", config.skipIndexCreation());
|
||||
argumentValues.put("optimize_db", config.optimizeDb());
|
||||
argumentValues.put("force", config.force());
|
||||
argumentValues.put("gzip_temp", config.gzipTempStorage());
|
||||
argumentValues.put("mmap_temp", config.mmapTempStorage());
|
||||
|
@ -209,7 +207,6 @@ public class Contexts {
|
|||
argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom());
|
||||
argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());
|
||||
argumentValues.put("simplify_tolerance", config.simplifyToleranceBelowMaxZoom());
|
||||
argumentValues.put("compact_db", config.compactDb());
|
||||
argumentValues.put("skip_filled_tiles", config.skipFilledTiles());
|
||||
argumentValues.put("tile_warning_size_mb", config.tileWarningSizeBytes());
|
||||
builtInArgs = Set.copyOf(argumentValues.keySet());
|
||||
|
|
|
@ -47,7 +47,7 @@ class ConfiguredMapTest {
|
|||
"--tmp=" + tmpDir,
|
||||
|
||||
// Override output location
|
||||
"--mbtiles=" + dbPath
|
||||
"--output=" + dbPath
|
||||
);
|
||||
mbtiles = Mbtiles.newReadOnlyDatabase(dbPath);
|
||||
}
|
||||
|
@ -59,7 +59,7 @@ class ConfiguredMapTest {
|
|||
|
||||
@Test
|
||||
void testMetadata() {
|
||||
Map<String, String> metadata = mbtiles.metadata().getAll();
|
||||
Map<String, String> metadata = mbtiles.metadataTable().getAll();
|
||||
assertEquals("OWG Simple Schema", metadata.get("name"));
|
||||
assertEquals("0", metadata.get("minzoom"));
|
||||
assertEquals("14", metadata.get("maxzoom"));
|
||||
|
|
|
@ -105,7 +105,7 @@ java -cp target/*-with-deps.jar com.onthegomap.planetiler.examples.MyProfile
|
|||
Then, to inspect the tiles:
|
||||
|
||||
```bash
|
||||
tileserver-gl-light --mbtiles data/toilets.mbtiles
|
||||
tileserver-gl-light data/toilets.mbtiles
|
||||
```
|
||||
|
||||
Finally, open http://localhost:8080 to see your tiles.
|
||||
|
@ -143,7 +143,7 @@ public void integrationTest(@TempDir Path tmpDir) throws Exception {
|
|||
MyProfile.main(
|
||||
"--osm_path=" + TestUtils.pathToResource("monaco-latest.osm.pbf"),
|
||||
"--tmp=" + tmpDir,
|
||||
"--mbtiles=" + mbtilesPath,
|
||||
"--output=" + mbtilesPath,
|
||||
));
|
||||
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(mbtilesPath)) {
|
||||
Map<String, String> metadata = mbtiles.metadata().getAll();
|
||||
|
|
|
@ -22,7 +22,7 @@ import java.util.List;
|
|||
* <li>then build the examples: {@code mvn clean package}</li>
|
||||
* <li>then run this example:
|
||||
* {@code java -cp target/*-with-deps.jar com.onthegomap.planetiler.examples.BikeRouteOverlay osm_path="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}</li>
|
||||
* <li>then run the demo tileserver: {@code tileserver-gl-light --mbtiles data/bikeroutes.mbtiles}</li>
|
||||
* <li>then run the demo tileserver: {@code tileserver-gl-light data/bikeroutes.mbtiles}</li>
|
||||
* <li>and view the output at <a href="http://localhost:8080">localhost:8080</a></li>
|
||||
* </ol>
|
||||
*/
|
||||
|
@ -175,7 +175,7 @@ public class BikeRouteOverlay implements Profile {
|
|||
// override this default with osm_path="path/to/data.osm.pbf"
|
||||
.addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area)
|
||||
// override this default with mbtiles="path/to/output.mbtiles"
|
||||
.overwriteOutput("mbtiles", Path.of("data", "bikeroutes.mbtiles"))
|
||||
.overwriteOutput(Path.of("data", "bikeroutes.mbtiles"))
|
||||
.run();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -54,7 +54,7 @@ public class OsmQaTiles implements Profile {
|
|||
Path.of("data", "sources", area + ".osm.pbf"),
|
||||
"planet".equalsIgnoreCase(area) ? "aws:latest" : ("geofabrik:" + area)
|
||||
)
|
||||
.overwriteOutput("mbtiles", Path.of("data", "qa.mbtiles"))
|
||||
.overwriteOutput(Path.of("data", "qa.mbtiles"))
|
||||
.run();
|
||||
}
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
* <li>then build the examples: {@code mvn clean package}</li>
|
||||
* <li>then run this example:
|
||||
* {@code java -cp target/*-fatjar.jar com.onthegomap.planetiler.examples.ToiletsOverlay osm_path="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}</li>
|
||||
* <li>then run the demo tileserver: {@code tileserver-gl-light --mbtiles=data/output.mbtiles}</li>
|
||||
* <li>then run the demo tileserver: {@code tileserver-gl-light data/output.mbtiles}</li>
|
||||
* <li>and view the output at <a href="http://localhost:8080">localhost:8080</a></li>
|
||||
* </ol>
|
||||
*/
|
||||
|
@ -103,7 +103,7 @@ public class ToiletsOverlay implements Profile {
|
|||
// override this default with osm_path="path/to/data.osm.pbf"
|
||||
.addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area)
|
||||
// override this default with mbtiles="path/to/output.mbtiles"
|
||||
.overwriteOutput("mbtiles", Path.of("data", "toilets.mbtiles"))
|
||||
.overwriteOutput(Path.of("data", "toilets.mbtiles"))
|
||||
.run();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,13 +4,14 @@ import com.onthegomap.planetiler.Planetiler;
|
|||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.archive.TileArchives;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.collection.LongLongMap;
|
||||
import com.onthegomap.planetiler.collection.LongLongMultimap;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
import com.onthegomap.planetiler.reader.osm.OsmInputFile;
|
||||
import com.onthegomap.planetiler.reader.osm.OsmReader;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
|
@ -31,7 +32,7 @@ import org.slf4j.LoggerFactory;
|
|||
* <li>then build the examples: {@code mvn clean package}</li>
|
||||
* <li>then run this example:
|
||||
* {@code java -cp target/*-fatjar.jar com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi}</li>
|
||||
* <li>then run the demo tileserver: {@code tileserver-gl-light --mbtiles=data/toilets.mbtiles}</li>
|
||||
* <li>then run the demo tileserver: {@code tileserver-gl-light data/toilets.mbtiles}</li>
|
||||
* <li>and view the output at <a href="http://localhost:8080">localhost:8080</a></li>
|
||||
* </ol>
|
||||
*/
|
||||
|
@ -57,7 +58,7 @@ public class ToiletsOverlayLowLevelApi {
|
|||
PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromJvmProperties());
|
||||
|
||||
// extract mbtiles metadata from profile
|
||||
TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile);
|
||||
TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile, config);
|
||||
|
||||
// overwrite output each time
|
||||
FileUtils.deleteFile(output);
|
||||
|
@ -112,7 +113,7 @@ public class ToiletsOverlayLowLevelApi {
|
|||
|
||||
// then process rendered features, grouped by tile, encoding them into binary vector tile format
|
||||
// and writing to the output mbtiles file.
|
||||
try (Mbtiles db = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
|
||||
try (WriteableTileArchive db = TileArchives.newWriter(output, config)) {
|
||||
TileArchiveWriter.writeOutput(featureGroup, db, () -> FileUtils.fileSize(output), tileArchiveMetadata, config,
|
||||
stats);
|
||||
} catch (IOException e) {
|
||||
|
|
|
@ -98,10 +98,10 @@ class BikeRouteOverlayTest {
|
|||
// Override temp dir location
|
||||
"tmp", tmpDir.toString(),
|
||||
// Override output location
|
||||
"mbtiles", dbPath.toString()
|
||||
"output", dbPath.toString()
|
||||
));
|
||||
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
|
||||
Map<String, String> metadata = mbtiles.metadata().getAll();
|
||||
Map<String, String> metadata = mbtiles.metadataTable().getAll();
|
||||
assertEquals("Bike Paths Overlay", metadata.get("name"));
|
||||
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
|
||||
|
||||
|
|
|
@ -93,10 +93,10 @@ class OsmQaTilesTest {
|
|||
// Override temp dir location
|
||||
"tmp", tmpDir.toString(),
|
||||
// Override output location
|
||||
"mbtiles", dbPath.toString()
|
||||
"output", dbPath.toString()
|
||||
));
|
||||
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
|
||||
Map<String, String> metadata = mbtiles.metadata().getAll();
|
||||
Map<String, String> metadata = mbtiles.metadataTable().getAll();
|
||||
assertEquals("osm qa", metadata.get("name"));
|
||||
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
|
||||
|
||||
|
|
|
@ -27,7 +27,7 @@ class ToiletsOverlayLowLevelApiTest {
|
|||
dbPath
|
||||
);
|
||||
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
|
||||
Map<String, String> metadata = mbtiles.metadata().getAll();
|
||||
Map<String, String> metadata = mbtiles.metadata().toMap();
|
||||
assertEquals("Toilets Overlay", metadata.get("name"));
|
||||
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
|
||||
|
||||
|
|
|
@ -58,10 +58,10 @@ class ToiletsProfileTest {
|
|||
// Override temp dir location
|
||||
"tmp", tmpDir.toString(),
|
||||
// Override output location
|
||||
"mbtiles", dbPath.toString()
|
||||
"output", dbPath.toString()
|
||||
));
|
||||
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
|
||||
Map<String, String> metadata = mbtiles.metadata().getAll();
|
||||
Map<String, String> metadata = mbtiles.metadata().toMap();
|
||||
assertEquals("Toilets Overlay", metadata.get("name"));
|
||||
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
|
||||
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit 62de454cf769e5bf2832f32d6b1f707860d442cf
|
||||
Subproject commit 292611de84b69f0ddbd4b603ec1d0f5d13257c33
|
|
@ -102,9 +102,10 @@ if [ "$DRY_RUN" == "true" ]; then
|
|||
fi
|
||||
|
||||
function run() {
|
||||
echo "$ $*"
|
||||
command="${*//&/\&}"
|
||||
echo "$ $command"
|
||||
if [ "$DRY_RUN" != "true" ]; then
|
||||
eval "$*"
|
||||
eval "$command"
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
|
@ -15,23 +15,23 @@ fi
|
|||
echo "Test java build"
|
||||
echo "::group::OpenMapTiles monaco (java)"
|
||||
rm -f data/out.mbtiles
|
||||
java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --mbtiles=data/out.mbtiles
|
||||
java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/out.mbtiles
|
||||
./scripts/check-monaco.sh data/out.mbtiles
|
||||
echo "::endgroup::"
|
||||
echo "::group::Example (java)"
|
||||
rm -f data/out.mbtiles
|
||||
java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --mbtiles=data/out.mbtiles
|
||||
java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/out.mbtiles
|
||||
./scripts/check-mbtiles.sh data/out.mbtiles
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::endgroup::"
|
||||
echo "::group::OpenMapTiles monaco (docker)"
|
||||
rm -f data/out.mbtiles
|
||||
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --mbtiles=data/out.mbtiles
|
||||
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/out.mbtiles
|
||||
./scripts/check-monaco.sh data/out.mbtiles
|
||||
echo "::endgroup::"
|
||||
echo "::group::Example (docker)"
|
||||
rm -f data/out.mbtiles
|
||||
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --mbtiles=data/out.mbtiles
|
||||
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/out.mbtiles
|
||||
./scripts/check-mbtiles.sh data/out.mbtiles
|
||||
echo "::endgroup::"
|
||||
|
|
Ładowanie…
Reference in New Issue