AWS
@@ -183,10 +184,11 @@ public class Planetiler {
),
ifSourceUsed(name, () -> {
var header = osmInputFile.getHeader();
- tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant());
- tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationseq",
+ tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationtime", header.instant());
+ tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationseq",
header.osmosisReplicationSequenceNumber());
- tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl());
+ tileArchiveMetadata.setExtraMetadata("planetiler:" + name + ":osmosisreplicationurl",
+ header.osmosisReplicationBaseUrl());
try (
var nodeLocations =
LongLongMap.from(config.nodeMapType(), config.nodeMapStorage(), nodeDbPath, config.nodeMapMadvise());
@@ -253,8 +255,8 @@ public class Planetiler {
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments. Can be a
* {@code .shp} file with other shapefile components in the same directory, or a {@code .zip} file
* containing the shapefile components.
- * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
- * name_url} argument is not set
+ * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
+ * {@code name_url} argument is not set
* @return this runner instance for chaining
* @see ShapefileReader
* @see Downloader
@@ -327,8 +329,8 @@ public class Planetiler {
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments. Can be a
* {@code .shp} file with other shapefile components in the same directory, or a {@code .zip} file
* containing the shapefile components.
- * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
- * name_url} argument is not set
+ * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
+ * {@code name_url} argument is not set
* @return this runner instance for chaining
* @see ShapefileReader
* @see Downloader
@@ -362,8 +364,8 @@ public class Planetiler {
* {@link org.geotools.referencing.CRS#decode(String)}
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
- * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
- * name_url} argument is not set
+ * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
+ * {@code name_url} argument is not set
* @return this runner instance for chaining
* @see GeoPackageReader
* @see Downloader
@@ -399,8 +401,8 @@ public class Planetiler {
*
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
- * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
- * name_url} argument is not set
+ * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
+ * {@code name_url} argument is not set
* @return this runner instance for chaining
* @see GeoPackageReader
* @see Downloader
@@ -415,12 +417,12 @@ public class Planetiler {
* To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to
* override the download URL set {@code name_url=http://url/of/natural_earth.zip}.
*
- * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the
* {@code .sqlite} file or a {@code .zip} file containing the sqlite file.
* @return this runner instance for chaining
* @see NaturalEarthReader
+ * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
*/
@Deprecated(forRemoval = true)
public Planetiler addNaturalEarthSource(String name, Path defaultPath) {
@@ -436,16 +438,15 @@ public class Planetiler {
* To override the location of the {@code sqlite} file, set {@code name_path=newpath.zip} in the arguments and to
* override the download URL set {@code name_url=http://url/of/natural_earth.zip}.
*
- * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
- *
* @param name string to use in stats and logs to identify this stage
* @param defaultPath path to the input file to use if {@code name} key is not set through arguments. Can be the
* {@code .sqlite} file or a {@code .zip} file containing the sqlite file.
- * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and {@code
- * name_url} argument is not set
+ * @param defaultUrl remote URL that the file to download if {@code download=true} argument is set and
+ * {@code name_url} argument is not set
* @return this runner instance for chaining
* @see NaturalEarthReader
* @see Downloader
+ * @deprecated can be replaced by {@link #addGeoPackageSource(String, Path, String)}.
*/
@Deprecated(forRemoval = true)
public Planetiler addNaturalEarthSource(String name, Path defaultPath, String defaultUrl) {
@@ -540,35 +541,69 @@ public class Planetiler {
}
/**
- * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists.
- *
- * To override the location of the file, set {@code argument=newpath} in the arguments.
+ * Sets the location of the output archive to write rendered tiles to.
*
- * @param argument the argument key to check for an override to {@code fallback}
- * @param fallback the fallback value if {@code argument} is not set in arguments
- * @return this runner instance for chaining
- * @see TileArchiveWriter
+ * @deprecated Use {@link #setOutput(String)} instead
*/
+ @Deprecated(forRemoval = true)
public Planetiler setOutput(String argument, Path fallback) {
- this.output = arguments.file(argument, "output tile archive", fallback);
+ this.output =
+ TileArchiveConfig
+ .from(arguments.getString("output|" + argument, "output tile archive path", fallback.toString()));
return this;
}
/**
- * Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists.
+ * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists.
*
- * To override the location of the file, set {@code argument=newpath} in the arguments.
+ * To override the location of the file, set {@code argument=newpath} in the arguments. To set options for the output
+ * drive add {@code output.mbtiles?arg=value} or add command-line argument {@code mbtiles_arg=value}.
*
- * @param argument the argument key to check for an override to {@code fallback}
- * @param fallback the fallback value if {@code argument} is not set in arguments
+ * @param defaultOutputUri The default output URI string to write to.
* @return this runner instance for chaining
- * @see TileArchiveWriter
+ * @see TileArchiveConfig For details on URI string formats and options.
*/
+ public Planetiler setOutput(String defaultOutputUri) {
+ this.output = TileArchiveConfig.from(arguments.getString("output", "output tile archive URI", defaultOutputUri));
+ return this;
+ }
+
+ /** Alias for {@link #setOutput(String)} which infers the output type based on extension. */
+ public Planetiler setOutput(Path path) {
+ return setOutput(path.toString());
+ }
+
+ /**
+ * Sets the location of the output archive to write rendered tiles to.
+ *
+ * @deprecated Use {@link #overwriteOutput(String)} instead
+ */
+ @Deprecated(forRemoval = true)
public Planetiler overwriteOutput(String argument, Path fallback) {
this.overwrite = true;
return setOutput(argument, fallback);
}
+ /**
+ * Sets the location of the output archive to write rendered tiles to. Overwrites if the archive already exists.
+ *
+ * To override the location of the file, set {@code argument=newpath} in the arguments. To set options for the output
+ * drive add {@code output.mbtiles?arg=value} or add command-line argument {@code mbtiles_arg=value}.
+ *
+ * @param defaultOutputUri The default output URI string to write to.
+ * @return this runner instance for chaining
+ * @see TileArchiveConfig For details on URI string formats and options.
+ */
+ public Planetiler overwriteOutput(String defaultOutputUri) {
+ this.overwrite = true;
+ return setOutput(defaultOutputUri);
+ }
+
+ /** Alias for {@link #overwriteOutput(String)} which infers the output type based on extension. */
+ public Planetiler overwriteOutput(Path defaultOutput) {
+ return overwriteOutput(defaultOutput.toString());
+ }
+
/**
* Reads all elements from all sourced that have been added, generates map features according to the profile, and
* writes the rendered tiles to the output archive.
@@ -600,19 +635,18 @@ public class Planetiler {
throw new IllegalArgumentException("Can only run once");
}
ran = true;
- tileArchiveMetadata = new TileArchiveMetadata(profile, config.arguments());
if (arguments.getBoolean("help", "show arguments then exit", false)) {
System.exit(0);
} else if (onlyDownloadSources) {
// don't check files if not generating map
} else if (overwrite || config.force()) {
- FileUtils.deleteFile(output);
- } else if (Files.exists(output)) {
- throw new IllegalArgumentException(output + " already exists, use the --force argument to overwrite.");
+ output.delete();
+ } else if (output.exists()) {
+ throw new IllegalArgumentException(output.uri() + " already exists, use the --force argument to overwrite.");
}
- LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output);
+ LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri());
if (!toDownload.isEmpty()) {
LOGGER.info(" download: Download sources {}", toDownload.stream().map(d -> d.id).toList());
@@ -635,7 +669,7 @@ public class Planetiler {
// in case any temp files are left from a previous run...
FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath);
Files.createDirectories(tmpDir);
- FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output);
+ FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalPath());
if (!toDownload.isEmpty()) {
download();
@@ -661,14 +695,16 @@ public class Planetiler {
}
bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats));
}
+ // must construct this after bounds providers are added in order to infer bounds from the input source if not provided
+ tileArchiveMetadata = new TileArchiveMetadata(profile, config);
- try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+ try (WriteableTileArchive archive = TileArchives.newWriter(output, config)) {
featureGroup =
FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats);
stats.monitorFile("nodes", nodeDbPath);
stats.monitorFile("features", featureDbPath);
stats.monitorFile("multipolygons", multipolygonPath);
- stats.monitorFile("archive", output);
+ stats.monitorFile("archive", output.getLocalPath());
for (Stage stage : stages) {
stage.task.run();
@@ -685,9 +721,8 @@ public class Planetiler {
featureGroup.prepare();
- TileArchiveWriter.writeOutput(featureGroup, archive, () -> FileUtils.fileSize(output), tileArchiveMetadata,
- config,
- stats);
+ TileArchiveWriter.writeOutput(featureGroup, archive, output::size, tileArchiveMetadata,
+ config, stats);
} catch (IOException e) {
throw new IllegalStateException("Unable to write to " + output, e);
}
@@ -716,7 +751,7 @@ public class Planetiler {
readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
// output only needed during write phase
- writePhase.addDisk(output, outputSize, "archive output");
+ writePhase.addDisk(output.getLocalPath(), outputSize, "archive output");
// if the user opts to remove an input source after reading to free up additional space for the output...
for (var input : inputPaths) {
if (input.freeAfterReading()) {
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java
index 6702de02..51b0b0e0 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java
@@ -40,5 +40,10 @@ public interface ReadableTileArchive extends Closeable {
*/
CloseableIterator getAllTileCoords();
+ /**
+ * Returns the metadata stored in this archive.
+ */
+ TileArchiveMetadata metadata();
+
// TODO access archive metadata
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java
new file mode 100644
index 00000000..021ccc50
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java
@@ -0,0 +1,196 @@
+package com.onthegomap.planetiler.archive;
+
+import static com.onthegomap.planetiler.util.LanguageUtils.nullIfEmpty;
+
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.util.FileUtils;
+import java.net.URI;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Definition for a tileset, parsed from a URI-like string.
+ *
+ * {@link #from(String)} can accept:
+ *
+ * - A platform-specific absolute or relative path like {@code "./archive.mbtiles"} or
+ * {@code "C:\root\archive.mbtiles"}
+ * - A URI pointing at a file, like {@code "file:///root/archive.pmtiles"} or
+ * {@code "file:///C:/root/archive.pmtiles"}
+ *
+ *
+ * Both of these can also have archive-specific options added to the end, for example
+ * {@code "output.mbtiles?compact=false&page_size=16384"}.
+ *
+ * @param format The {@link Format format} of the archive, either inferred from the filename extension or the
+ * {@code ?format=} query parameter
+ * @param scheme Scheme for accessing the archive
+ * @param uri Full URI including scheme, location, and options
+ * @param options Parsed query parameters from the definition string
+ */
+public record TileArchiveConfig(
+ Format format,
+ Scheme scheme,
+ URI uri,
+ Map options
+) {
+
+ private static TileArchiveConfig.Scheme getScheme(URI uri) {
+ String scheme = uri.getScheme();
+ if (scheme == null) {
+ return Scheme.FILE;
+ }
+ for (var value : TileArchiveConfig.Scheme.values()) {
+ if (value.id().equals(scheme)) {
+ return value;
+ }
+ }
+ throw new IllegalArgumentException("Unsupported scheme " + scheme + " from " + uri);
+ }
+
+ private static String getExtension(URI uri) {
+ String path = uri.getPath();
+ if (path != null && (path.contains("."))) {
+ return nullIfEmpty(path.substring(path.lastIndexOf(".") + 1));
+ }
+ return null;
+ }
+
+ private static Map parseQuery(URI uri) {
+ String query = uri.getRawQuery();
+ Map result = new HashMap<>();
+ if (query != null) {
+ for (var part : query.split("&")) {
+ var split = part.split("=", 2);
+ result.put(
+ URLDecoder.decode(split[0], StandardCharsets.UTF_8),
+ split.length == 1 ? "true" : URLDecoder.decode(split[1], StandardCharsets.UTF_8)
+ );
+ }
+ }
+ return result;
+ }
+
+ private static TileArchiveConfig.Format getFormat(URI uri) {
+ String format = parseQuery(uri).get("format");
+ if (format == null) {
+ format = getExtension(uri);
+ }
+ if (format == null) {
+ return TileArchiveConfig.Format.MBTILES;
+ }
+ for (var value : TileArchiveConfig.Format.values()) {
+ if (value.id().equals(format)) {
+ return value;
+ }
+ }
+ throw new IllegalArgumentException("Unsupported format " + format + " from " + uri);
+ }
+
+ /**
+ * Parses a string definition of a tileset from a URI-like string.
+ */
+ public static TileArchiveConfig from(String string) {
+ // unix paths parse fine as URIs, but need to explicitly parse windows paths with backslashes
+ if (string.contains("\\")) {
+ String[] parts = string.split("\\?", 2);
+ string = Path.of(parts[0]).toUri().toString();
+ if (parts.length > 1) {
+ string += "?" + parts[1];
+ }
+ }
+ return from(URI.create(string));
+ }
+
+ /**
+ * Parses a string definition of a tileset from a URI.
+ */
+ public static TileArchiveConfig from(URI uri) {
+ if (uri.getScheme() == null) {
+ String base = Path.of(uri.getPath()).toAbsolutePath().toUri().normalize().toString();
+ if (uri.getRawQuery() != null) {
+ base += "?" + uri.getRawQuery();
+ }
+ uri = URI.create(base);
+ }
+ return new TileArchiveConfig(
+ getFormat(uri),
+ getScheme(uri),
+ uri,
+ parseQuery(uri)
+ );
+ }
+
+ /**
+ * Returns the local path on disk that this archive reads/writes to, or {@code null} if it is not on disk (ie. an HTTP
+ * repository).
+ */
+ public Path getLocalPath() {
+ return scheme == Scheme.FILE ? Path.of(URI.create(uri.toString().replaceAll("\\?.*$", ""))) : null;
+ }
+
+
+ /**
+ * Deletes the archive if possible.
+ */
+ public void delete() {
+ if (scheme == Scheme.FILE) {
+ FileUtils.delete(getLocalPath());
+ }
+ }
+
+ /**
+ * Returns {@code true} if the archive already exists, {@code false} otherwise.
+ */
+ public boolean exists() {
+ return getLocalPath() != null && Files.exists(getLocalPath());
+ }
+
+ /**
+ * Returns the current size of this archive.
+ */
+ public long size() {
+ return getLocalPath() == null ? 0 : FileUtils.size(getLocalPath());
+ }
+
+ /**
+ * Returns an {@link Arguments} instance that returns the value for options directly from the query parameters in the
+ * URI, or from {@code arguments} prefixed by {@code "format_"}.
+ */
+ public Arguments applyFallbacks(Arguments arguments) {
+ return Arguments.of(options).orElse(arguments.withPrefix(format.id));
+ }
+
+ public enum Format {
+ MBTILES("mbtiles"),
+ PMTILES("pmtiles");
+
+ private final String id;
+
+ Format(String id) {
+ this.id = id;
+ }
+
+ public String id() {
+ return id;
+ }
+ }
+
+ public enum Scheme {
+ FILE("file");
+
+ private final String id;
+
+ Scheme(String id) {
+ this.id = id;
+ }
+
+ public String id() {
+ return id;
+ }
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java
index 842e2ae1..5184d7cd 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java
@@ -1,42 +1,90 @@
package com.onthegomap.planetiler.archive;
-import com.onthegomap.planetiler.Profile;
-import com.onthegomap.planetiler.config.Arguments;
-import com.onthegomap.planetiler.util.BuildInfo;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
+import static com.onthegomap.planetiler.util.Format.joinCoordinates;
-/** Controls information that {@link TileArchiveWriter} will write to the archive metadata. */
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.geo.GeoUtils;
+import com.onthegomap.planetiler.util.BuildInfo;
+import com.onthegomap.planetiler.util.LayerStats;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.Envelope;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Metadata associated with a tile archive. */
public record TileArchiveMetadata(
- String name,
- String description,
- String attribution,
- String version,
- String type,
- Map planetilerSpecific
+ @JsonProperty(NAME_KEY) String name,
+ @JsonProperty(DESCRIPTION_KEY) String description,
+ @JsonProperty(ATTRIBUTION_KEY) String attribution,
+ @JsonProperty(VERSION_KEY) String version,
+ @JsonProperty(TYPE_KEY) String type,
+ @JsonProperty(FORMAT_KEY) String format,
+ @JsonIgnore Envelope bounds,
+ @JsonIgnore CoordinateXY center,
+ @JsonProperty(ZOOM_KEY) Double zoom,
+ @JsonProperty(MINZOOM_KEY) Integer minzoom,
+ @JsonProperty(MAXZOOM_KEY) Integer maxzoom,
+ @JsonIgnore List vectorLayers,
+ @JsonAnyGetter Map others
) {
- public TileArchiveMetadata(Profile profile) {
+ public static final String NAME_KEY = "name";
+ public static final String DESCRIPTION_KEY = "description";
+ public static final String ATTRIBUTION_KEY = "attribution";
+ public static final String VERSION_KEY = "version";
+ public static final String TYPE_KEY = "type";
+ public static final String FORMAT_KEY = "format";
+ public static final String BOUNDS_KEY = "bounds";
+ public static final String CENTER_KEY = "center";
+ public static final String ZOOM_KEY = "zoom";
+ public static final String MINZOOM_KEY = "minzoom";
+ public static final String MAXZOOM_KEY = "maxzoom";
+ public static final String VECTOR_LAYERS_KEY = "vector_layers";
+
+ public static final String MVT_FORMAT = "pbf";
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TileArchiveMetadata.class);
+ private static final ObjectMapper mapper = new ObjectMapper()
+ .registerModules(new Jdk8Module())
+ .setSerializationInclusion(NON_ABSENT);
+
+ public TileArchiveMetadata(Profile profile, PlanetilerConfig config) {
+ this(profile, config, null);
+ }
+
+ public TileArchiveMetadata(Profile profile, PlanetilerConfig config, List vectorLayers) {
this(
- profile.name(),
- profile.description(),
- profile.attribution(),
- profile.version(),
- profile.isOverlay() ? "overlay" : "baselayer",
+ getString(config, NAME_KEY, profile.name()),
+ getString(config, DESCRIPTION_KEY, profile.description()),
+ getString(config, ATTRIBUTION_KEY, profile.attribution()),
+ getString(config, VERSION_KEY, profile.version()),
+ getString(config, TYPE_KEY, profile.isOverlay() ? "overlay" : "baselayer"),
+ getString(config, FORMAT_KEY, MVT_FORMAT),
+ config.bounds().latLon(),
+ new CoordinateXY(config.bounds().latLon().centre()),
+ GeoUtils.getZoomFromLonLatBounds(config.bounds().latLon()),
+ config.minzoom(),
+ config.maxzoom(),
+ vectorLayers,
mapWithBuildInfo()
);
}
- public TileArchiveMetadata(Profile profile, Arguments args) {
- this(
- args.getString("mbtiles_name", "'name' attribute for tileset metadata", profile.name()),
- args.getString("mbtiles_description", "'description' attribute for tileset metadata", profile.description()),
- args.getString("mbtiles_attribution", "'attribution' attribute for tileset metadata", profile.attribution()),
- args.getString("mbtiles_version", "'version' attribute for tileset metadata", profile.version()),
- args.getString("mbtiles_type", "'type' attribute for tileset metadata",
- profile.isOverlay() ? "overlay" : "baselayer"),
- mapWithBuildInfo()
- );
+ private static String getString(PlanetilerConfig config, String key, String fallback) {
+ return config.arguments()
+ .getString("archive_" + key + "|mbtiles_" + key, "'" + key + "' attribute for tileset metadata", fallback);
}
private static Map mapWithBuildInfo() {
@@ -56,20 +104,39 @@ public record TileArchiveMetadata(
return result;
}
- public TileArchiveMetadata set(String key, Object value) {
+ /** Sets an extra metadata entry in {@link #others}. */
+ public TileArchiveMetadata setExtraMetadata(String key, Object value) {
if (key != null && value != null) {
- planetilerSpecific.put(key, value.toString());
+ others.put(key, value.toString());
}
return this;
}
- public Map getAll() {
- var allKvs = new LinkedHashMap(planetilerSpecific);
- allKvs.put("name", this.name);
- allKvs.put("description", this.description);
- allKvs.put("attribution", this.attribution);
- allKvs.put("version", this.version);
- allKvs.put("type", this.type);
- return allKvs;
+ /**
+ * Returns a map with all key-value pairs from this metadata entry, including {@link #others} hoisted to top-level
+ * keys.
+ */
+ public Map toMap() {
+ Map result = new LinkedHashMap<>(mapper.convertValue(this, new TypeReference<>() {}));
+ if (bounds != null) {
+ result.put(BOUNDS_KEY, joinCoordinates(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY()));
+ }
+ if (center != null) {
+ result.put(CENTER_KEY, joinCoordinates(center.getX(), center.getY()));
+ }
+ if (vectorLayers != null) {
+ try {
+ result.put(VECTOR_LAYERS_KEY, mapper.writeValueAsString(vectorLayers));
+ } catch (JsonProcessingException e) {
+ LOGGER.warn("Error encoding vector_layers as json", e);
+ }
+ }
+ return result;
+ }
+
+ /** Returns a copy of this instance with {@link #vectorLayers} set to {@code layerStats}. */
+ public TileArchiveMetadata withLayerStats(List layerStats) {
+ return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, zoom, minzoom,
+ maxzoom, layerStats, others);
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java
index ad95e1c1..27a34144 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java
@@ -15,7 +15,6 @@ import com.onthegomap.planetiler.stats.Timer;
import com.onthegomap.planetiler.util.DiskBacked;
import com.onthegomap.planetiler.util.Format;
import com.onthegomap.planetiler.util.Hashing;
-import com.onthegomap.planetiler.util.LayerStats;
import com.onthegomap.planetiler.worker.WorkQueue;
import com.onthegomap.planetiler.worker.Worker;
import com.onthegomap.planetiler.worker.WorkerPipeline;
@@ -52,7 +51,6 @@ public class TileArchiveWriter {
private final WriteableTileArchive archive;
private final PlanetilerConfig config;
private final Stats stats;
- private final LayerStats layerStats;
private final Counter.Readable[] tilesByZoom;
private final Counter.Readable[] totalTileSizesByZoom;
private final LongAccumulator[] maxTileSizesByZoom;
@@ -61,14 +59,12 @@ public class TileArchiveWriter {
private final TileArchiveMetadata tileArchiveMetadata;
private TileArchiveWriter(Iterable inputTiles, WriteableTileArchive archive,
- PlanetilerConfig config,
- TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) {
+ PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) {
this.inputTiles = inputTiles;
this.archive = archive;
this.config = config;
this.tileArchiveMetadata = tileArchiveMetadata;
this.stats = stats;
- this.layerStats = layerStats;
tilesByZoom = IntStream.rangeClosed(0, config.maxzoom())
.mapToObj(i -> Counter.newSingleThreadCounter())
.toArray(Counter.Readable[]::new);
@@ -111,8 +107,9 @@ public class TileArchiveWriter {
readWorker = reader.readWorker();
}
- TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats,
- features.layerStats());
+ TileArchiveWriter writer =
+ new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats()
+ .getTileStats()), stats);
var pipeline = WorkerPipeline.start("archive", stats);
@@ -231,7 +228,6 @@ public class TileArchiveWriter {
byte[] lastBytes = null, lastEncoded = null;
Long lastTileDataHash = null;
boolean lastIsFill = false;
- boolean compactDb = config.compactDb();
boolean skipFilled = config.skipFilledTiles();
for (TileBatch batch : prev) {
@@ -265,7 +261,7 @@ public class TileArchiveWriter {
lastEncoded = encoded;
lastBytes = bytes;
last = tileFeatures;
- if (compactDb && en.likelyToBeDuplicated() && bytes != null) {
+ if (archive.deduplicates() && en.likelyToBeDuplicated() && bytes != null) {
tileDataHash = generateContentHash(bytes);
} else {
tileDataHash = null;
@@ -292,7 +288,8 @@ public class TileArchiveWriter {
private void tileWriter(Iterable tileBatches) throws ExecutionException, InterruptedException {
- archive.initialize(config, tileArchiveMetadata, layerStats);
+ archive.initialize(tileArchiveMetadata);
+ var order = archive.tileOrder();
TileCoord lastTile = null;
Timer time = null;
@@ -303,8 +300,9 @@ public class TileArchiveWriter {
TileEncodingResult encodedTile;
while ((encodedTile = encodedTiles.poll()) != null) {
TileCoord tileCoord = encodedTile.coord();
- assert lastTile == null || lastTile.compareTo(tileCoord) < 0 : "Tiles out of order %s before %s"
- .formatted(lastTile, tileCoord);
+ assert lastTile == null ||
+ order.encode(tileCoord) > order.encode(lastTile) : "Tiles out of order %s before %s"
+ .formatted(lastTile, tileCoord);
lastTile = encodedTile.coord();
int z = tileCoord.z();
if (z != currentZ) {
@@ -331,7 +329,7 @@ public class TileArchiveWriter {
}
- archive.finish(config);
+ archive.finish(tileArchiveMetadata);
}
private void printTileStats() {
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java
new file mode 100644
index 00000000..fdd4617b
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java
@@ -0,0 +1,75 @@
+package com.onthegomap.planetiler.archive;
+
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.mbtiles.Mbtiles;
+import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
+import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
+import java.io.IOException;
+import java.nio.file.Path;
+
+/** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */
+public class TileArchives {
+ private TileArchives() {}
+
+ /**
+ * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive} that will be parsed with
+ * {@link TileArchiveConfig}.
+ *
+ * @throws IOException if an error occurs creating the resource.
+ */
+ public static WriteableTileArchive newWriter(String archive, PlanetilerConfig config) throws IOException {
+ return newWriter(TileArchiveConfig.from(archive), config);
+ }
+
+ /**
+ * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive} that will be parsed with
+ * {@link TileArchiveConfig}.
+ *
+ * @throws IOException if an error occurs opening the resource.
+ */
+ public static ReadableTileArchive newReader(String archive, PlanetilerConfig config) throws IOException {
+ return newReader(TileArchiveConfig.from(archive), config);
+ }
+
+ /**
+ * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}.
+ *
+ * @throws IOException if an error occurs creating the resource.
+ */
+ public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
+ throws IOException {
+ var options = archive.applyFallbacks(config.arguments());
+ return switch (archive.format()) {
+ case MBTILES ->
+ // pass-through legacy arguments for fallback
+ Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments()
+ .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION)));
+ case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath());
+ };
+ }
+
+ /**
+ * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}.
+ *
+ * @throws IOException if an error occurs opening the resource.
+ */
+ public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
+ throws IOException {
+ var options = archive.applyFallbacks(config.arguments());
+ return switch (archive.format()) {
+ case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options);
+ case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath());
+ };
+ }
+
+ /** Alias for {@link #newReader(String, PlanetilerConfig)}. */
+ public static ReadableTileArchive newReader(Path path, PlanetilerConfig config) throws IOException {
+ return newReader(path.toString(), config);
+ }
+
+ /** Alias for {@link #newWriter(String, PlanetilerConfig)}. */
+ public static WriteableTileArchive newWriter(Path path, PlanetilerConfig config) throws IOException {
+ return newWriter(path.toString(), config);
+ }
+
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java
index 7449c0b1..3c3b6df7 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java
@@ -2,7 +2,6 @@ package com.onthegomap.planetiler.archive;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.geo.TileOrder;
-import com.onthegomap.planetiler.util.LayerStats;
import java.io.Closeable;
import javax.annotation.concurrent.NotThreadSafe;
@@ -15,6 +14,36 @@ import javax.annotation.concurrent.NotThreadSafe;
@NotThreadSafe
public interface WriteableTileArchive extends Closeable {
+ /**
+ * Returns true if this tile archive deduplicates tiles with the same content.
+ *
+ * If false, then {@link TileWriter} will skip computing tile hashes.
+ */
+ boolean deduplicates();
+
+ /**
+ * Specify the preferred insertion order for this archive, e.g. {@link TileOrder#TMS} or {@link TileOrder#HILBERT}.
+ */
+ TileOrder tileOrder();
+
+ /**
+ * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
+ * required state here.
+ */
+ default void initialize(TileArchiveMetadata metadata) {}
+
+ /**
+ * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might
+ * depend on {@link PlanetilerConfig}.
+ */
+ TileWriter newTileWriter();
+
+ /**
+ * Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on
+ * disk.
+ */
+ default void finish(TileArchiveMetadata tileArchiveMetadata) {}
+
interface TileWriter extends Closeable {
void write(TileEncodingResult encodingResult);
@@ -30,29 +59,5 @@ public interface WriteableTileArchive extends Closeable {
default void printStats() {}
}
-
- /**
- * Specify the preferred insertion order for this archive, e.g. {@link TileOrder#TMS} or {@link TileOrder#HILBERT}.
- */
- TileOrder tileOrder();
-
- /**
- * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
- * required state here.
- */
- void initialize(PlanetilerConfig config, TileArchiveMetadata metadata, LayerStats layerStats);
-
- /**
- * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might
- * depend on {@link PlanetilerConfig}.
- */
- TileWriter newTileWriter();
-
- /**
- * Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on
- * disk.
- */
- void finish(PlanetilerConfig config);
-
// TODO update archive metadata
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java
index 02ad44ea..573ad2f5 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java
@@ -12,6 +12,8 @@ import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -20,6 +22,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;
+import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.locationtech.jts.geom.Envelope;
import org.slf4j.Logger;
@@ -28,6 +31,12 @@ import org.slf4j.LoggerFactory;
/**
* Lightweight abstraction over ways to provide key/value pair arguments to a program like jvm properties, environmental
* variables, or a config file.
+ *
+ * When looking up a key, tries to find a case-and-separator-insensitive match, for example {@code "CONFIG_OPTION"} will
+ * match {@code "config-option"} and {@code "config_option"}.
+ *
+ * If you replace an option with a new value, you can read a value from the new option and fall back to old one by using
+ * {@code "new_flag|old_flag"} as the key.
*/
public class Arguments {
@@ -42,15 +51,6 @@ public class Arguments {
this.keys = keys;
}
- private static Arguments from(UnaryOperator provider, Supplier extends Collection> rawKeys,
- UnaryOperator forward, UnaryOperator reverse) {
- Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> {
- String reversed = reverse.apply(key);
- return key.equalsIgnoreCase(reversed) ? Stream.empty() : Stream.of(reversed);
- }).toList();
- return new Arguments(key -> provider.apply(forward.apply(key)), keys);
- }
-
/**
* Returns arguments from JVM system properties prefixed with {@code planetiler.}
*
@@ -64,10 +64,7 @@ public class Arguments {
}
static Arguments fromJvmProperties(UnaryOperator getter, Supplier extends Collection> keys) {
- return from(getter, keys,
- key -> "planetiler." + key.toLowerCase(Locale.ROOT),
- key -> key.replaceFirst("^planetiler\\.", "").toLowerCase(Locale.ROOT)
- );
+ return fromPrefixed(getter, keys, "planetiler", ".", false);
}
/**
@@ -83,10 +80,7 @@ public class Arguments {
}
static Arguments fromEnvironment(UnaryOperator getter, Supplier> keys) {
- return from(getter, keys,
- key -> "PLANETILER_" + key.toUpperCase(Locale.ROOT),
- key -> key.replaceFirst("^PLANETILER_", "").toLowerCase(Locale.ROOT)
- );
+ return fromPrefixed(getter, keys, "PLANETILER", "_", true);
}
/**
@@ -191,8 +185,21 @@ public class Arguments {
.orElse(fromEnvironment());
}
+ private static String normalize(String key, String separator, boolean upperCase) {
+ String result = key.replaceAll("[._-]", separator);
+ return upperCase ? result.toUpperCase(Locale.ROOT) : result.toLowerCase(Locale.ROOT);
+ }
+
+ private static String normalize(String key) {
+ return normalize(key, "_", false);
+ }
+
public static Arguments of(Map map) {
- return new Arguments(map::get, map::keySet);
+ Map updated = new LinkedHashMap<>();
+ for (var entry : map.entrySet()) {
+ updated.put(normalize(entry.getKey()), entry.getValue());
+ }
+ return new Arguments(updated::get, updated::keySet);
}
/** Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs. */
@@ -204,12 +211,36 @@ public class Arguments {
return of(map);
}
+ private static Arguments from(UnaryOperator provider, Supplier extends Collection> rawKeys,
+ UnaryOperator forward, UnaryOperator reverse) {
+ Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> {
+ String reversed = reverse.apply(key);
+ return normalize(key).equals(normalize(reversed)) ? Stream.empty() : Stream.of(reversed);
+ }).toList();
+ return new Arguments(key -> provider.apply(forward.apply(key)), keys);
+ }
+
+ private static Arguments fromPrefixed(UnaryOperator provider, Supplier extends Collection> keys,
+ String prefix, String separator, boolean uppperCase) {
+ var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)),
+ Pattern.CASE_INSENSITIVE);
+ return from(provider, keys,
+ key -> normalize(prefix + separator + key, separator, uppperCase),
+ key -> normalize(prefixRegex.matcher(key).replaceFirst(""))
+ );
+ }
+
private String get(String key) {
- String value = provider.apply(key);
- if (value == null) {
- value = provider.apply(key.replace('-', '_'));
- if (value == null) {
- value = provider.apply(key.replace('_', '-'));
+ String[] options = key.split("\\|");
+ String value = null;
+ for (int i = 0; i < options.length; i++) {
+ String option = options[i].strip();
+ value = provider.apply(normalize(option));
+ if (value != null) {
+ if (i != 0) {
+ LOGGER.warn("Argument '{}' is deprecated", option);
+ }
+ break;
}
}
return value;
@@ -274,8 +305,8 @@ public class Arguments {
}
protected void logArgValue(String key, String description, Object result) {
- if (!silent) {
- LOGGER.debug("argument: {}={} ({})", key, result, description);
+ if (!silent && LOGGER.isDebugEnabled()) {
+ LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description);
}
}
@@ -460,7 +491,7 @@ public class Arguments {
public Map toMap() {
Map result = new HashMap<>();
for (var key : keys.get()) {
- result.put(key, get(key));
+ result.put(normalize(key), get(key));
}
return result;
}
@@ -484,4 +515,27 @@ public class Arguments {
public boolean silenced() {
return silent;
}
+
+ public Arguments copy() {
+ return new Arguments(provider, keys);
+ }
+
+ /**
+ * Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}.
+ */
+ public Arguments withPrefix(String prefix) {
+ return fromPrefixed(provider, keys, prefix, "_", false);
+ }
+
+ /** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */
+ public Arguments subset(String... allowedKeys) {
+ Set allowed = new HashSet<>();
+ for (String key : allowedKeys) {
+ allowed.add(normalize(key));
+ }
+ return new Arguments(
+ key -> allowed.contains(normalize(key)) ? provider.apply(key) : null,
+ () -> keys.get().stream().filter(key -> allowed.contains(normalize(key))).toList()
+ );
+ }
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
index ec4f0562..ba50f9db 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
@@ -24,8 +24,6 @@ public record PlanetilerConfig(
int minzoom,
int maxzoom,
int maxzoomForRendering,
- boolean skipIndexCreation,
- boolean optimizeDb,
boolean force,
boolean gzipTempStorage,
boolean mmapTempStorage,
@@ -47,7 +45,6 @@ public record PlanetilerConfig(
double simplifyToleranceAtMaxZoom,
double simplifyToleranceBelowMaxZoom,
boolean osmLazyReads,
- boolean compactDb,
boolean skipFilledTiles,
int tileWarningSizeBytes,
Boolean color
@@ -125,8 +122,6 @@ public record PlanetilerConfig(
minzoom,
maxzoom,
renderMaxzoom,
- arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false),
- arguments.getBoolean("optimize_db", "Vacuum analyze mbtiles after writing", false),
arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),
arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true),
@@ -168,9 +163,6 @@ public record PlanetilerConfig(
arguments.getBoolean("osm_lazy_reads",
"Read OSM blocks from disk in worker threads",
true),
- arguments.getBoolean("compact_db",
- "Reduce the DB size by separating and deduping the tile data",
- true),
arguments.getBoolean("skip_filled_tiles",
"Skip writing tiles containing only polygon fills to the output",
false),
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
index 66a5f16d..723f6900 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
@@ -1,6 +1,7 @@
package com.onthegomap.planetiler.mbtiles;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
+import static com.onthegomap.planetiler.util.Format.joinCoordinates;
import com.carrotsearch.hppc.LongIntHashMap;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -11,14 +12,14 @@ import com.onthegomap.planetiler.archive.ReadableTileArchive;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.archive.WriteableTileArchive;
-import com.onthegomap.planetiler.config.PlanetilerConfig;
-import com.onthegomap.planetiler.geo.GeoUtils;
+import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.geo.TileOrder;
import com.onthegomap.planetiler.reader.FileFormatException;
import com.onthegomap.planetiler.util.CloseableIterator;
import com.onthegomap.planetiler.util.Format;
import com.onthegomap.planetiler.util.LayerStats;
+import com.onthegomap.planetiler.util.Parse;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Connection;
@@ -27,21 +28,20 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.OptionalLong;
import java.util.TreeMap;
import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
import java.util.stream.IntStream;
-import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateXY;
import org.locationtech.jts.geom.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,6 +54,17 @@ import org.sqlite.SQLiteConfig;
*/
public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive {
+ // Options that can be set through "file.mbtiles?compact=true" query parameters
+ // or "file.mbtiles" with "--mbtiles-compact=true" command-line flag
+ public static final String COMPACT_DB = "compact";
+ public static final String SKIP_INDEX_CREATION = "no_index";
+ public static final String VACUUM_ANALYZE = "vacuum_analyze";
+
+ public static final String LEGACY_COMPACT_DB = "compact_db";
+ public static final String LEGACY_SKIP_INDEX_CREATION = "skip_mbtiles_index_creation";
+ public static final String LEGACY_VACUUM_ANALYZE = "optimize_db";
+
+
// https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
private static final int MBTILES_APPLICATION_ID = 0x4d504258;
@@ -93,72 +104,115 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
private final Connection connection;
private final boolean compactDb;
+ private final boolean skipIndexCreation;
+ private final boolean vacuumAnalyze;
private PreparedStatement getTileStatement = null;
- private Mbtiles(Connection connection, boolean compactDb) {
+ private Mbtiles(Connection connection, Arguments arguments) {
this.connection = connection;
- this.compactDb = compactDb;
+ this.compactDb = arguments.getBoolean(
+ COMPACT_DB + "|" + LEGACY_COMPACT_DB,
+ "mbtiles: reduce the DB size by separating and deduping the tile data",
+ true
+ );
+ this.skipIndexCreation = arguments.getBoolean(
+ SKIP_INDEX_CREATION + "|" + LEGACY_SKIP_INDEX_CREATION,
+ "mbtiles: skip adding index to sqlite DB",
+ false
+ );
+ this.vacuumAnalyze = arguments.getBoolean(
+ VACUUM_ANALYZE + "|" + LEGACY_VACUUM_ANALYZE,
+ "mbtiles: vacuum analyze sqlite DB after writing",
+ false
+ );
}
/** Returns a new mbtiles file that won't get written to disk. Useful for toy use-cases like unit tests. */
public static Mbtiles newInMemoryDatabase(boolean compactDb) {
- try {
- SQLiteConfig config = new SQLiteConfig();
- config.setApplicationId(MBTILES_APPLICATION_ID);
- return new Mbtiles(DriverManager.getConnection("jdbc:sqlite::memory:", config.toProperties()), compactDb);
- } catch (SQLException throwables) {
- throw new IllegalStateException("Unable to create in-memory database", throwables);
- }
+ return newInMemoryDatabase(Arguments.of(COMPACT_DB, compactDb ? "true" : "false"));
}
- /** @see {@link #newInMemoryDatabase(boolean)} */
+ /** Returns an in-memory database with extra mbtiles and pragma options set from {@code options}. */
+ public static Mbtiles newInMemoryDatabase(Arguments options) {
+ SQLiteConfig config = new SQLiteConfig();
+ config.setApplicationId(MBTILES_APPLICATION_ID);
+ return new Mbtiles(newConnection("jdbc:sqlite::memory:", config, options), options);
+ }
+
+ /** Alias for {@link #newInMemoryDatabase(boolean)} */
public static Mbtiles newInMemoryDatabase() {
return newInMemoryDatabase(true);
}
- /** Returns a new connection to an mbtiles file optimized for fast bulk writes. */
- public static Mbtiles newWriteToFileDatabase(Path path, boolean compactDb) {
- try {
- SQLiteConfig config = new SQLiteConfig();
- config.setJournalMode(SQLiteConfig.JournalMode.OFF);
- config.setSynchronous(SQLiteConfig.SynchronousMode.OFF);
- config.setCacheSize(1_000_000); // 1GB
- config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
- config.setTempStore(SQLiteConfig.TempStore.MEMORY);
- config.setApplicationId(MBTILES_APPLICATION_ID);
- return new Mbtiles(DriverManager.getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties()),
- compactDb);
- } catch (SQLException throwables) {
- throw new IllegalArgumentException("Unable to open " + path, throwables);
- }
+ /**
+ * Returns a new connection to an mbtiles file optimized for fast bulk writes with extra mbtiles and pragma options
+ * set from {@code options}.
+ */
+ public static Mbtiles newWriteToFileDatabase(Path path, Arguments options) {
+ Objects.requireNonNull(path);
+ SQLiteConfig sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setJournalMode(SQLiteConfig.JournalMode.OFF);
+ sqliteConfig.setSynchronous(SQLiteConfig.SynchronousMode.OFF);
+ sqliteConfig.setCacheSize(1_000_000); // 1GB
+ sqliteConfig.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
+ sqliteConfig.setTempStore(SQLiteConfig.TempStore.MEMORY);
+ sqliteConfig.setApplicationId(MBTILES_APPLICATION_ID);
+ var connection = newConnection("jdbc:sqlite:" + path.toAbsolutePath(), sqliteConfig, options);
+ return new Mbtiles(connection, options);
}
/** Returns a new connection to an mbtiles file optimized for reads. */
public static Mbtiles newReadOnlyDatabase(Path path) {
+ return newReadOnlyDatabase(path, Arguments.of());
+ }
+
+ /**
+ * Returns a new connection to an mbtiles file optimized for reads with extra mbtiles and pragma options set from
+ * {@code options}.
+ */
+ public static Mbtiles newReadOnlyDatabase(Path path, Arguments options) {
+ Objects.requireNonNull(path);
+ SQLiteConfig config = new SQLiteConfig();
+ config.setReadOnly(true);
+ config.setCacheSize(100_000);
+ config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
+ config.setPageSize(32_768);
+ // helps with 3 or more threads concurrently accessing:
+ // config.setOpenMode(SQLiteOpenMode.NOMUTEX);
+ Connection connection = newConnection("jdbc:sqlite:" + path.toAbsolutePath(), config, options);
+ return new Mbtiles(connection, options);
+ }
+
+ private static Connection newConnection(String url, SQLiteConfig defaults, Arguments args) {
try {
- SQLiteConfig config = new SQLiteConfig();
- config.setReadOnly(true);
- config.setCacheSize(100_000);
- config.setLockingMode(SQLiteConfig.LockingMode.EXCLUSIVE);
- config.setPageSize(32_768);
- // helps with 3 or more threads concurrently accessing:
- // config.setOpenMode(SQLiteOpenMode.NOMUTEX);
- Connection connection = DriverManager
- .getConnection("jdbc:sqlite:" + path.toAbsolutePath(), config.toProperties());
- return new Mbtiles(connection, false /* in read-only mode, it's irrelevant if compact or not */);
+ args = args.copy().silence();
+ var config = new SQLiteConfig(defaults.toProperties());
+ for (var pragma : SQLiteConfig.Pragma.values()) {
+ var value = args.getString(pragma.getPragmaName(), pragma.getPragmaName(), null);
+ if (value != null) {
+ LOGGER.info("Setting custom mbtiles sqlite pragma {}={}", pragma.getPragmaName(), value);
+ config.setPragma(pragma, value);
+ }
+ }
+ return DriverManager.getConnection(url, config.toProperties());
} catch (SQLException throwables) {
- throw new IllegalArgumentException("Unable to open " + path, throwables);
+ throw new IllegalArgumentException("Unable to open " + url, throwables);
}
}
+ @Override
+ public boolean deduplicates() {
+ return compactDb;
+ }
+
@Override
public TileOrder tileOrder() {
return TileOrder.TMS;
}
@Override
- public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
- if (config.skipIndexCreation()) {
+ public void initialize(TileArchiveMetadata tileArchiveMetadata) {
+ if (skipIndexCreation) {
createTablesWithoutIndexes();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Skipping index creation. Add later by executing: {}",
@@ -168,26 +222,12 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
createTablesWithIndexes();
}
- var metadata = metadata()
- .setName(tileArchiveMetadata.name())
- .setFormat("pbf")
- .setDescription(tileArchiveMetadata.description())
- .setAttribution(tileArchiveMetadata.attribution())
- .setVersion(tileArchiveMetadata.version())
- .setType(tileArchiveMetadata.type())
- .setBoundsAndCenter(config.bounds().latLon())
- .setMinzoom(config.minzoom())
- .setMaxzoom(config.maxzoom())
- .setJson(new MetadataJson(layerStats.getTileStats()));
-
- for (var entry : tileArchiveMetadata.planetilerSpecific().entrySet()) {
- metadata.setMetadata(entry.getKey(), entry.getValue());
- }
+ metadataTable().set(tileArchiveMetadata);
}
@Override
- public void finish(PlanetilerConfig config) {
- if (config.optimizeDb()) {
+ public void finish(TileArchiveMetadata tileArchiveMetadata) {
+ if (vacuumAnalyze) {
vacuumAnalyze();
}
}
@@ -341,9 +381,9 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
return newTileWriter();
}
- /** Returns the contents of the metadata table. */
- public Metadata metadata() {
- return new Metadata();
+ @Override
+ public TileArchiveMetadata metadata() {
+ return new Metadata().get();
}
/** Returns the contents of the metadata table. */
@@ -389,12 +429,21 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
return connection;
}
+ public boolean skipIndexCreation() {
+ return skipIndexCreation;
+ }
+
+ public boolean compactDb() {
+ return compactDb;
+ }
+
/**
* Data contained in the {@code json} row of the metadata table
*
* @see MBtiles
* schema
*/
+ // TODO add tilestats
public record MetadataJson(
@JsonProperty("vector_layers") List vectorLayers
) {
@@ -405,7 +454,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
public static MetadataJson fromJson(String json) {
try {
- return objectMapper.readValue(json, MetadataJson.class);
+ return json == null ? null : objectMapper.readValue(json, MetadataJson.class);
} catch (JsonProcessingException e) {
throw new IllegalStateException("Invalid metadata json: " + json, e);
}
@@ -466,6 +515,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
/** Contents of a row of the tiles_data table. */
private record TileDataEntry(int tileDataId, byte[] tileData) {
+
@Override
public String toString() {
return "TileDataEntry [tileDataId=" + tileDataId + ", tileData=" + Arrays.toString(tileData) + "]";
@@ -494,6 +544,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
/** Iterates through tile coordinates one at a time without materializing the entire list in memory. */
private class TileCoordIterator implements CloseableIterator {
+
private final Statement statement;
private final ResultSet rs;
private boolean hasNext = false;
@@ -568,7 +619,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
insertStmtTableName = tableName;
insertStmtInsertIgnore = insertIgnore;
insertStmtValuesPlaceHolder = columns.stream().map(c -> "?").collect(Collectors.joining(",", "(", ")"));
- insertStmtColumnsCsv = columns.stream().collect(Collectors.joining(","));
+ insertStmtColumnsCsv = String.join(",", columns);
batchStatement = createBatchInsertPreparedStatement(batchLimit);
}
@@ -779,16 +830,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
/** Data contained in the metadata table. */
public class Metadata {
- private static final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
-
- static {
- nf.setMaximumFractionDigits(5);
- }
-
- private static String join(double... items) {
- return DoubleStream.of(items).mapToObj(nf::format).collect(Collectors.joining(","));
- }
-
+ /** Inserts a row into the metadata table that sets {@code name=value}. */
public Metadata setMetadata(String name, Object value) {
if (value != null) {
String stringValue = value.toString();
@@ -810,79 +852,15 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
return this;
}
- public Metadata setName(String value) {
- return setMetadata("name", value);
- }
-
- /** Format of the tile data, should always be pbf {@code pbf}. */
- public Metadata setFormat(String format) {
- return setMetadata("format", format);
- }
-
- public Metadata setBounds(double left, double bottom, double right, double top) {
- return setMetadata("bounds", join(left, bottom, right, top));
- }
-
- public Metadata setBounds(Envelope envelope) {
- return setBounds(envelope.getMinX(), envelope.getMinY(), envelope.getMaxX(), envelope.getMaxY());
- }
-
- public Metadata setCenter(double longitude, double latitude, double zoom) {
- return setMetadata("center", join(longitude, latitude, zoom));
- }
-
- public Metadata setBoundsAndCenter(Envelope envelope) {
- return setBounds(envelope).setCenter(envelope);
- }
-
- /** Estimate a reasonable center for the map to fit an envelope. */
- public Metadata setCenter(Envelope envelope) {
- Coordinate center = envelope.centre();
- double zoom = Math.ceil(GeoUtils.getZoomFromLonLatBounds(envelope));
- return setCenter(center.x, center.y, zoom);
- }
-
- public Metadata setMinzoom(int value) {
- return setMetadata("minzoom", value);
- }
-
- public Metadata setMaxzoom(int maxZoom) {
- return setMetadata("maxzoom", maxZoom);
- }
-
- public Metadata setAttribution(String value) {
- return setMetadata("attribution", value);
- }
-
- public Metadata setDescription(String value) {
- return setMetadata("description", value);
- }
-
- /** {@code overlay} or {@code baselayer}. */
- public Metadata setType(String value) {
- return setMetadata("type", value);
- }
-
- public Metadata setTypeIsOverlay() {
- return setType("overlay");
- }
-
- public Metadata setTypeIsBaselayer() {
- return setType("baselayer");
- }
-
- public Metadata setVersion(String value) {
- return setMetadata("version", value);
- }
-
- public Metadata setJson(String value) {
- return setMetadata("json", value);
- }
-
+ /**
+ * Inserts a row into the metadata table that sets the value for {@code "json"} key to {@code value} serialized as a
+ * string.
+ */
public Metadata setJson(MetadataJson value) {
- return value == null ? this : setJson(value.toJson());
+ return value == null ? this : setMetadata("json", value.toJson());
}
+ /** Returns all key-value pairs from the metadata table. */
public Map getAll() {
TreeMap result = new TreeMap<>();
try (Statement statement = connection.createStatement()) {
@@ -895,10 +873,86 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
);
}
} catch (SQLException throwables) {
- LOGGER.warn("Error retrieving metadata: " + throwables);
+ LOGGER.warn("Error retrieving metadata: {}", throwables.toString());
LOGGER.trace("Error retrieving metadata details: ", throwables);
}
return result;
}
+
+ /**
+ * Inserts rows into the metadata table that set all of the well-known metadata keys from
+ * {@code tileArchiveMetadata} and passes through the raw values of any options not explicitly called out in the
+ * MBTiles specification.
+ *
+ * @see MBTiles 1.3
+ * specification
+ */
+ public Metadata set(TileArchiveMetadata tileArchiveMetadata) {
+ var map = new LinkedHashMap<>(tileArchiveMetadata.toMap());
+
+ setMetadata(TileArchiveMetadata.FORMAT_KEY, tileArchiveMetadata.format());
+ var center = tileArchiveMetadata.center();
+ var zoom = tileArchiveMetadata.zoom();
+ if (center != null) {
+ if (zoom != null) {
+ setMetadata(TileArchiveMetadata.CENTER_KEY, joinCoordinates(center.x, center.y, Math.ceil(zoom)));
+ } else {
+ setMetadata(TileArchiveMetadata.CENTER_KEY, joinCoordinates(center.x, center.y));
+ }
+ }
+ var bounds = tileArchiveMetadata.bounds();
+ if (bounds != null) {
+ setMetadata(TileArchiveMetadata.BOUNDS_KEY,
+ joinCoordinates(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY()));
+ }
+ setJson(new MetadataJson(tileArchiveMetadata.vectorLayers()));
+
+ map.remove(TileArchiveMetadata.FORMAT_KEY);
+ map.remove(TileArchiveMetadata.CENTER_KEY);
+ map.remove(TileArchiveMetadata.ZOOM_KEY);
+ map.remove(TileArchiveMetadata.BOUNDS_KEY);
+ map.remove(TileArchiveMetadata.VECTOR_LAYERS_KEY);
+
+ for (var entry : map.entrySet()) {
+ setMetadata(entry.getKey(), entry.getValue());
+ }
+ return this;
+ }
+
+ /**
+ * Returns a {@link TileArchiveMetadata} instance parsed from all the rows in the metadata table.
+ */
+ public TileArchiveMetadata get() {
+ Map map = new HashMap<>(getAll());
+ String[] bounds = map.containsKey(TileArchiveMetadata.BOUNDS_KEY) ?
+ map.remove(TileArchiveMetadata.BOUNDS_KEY).split(",") : null;
+ String[] center = map.containsKey(TileArchiveMetadata.CENTER_KEY) ?
+ map.remove(TileArchiveMetadata.CENTER_KEY).split(",") : null;
+ var metadataJson = MetadataJson.fromJson(map.remove("json"));
+ return new TileArchiveMetadata(
+ map.remove(TileArchiveMetadata.NAME_KEY),
+ map.remove(TileArchiveMetadata.DESCRIPTION_KEY),
+ map.remove(TileArchiveMetadata.ATTRIBUTION_KEY),
+ map.remove(TileArchiveMetadata.VERSION_KEY),
+ map.remove(TileArchiveMetadata.TYPE_KEY),
+ map.remove(TileArchiveMetadata.FORMAT_KEY),
+ bounds == null || bounds.length < 4 ? null : new Envelope(
+ Double.parseDouble(bounds[0]),
+ Double.parseDouble(bounds[2]),
+ Double.parseDouble(bounds[1]),
+ Double.parseDouble(bounds[3])
+ ),
+ center == null || center.length < 2 ? null : new CoordinateXY(
+ Double.parseDouble(center[0]),
+ Double.parseDouble(center[1])
+ ),
+ center == null || center.length < 3 ? null : Double.parseDouble(center[2]),
+ Parse.parseIntOrNull(map.remove(TileArchiveMetadata.MINZOOM_KEY)),
+ Parse.parseIntOrNull(map.remove(TileArchiveMetadata.MAXZOOM_KEY)),
+ metadataJson == null ? null : metadataJson.vectorLayers,
+ // any left-overs:
+ map
+ );
+ }
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
index 8fa73fd6..99a43f15 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
@@ -189,7 +189,7 @@ public class Verify {
}
private void checkBasicStructure() {
- check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name"));
+ check("contains name attribute", () -> mbtiles.metadata().toMap().containsKey("name"));
check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent());
checkWithMessage("all tiles are valid", () -> {
List invalidTiles = mbtiles.getAllTileCoords().stream()
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java
index d05716ec..107ce094 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/Pmtiles.java
@@ -23,6 +23,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.Envelope;
public class Pmtiles {
public enum Compression {
@@ -201,6 +203,22 @@ public class Pmtiles {
throw new FileFormatException("Failed to read enough bytes for PMTiles header.");
}
}
+
+ public Envelope bounds() {
+ return new Envelope(
+ minLonE7 / 1e7,
+ maxLonE7 / 1e7,
+ minLatE7 / 1e7,
+ maxLatE7 / 1e7
+ );
+ }
+
+ public CoordinateXY center() {
+ return new CoordinateXY(
+ centerLonE7 / 1e7,
+ centerLatE7 / 1e7
+ );
+ }
}
public static final class Entry implements Comparable {
@@ -366,7 +384,7 @@ public class Pmtiles {
try {
return objectMapper.readValue(bytes, JsonMetadata.class);
} catch (IOException e) {
- throw new IllegalStateException("Invalid metadata json: " + bytes, e);
+ throw new IllegalStateException("Invalid metadata json: " + new String(bytes, StandardCharsets.UTF_8), e);
}
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java
index 05d0c265..fbabc851 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/ReadablePmtiles.java
@@ -1,13 +1,19 @@
package com.onthegomap.planetiler.pmtiles;
import com.onthegomap.planetiler.archive.ReadableTileArchive;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.util.CloseableIterator;
import com.onthegomap.planetiler.util.Gzip;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.stream.IntStream;
import java.util.stream.Stream;
@@ -22,6 +28,10 @@ public class ReadablePmtiles implements ReadableTileArchive {
this.header = Pmtiles.Header.fromBytes(getBytes(0, Pmtiles.HEADER_LEN));
}
+ public static ReadableTileArchive newReadFromFile(Path path) throws IOException {
+ return new ReadablePmtiles(FileChannel.open(path, StandardOpenOption.READ));
+ }
+
private synchronized byte[] getBytes(long start, int length) throws IOException {
channel.position(start);
var buf = ByteBuffer.allocate(length);
@@ -103,6 +113,34 @@ public class ReadablePmtiles implements ReadableTileArchive {
return Pmtiles.JsonMetadata.fromBytes(buf);
}
+ @Override
+ public TileArchiveMetadata metadata() {
+ try {
+ var jsonMetadata = getJsonMetadata();
+ var map = new LinkedHashMap<>(jsonMetadata.otherMetadata());
+ return new TileArchiveMetadata(
+ map.remove(TileArchiveMetadata.NAME_KEY),
+ map.remove(TileArchiveMetadata.DESCRIPTION_KEY),
+ map.remove(TileArchiveMetadata.ATTRIBUTION_KEY),
+ map.remove(TileArchiveMetadata.VERSION_KEY),
+ map.remove(TileArchiveMetadata.TYPE_KEY),
+ switch (header.tileType()) {
+ case MVT -> TileArchiveMetadata.MVT_FORMAT;
+ default -> null;
+ },
+ header.bounds(),
+ header.center(),
+ (double) header.centerZoom(),
+ (int) header.minZoom(),
+ (int) header.maxZoom(),
+ jsonMetadata.vectorLayers(),
+ map
+ );
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
private static class TileCoordIterator implements CloseableIterator {
private final Stream stream;
private final Iterator iterator;
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
index ed541ed6..e99b4b6b 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
@@ -12,7 +12,6 @@ import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.geo.TileOrder;
import com.onthegomap.planetiler.util.Format;
import com.onthegomap.planetiler.util.Gzip;
-import com.onthegomap.planetiler.util.LayerStats;
import com.onthegomap.planetiler.util.SeekableInMemoryByteChannel;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -24,10 +23,10 @@ import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import java.util.OptionalLong;
-import org.locationtech.jts.geom.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,8 +45,6 @@ public final class WriteablePmtiles implements WriteableTileArchive {
private long currentOffset = 0;
private long numUnhashedTiles = 0;
private long numAddressedTiles = 0;
- private LayerStats layerStats;
- private TileArchiveMetadata tileArchiveMetadata;
private boolean isClustered = true;
private WriteablePmtiles(SeekableByteChannel channel) throws IOException {
@@ -91,7 +88,7 @@ public final class WriteablePmtiles implements WriteableTileArchive {
* @return byte arrays of the root and all leaf directories, and the # of leaves.
* @throws IOException if compression fails
*/
- protected static Directories makeDirectories(List entries) throws IOException {
+ static Directories makeDirectories(List entries) throws IOException {
int maxEntriesRootOnly = 16384;
int attemptNum = 1;
if (entries.size() < maxEntriesRootOnly) {
@@ -124,19 +121,18 @@ public final class WriteablePmtiles implements WriteableTileArchive {
return new WriteablePmtiles(bytes);
}
+ @Override
+ public boolean deduplicates() {
+ return true;
+ }
+
@Override
public TileOrder tileOrder() {
return TileOrder.HILBERT;
}
@Override
- public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
- this.layerStats = layerStats;
- this.tileArchiveMetadata = tileArchiveMetadata;
- }
-
- @Override
- public void finish(PlanetilerConfig config) {
+ public void finish(TileArchiveMetadata tileArchiveMetadata) {
if (!isClustered) {
LOGGER.info("Tile data was not written in order, sorting entries...");
Collections.sort(entries);
@@ -144,10 +140,32 @@ public final class WriteablePmtiles implements WriteableTileArchive {
}
try {
Directories directories = makeDirectories(entries);
- byte[] jsonBytes = new Pmtiles.JsonMetadata(layerStats.getTileStats(), tileArchiveMetadata.getAll()).toBytes();
+ var otherMetadata = new LinkedHashMap<>(tileArchiveMetadata.toMap());
+
+ // exclude keys included in top-level header
+ otherMetadata.remove(TileArchiveMetadata.CENTER_KEY);
+ otherMetadata.remove(TileArchiveMetadata.ZOOM_KEY);
+ otherMetadata.remove(TileArchiveMetadata.BOUNDS_KEY);
+ otherMetadata.remove(TileArchiveMetadata.FORMAT_KEY);
+ otherMetadata.remove(TileArchiveMetadata.MINZOOM_KEY);
+ otherMetadata.remove(TileArchiveMetadata.MAXZOOM_KEY);
+ otherMetadata.remove(TileArchiveMetadata.VECTOR_LAYERS_KEY);
+
+ byte[] jsonBytes =
+ new Pmtiles.JsonMetadata(tileArchiveMetadata.vectorLayers(), otherMetadata).toBytes();
jsonBytes = Gzip.gzip(jsonBytes);
- Envelope envelope = config.bounds().latLon();
+ String formatString = tileArchiveMetadata.format();
+ var outputFormat =
+ TileArchiveMetadata.MVT_FORMAT.equals(formatString) ? Pmtiles.TileType.MVT : Pmtiles.TileType.UNKNOWN;
+
+ var bounds = tileArchiveMetadata.bounds() == null ? GeoUtils.WORLD_LAT_LON_BOUNDS : tileArchiveMetadata.bounds();
+ var center = tileArchiveMetadata.center() == null ? bounds.centre() : tileArchiveMetadata.center();
+ int zoom = (int) Math.ceil(tileArchiveMetadata.zoom() == null ? GeoUtils.getZoomFromLonLatBounds(bounds) :
+ tileArchiveMetadata.zoom());
+ int minzoom = tileArchiveMetadata.minzoom() == null ? 0 : tileArchiveMetadata.minzoom();
+ int maxzoom =
+ tileArchiveMetadata.maxzoom() == null ? PlanetilerConfig.MAX_MAXZOOM : tileArchiveMetadata.maxzoom();
Pmtiles.Header header = new Pmtiles.Header(
(byte) 3,
@@ -165,16 +183,16 @@ public final class WriteablePmtiles implements WriteableTileArchive {
isClustered,
Pmtiles.Compression.GZIP,
Pmtiles.Compression.GZIP,
- Pmtiles.TileType.MVT,
- (byte) config.minzoom(),
- (byte) config.maxzoom(),
- (int) (envelope.getMinX() * 10_000_000),
- (int) (envelope.getMinY() * 10_000_000),
- (int) (envelope.getMaxX() * 10_000_000),
- (int) (envelope.getMaxY() * 10_000_000),
- (byte) Math.ceil(GeoUtils.getZoomFromLonLatBounds(envelope)),
- (int) ((envelope.getMinX() + envelope.getMaxX()) / 2 * 10_000_000),
- (int) ((envelope.getMinY() + envelope.getMaxY()) / 2 * 10_000_000)
+ outputFormat,
+ (byte) minzoom,
+ (byte) maxzoom,
+ (int) (bounds.getMinX() * 10_000_000),
+ (int) (bounds.getMinY() * 10_000_000),
+ (int) (bounds.getMaxX() * 10_000_000),
+ (int) (bounds.getMaxY() * 10_000_000),
+ (byte) zoom,
+ (int) center.x * 10_000_000,
+ (int) center.y * 10_000_000
);
LOGGER.info("Writing metadata and leaf directories...");
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
index 75e55405..e0891438 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java
@@ -43,15 +43,17 @@ public interface Stats extends AutoCloseable {
*/
default void printSummary() {
Format format = Format.defaultInstance();
- Logger LOGGER = LoggerFactory.getLogger(getClass());
- LOGGER.info("");
- LOGGER.info("-".repeat(40));
- timers().printSummary();
- LOGGER.info("-".repeat(40));
- for (var entry : monitoredFiles().entrySet()) {
- long size = FileUtils.size(entry.getValue());
- if (size > 0) {
- LOGGER.info("\t" + entry.getKey() + "\t" + format.storage(size, false) + "B");
+ Logger logger = LoggerFactory.getLogger(getClass());
+ if (logger.isInfoEnabled()) {
+ logger.info("");
+ logger.info("-".repeat(40));
+ timers().printSummary();
+ logger.info("-".repeat(40));
+ for (var entry : monitoredFiles().entrySet()) {
+ long size = FileUtils.size(entry.getValue());
+ if (size > 0) {
+ logger.info("\t{}\t{}B", entry.getKey(), format.storage(size, false));
+ }
}
}
}
@@ -110,7 +112,9 @@ public interface Stats extends AutoCloseable {
/** Adds a stat that will track the size of a file or directory located at {@code path}. */
default void monitorFile(String name, Path path) {
- monitoredFiles().put(name, path);
+ if (path != null) {
+ monitoredFiles().put(name, path);
+ }
}
/** Adds a stat that will track the estimated in-memory size of {@code object}. */
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java
index ccca9b45..e78c7a64 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java
@@ -217,17 +217,19 @@ public class FileUtils {
*/
public static void createParentDirectories(Path... paths) {
for (var path : paths) {
- try {
- if (Files.isDirectory(path) && !Files.exists(path)) {
- Files.createDirectories(path);
- } else {
- Path parent = path.getParent();
- if (parent != null && !Files.exists(parent)) {
- Files.createDirectories(parent);
+ if (path != null) {
+ try {
+ if (Files.isDirectory(path) && !Files.exists(path)) {
+ Files.createDirectories(path);
+ } else {
+ Path parent = path.getParent();
+ if (parent != null && !Files.exists(parent)) {
+ Files.createDirectories(parent);
+ }
}
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to create parent directories " + path, e);
}
- } catch (IOException e) {
- throw new IllegalStateException("Unable to create parent directories " + path, e);
}
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java
index cd8a4a2a..6614766f 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Format.java
@@ -8,6 +8,8 @@ import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+import java.util.stream.DoubleStream;
import org.apache.commons.text.StringEscapeUtils;
import org.locationtech.jts.geom.Coordinate;
@@ -19,6 +21,26 @@ public class Format {
public static final Locale DEFAULT_LOCALE = Locale.getDefault(Locale.Category.FORMAT);
private static final ConcurrentMap instances = new ConcurrentHashMap<>();
+ @SuppressWarnings("java:S5164")
+ private static final NumberFormat latLonNF = NumberFormat.getNumberInstance(Locale.US);
+ private static final NavigableMap STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries(
+ Map.entry(1_000L, "k"),
+ Map.entry(1_000_000L, "M"),
+ Map.entry(1_000_000_000L, "G"),
+ Map.entry(1_000_000_000_000L, "T"),
+ Map.entry(1_000_000_000_000_000L, "P")
+ ));
+ private static final NavigableMap NUMERIC_SUFFIXES = new TreeMap<>(Map.ofEntries(
+ Map.entry(1_000L, "k"),
+ Map.entry(1_000_000L, "M"),
+ Map.entry(1_000_000_000L, "B"),
+ Map.entry(1_000_000_000_000L, "T"),
+ Map.entry(1_000_000_000_000_000L, "Q")
+ ));
+
+ static {
+ latLonNF.setMaximumFractionDigits(5);
+ }
// `NumberFormat` instances are not thread safe, so we need to wrap them inside a `ThreadLocal`.
//
@@ -49,6 +71,11 @@ public class Format {
});
}
+ /** Returns a string with {@code items} rounded to 5 decimals and joined with a comma. */
+ public static synchronized String joinCoordinates(double... items) {
+ return DoubleStream.of(items).mapToObj(latLonNF::format).collect(Collectors.joining(","));
+ }
+
public static Format forLocale(Locale locale) {
return instances.computeIfAbsent(locale, Format::new);
}
@@ -57,21 +84,6 @@ public class Format {
return forLocale(DEFAULT_LOCALE);
}
- private static final NavigableMap STORAGE_SUFFIXES = new TreeMap<>(Map.ofEntries(
- Map.entry(1_000L, "k"),
- Map.entry(1_000_000L, "M"),
- Map.entry(1_000_000_000L, "G"),
- Map.entry(1_000_000_000_000L, "T"),
- Map.entry(1_000_000_000_000_000L, "P")
- ));
- private static final NavigableMap NUMERIC_SUFFIXES = new TreeMap<>(Map.ofEntries(
- Map.entry(1_000L, "k"),
- Map.entry(1_000_000L, "M"),
- Map.entry(1_000_000_000L, "B"),
- Map.entry(1_000_000_000_000L, "T"),
- Map.entry(1_000_000_000_000_000L, "Q")
- ));
-
public static String padRight(String str, int size) {
StringBuilder strBuilder = new StringBuilder(str);
while (strBuilder.length() < size) {
@@ -88,6 +100,23 @@ public class Format {
return strBuilder.toString();
}
+ /** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */
+ public static String quote(Object string) {
+ if (string == null) {
+ return "null";
+ }
+ return '"' + StringEscapeUtils.escapeJava(string.toString()) + '"';
+ }
+
+ /** Returns an openstreetmap.org map link for a lat/lon */
+ public static String osmDebugUrl(int zoom, Coordinate coord) {
+ return "https://www.openstreetmap.org/#map=%d/%.5f/%.5f".formatted(
+ zoom,
+ coord.y,
+ coord.x
+ );
+ }
+
/** Returns a number of bytes formatted like "123" "1.2k" "240M", etc. */
public String storage(Number num, boolean pad) {
return format(num, pad, STORAGE_SUFFIXES);
@@ -161,21 +190,4 @@ public class Format {
}
return simplified.toString().replace("PT", "").toLowerCase(Locale.ROOT);
}
-
- /** Returns Java code that can re-create {@code string}: {@code null} if null, or {@code "contents"} if not empty. */
- public static String quote(Object string) {
- if (string == null) {
- return "null";
- }
- return '"' + StringEscapeUtils.escapeJava(string.toString()) + '"';
- }
-
- /** Returns an openstreetmap.org map link for a lat/lon */
- public static String osmDebugUrl(int zoom, Coordinate coord) {
- return "https://www.openstreetmap.org/#map=%d/%.5f/%.5f".formatted(
- zoom,
- coord.y,
- coord.x
- );
- }
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java
index b21182ee..1cc3fa03 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/ResourceUsage.java
@@ -70,7 +70,7 @@ public class ResourceUsage {
/** Requests {@code amount} bytes on the file system that contains {@code path}. */
public ResourceUsage addDisk(Path path, long amount, String description) {
- return add(new DiskUsage(path), amount, description);
+ return path == null ? this : add(new DiskUsage(path), amount, description);
}
/** Requests {@code amount} bytes of RAM in the JVM heap. */
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
index 1769797a..19e9e40f 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
@@ -15,6 +15,7 @@ import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.geo.TileOrder;
import com.onthegomap.planetiler.mbtiles.Mbtiles;
+import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
import com.onthegomap.planetiler.reader.SimpleFeature;
import com.onthegomap.planetiler.reader.SimpleReader;
import com.onthegomap.planetiler.reader.SourceFeature;
@@ -141,14 +142,14 @@ class PlanetilerTests {
FeatureGroup featureGroup = FeatureGroup.newInMemoryFeatureGroup(TileOrder.TMS, profile, stats);
runner.run(featureGroup, profile, config);
featureGroup.prepare();
- try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.compactDb())) {
- TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config.arguments()),
+ try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.arguments())) {
+ TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config),
config,
stats);
var tileMap = TestUtils.getTileMap(db);
tileMap.values().forEach(fs -> fs.forEach(f -> f.geometry().validate()));
- int tileDataCount = config.compactDb() ? TestUtils.getTilesDataCount(db) : 0;
- return new PlanetilerResults(tileMap, db.metadata().getAll(), tileDataCount);
+ int tileDataCount = db.compactDb() ? TestUtils.getTilesDataCount(db) : 0;
+ return new PlanetilerResults(tileMap, db.metadata().toMap(), tileDataCount);
}
}
@@ -248,20 +249,15 @@ class PlanetilerTests {
"format", "pbf",
"minzoom", "0",
"maxzoom", "14",
- "center", "0,0,0",
+ "center", "0,0",
"bounds", "-180,-85.05113,180,85.05113"
), results.metadata);
assertSubmap(Map.of(
"planetiler:version", BuildInfo.get().version()
), results.metadata);
assertSameJson(
- """
- {
- "vector_layers": [
- ]
- }
- """,
- results.metadata.get("json")
+ "[]",
+ results.metadata.get("vector_layers")
);
}
@@ -269,11 +265,11 @@ class PlanetilerTests {
void testOverrideMetadata() throws Exception {
var results = runWithReaderFeatures(
Map.of(
- "mbtiles_name", "override_name",
- "mbtiles_description", "override_description",
- "mbtiles_attribution", "override_attribution",
- "mbtiles_version", "override_version",
- "mbtiles_type", "override_type"
+ "archive_name", "override_name",
+ "archive_description", "override_description",
+ "archive_attribution", "override_attribution",
+ "archive_version", "override_version",
+ "archive_type", "override_type"
),
List.of(),
(sourceFeature, features) -> {
@@ -331,13 +327,11 @@ class PlanetilerTests {
), results.tiles);
assertSameJson(
"""
- {
- "vector_layers": [
- {"id": "layer", "fields": {"name": "String", "attr": "String"}, "minzoom": 13, "maxzoom": 15}
- ]
- }
+ [
+ {"id": "layer", "fields": {"name": "String", "attr": "String"}, "minzoom": 13, "maxzoom": 15}
+ ]
""",
- results.metadata.get("json")
+ results.metadata.get("vector_layers")
);
}
@@ -1686,13 +1680,14 @@ class PlanetilerTests {
@ValueSource(strings = {
"",
"--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4",
- "--emit-tiles-in-order=false",
"--free-osm-after-read",
"--osm-parse-node-bounds",
+ "--output-format=pmtiles"
})
void testPlanetilerRunner(String args) throws Exception {
+ boolean pmtiles = args.contains("pmtiles");
Path originalOsm = TestUtils.pathToResource("monaco-latest.osm.pbf");
- Path mbtiles = tempDir.resolve("output.mbtiles");
+ Path output = tempDir.resolve(pmtiles ? "output.pmtiles" : "output.mbtiles");
Path tempOsm = tempDir.resolve("monaco-temp.osm.pbf");
Files.copy(originalOsm, tempOsm);
Planetiler.create(Arguments.fromArgs(
@@ -1710,7 +1705,7 @@ class PlanetilerTests {
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null)
- .setOutput("mbtiles", mbtiles)
+ .setOutput(output)
.run();
// make sure it got deleted after write
@@ -1718,7 +1713,9 @@ class PlanetilerTests {
assertFalse(Files.exists(tempOsm));
}
- try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
+ try (
+ var db = pmtiles ? ReadablePmtiles.newReadFromFile(output) : Mbtiles.newReadOnlyDatabase(output)
+ ) {
int features = 0;
var tileMap = TestUtils.getTileMap(db);
for (var tile : tileMap.values()) {
@@ -1735,7 +1732,7 @@ class PlanetilerTests {
"planetiler:osm:osmosisreplicationtime", "2021-04-21T20:21:46Z",
"planetiler:osm:osmosisreplicationseq", "2947",
"planetiler:osm:osmosisreplicationurl", "http://download.geofabrik.de/europe/monaco-updates"
- ), db.metadata().getAll());
+ ), db.metadata().toMap());
}
}
@@ -1760,7 +1757,7 @@ class PlanetilerTests {
.addShapefileGlobSource("shapefile-glob-zip", resourceDir.resolve("shapefile.zip"), "*.shp")
// Match *.shp within shapefile.zip
.addShapefileSource("shapefile", resourceDir.resolve("shapefile.zip"))
- .setOutput("mbtiles", mbtiles)
+ .setOutput(mbtiles)
.run();
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
@@ -1806,7 +1803,7 @@ class PlanetilerTests {
}
})
.addGeoPackageSource("geopackage", TestUtils.pathToResource(inputFile), null)
- .setOutput("mbtiles", mbtiles)
+ .setOutput(mbtiles)
.run();
try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
@@ -1834,7 +1831,7 @@ class PlanetilerTests {
.addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
.addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
.addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg.zip"), null)
- .setOutput("mbtiles", tempDir.resolve("output.mbtiles"))
+ .setOutput(tempDir.resolve("output.mbtiles"))
.run();
}
@@ -1909,9 +1906,8 @@ class PlanetilerTests {
private PlanetilerResults runForCompactTest(boolean compactDbEnabled) throws Exception {
-
return runWithReaderFeatures(
- Map.of("threads", "1", "compact-db", Boolean.toString(compactDbEnabled)),
+ Map.of("threads", "1", "mbtiles-compact", Boolean.toString(compactDbEnabled)),
List.of(
newReaderFeature(WORLD_POLYGON, Map.of())
),
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
index 819eac6b..d7de8d7b 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
@@ -15,6 +15,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.onthegomap.planetiler.archive.ReadableTileArchive;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.geo.GeometryException;
@@ -199,7 +200,8 @@ public class TestUtils {
return round(input, 1e5);
}
- public static Map> getTileMap(Mbtiles db) throws SQLException, IOException {
+ public static Map> getTileMap(ReadableTileArchive db)
+ throws IOException {
Map> tiles = new TreeMap<>();
for (var tile : getAllTiles(db)) {
var bytes = gunzip(tile.bytes());
@@ -218,21 +220,10 @@ public class TestUtils {
}
}
- public static Set getAllTiles(Mbtiles db) throws SQLException {
- Set result = new HashSet<>();
- try (Statement statement = db.connection().createStatement()) {
- ResultSet rs = statement.executeQuery("select zoom_level, tile_column, tile_row, tile_data from tiles");
- while (rs.next()) {
- int z = rs.getInt("zoom_level");
- int rawy = rs.getInt("tile_row");
- int x = rs.getInt("tile_column");
- result.add(new Mbtiles.TileEntry(
- TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z),
- rs.getBytes("tile_data")
- ));
- }
- }
- return result;
+ public static Set getAllTiles(ReadableTileArchive db) {
+ return db.getAllTileCoords().stream()
+ .map(coord -> new Mbtiles.TileEntry(coord, db.getTile(coord)))
+ .collect(Collectors.toSet());
}
public static int getTilesDataCount(Mbtiles db) throws SQLException {
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java
new file mode 100644
index 00000000..78ff0f06
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveConfigTest.java
@@ -0,0 +1,36 @@
+package com.onthegomap.planetiler.archive;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.nio.file.Path;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+
+class TileArchiveConfigTest {
+
+ @Test
+ void testMbtiles() {
+ var config = TileArchiveConfig.from("output.mbtiles");
+ assertEquals(TileArchiveConfig.Format.MBTILES, config.format());
+ assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme());
+ assertEquals(Map.of(), config.options());
+ assertEquals(Path.of("output.mbtiles").toAbsolutePath(), config.getLocalPath());
+ }
+
+ @Test
+ void testMbtilesWithOptions() {
+ var config = TileArchiveConfig.from("output.mbtiles?compact=true");
+ assertEquals(TileArchiveConfig.Format.MBTILES, config.format());
+ assertEquals(TileArchiveConfig.Scheme.FILE, config.scheme());
+ assertEquals(Map.of("compact", "true"), config.options());
+ assertEquals(Path.of("output.mbtiles").toAbsolutePath(), config.getLocalPath());
+ }
+
+ @Test
+ void testPmtiles() {
+ assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("output.pmtiles").format());
+ assertEquals(TileArchiveConfig.Format.PMTILES, TileArchiveConfig.from("output.mbtiles?format=pmtiles").format());
+ assertEquals(TileArchiveConfig.Format.PMTILES,
+ TileArchiveConfig.from("file:///output.mbtiles?format=pmtiles").format());
+ }
+}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java
new file mode 100644
index 00000000..985e5820
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileArchiveMetadataTest.java
@@ -0,0 +1,67 @@
+package com.onthegomap.planetiler.archive;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.geo.GeoUtils;
+import java.util.Map;
+import java.util.TreeMap;
+import org.junit.jupiter.api.Test;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.Envelope;
+
+class TileArchiveMetadataTest {
+
+ @Test
+ void testAddMetadataWorldBounds() {
+ var bounds = GeoUtils.WORLD_LAT_LON_BOUNDS;
+ var metadata = new TileArchiveMetadata(new Profile.NullProfile(), PlanetilerConfig.from(Arguments.of(Map.of(
+ "bounds", bounds.getMinX() + "," + bounds.getMinY() + "," + bounds.getMaxX() + "," + bounds.getMaxY()
+ ))));
+ assertEquals(bounds, metadata.bounds());
+ assertEquals(new CoordinateXY(0, 0), metadata.center());
+ assertEquals(0d, metadata.zoom().doubleValue());
+ }
+
+ @Test
+ void testAddMetadataSmallBounds() {
+ var bounds = new Envelope(-73.6632, -69.7598, 41.1274, 43.0185);
+ var metadata = new TileArchiveMetadata(new Profile.NullProfile(), PlanetilerConfig.from(Arguments.of(Map.of(
+ "bounds", "-73.6632,41.1274,-69.7598,43.0185"
+ ))));
+ assertEquals(bounds, metadata.bounds());
+ assertEquals(-71.7115, metadata.center().x, 1e-5);
+ assertEquals(42.07295, metadata.center().y, 1e-5);
+ assertEquals(7, Math.ceil(metadata.zoom()));
+ }
+
+ @Test
+ void testToMap() {
+ var bounds = "-73.6632,41.1274,-69.7598,43.0185";
+ var metadata = new TileArchiveMetadata(
+ new Profile.NullProfile(),
+ PlanetilerConfig.from(Arguments.of(Map.of(
+ "bounds", bounds
+ ))));
+ var map = new TreeMap<>(metadata.toMap());
+ assertNotNull(map.remove("planetiler:version"));
+ assertNotNull(map.remove("planetiler:githash"));
+ assertNotNull(map.remove("planetiler:buildtime"));
+ assertEquals(
+ new TreeMap(Map.of(
+ "name", "Null",
+ "type", "baselayer",
+ "format", "pbf",
+ "zoom", "6.5271217861412305",
+ "minzoom", "0",
+ "maxzoom", "14",
+ "bounds", "-73.6632,41.1274,-69.7598,43.0185",
+ "center", "-71.7115,42.07295"
+ )),
+ map
+ );
+ }
+}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java
index a4147f93..a51e4f3a 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/config/ArgumentsTest.java
@@ -9,6 +9,7 @@ import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Envelope;
@@ -293,4 +294,59 @@ class ArgumentsTest {
assertEquals(false, args.getBooleanObject("BOOL_FALSE", "test"));
assertEquals(false, args.getBooleanObject("BOOL_NO", "test"));
}
+
+ @Test
+ void testDeprecatedArgs() {
+ assertEquals("newvalue",
+ Arguments.of("oldkey", "oldvalue", "newkey", "newvalue")
+ .getString("newkey|oldkey", "key", "fallback"));
+ assertEquals("oldvalue",
+ Arguments.of("oldkey", "oldvalue")
+ .getString("newkey|oldkey", "key", "fallback"));
+ assertEquals("fallback",
+ Arguments.of()
+ .getString("newkey|oldkey", "key", "fallback"));
+ }
+
+ @Test
+ void testWithPrefix() {
+ var args = Arguments.of("prefix_a", "a_val", "prefix-b", "b_val", "other", "other_val").withPrefix("prefix");
+ assertEquals("a_val", args.getArg("a"));
+ assertEquals("b_val", args.getArg("b"));
+ assertNull(args.getArg("other"));
+ assertNull(args.getArg("prefix_a"));
+ assertNull(args.getArg("prefix_b"));
+ assertNull(args.getArg("prefix_other"));
+ assertEquals(Set.of("a", "b"), args.toMap().keySet());
+ }
+
+ @Test
+ void testPrefixFromEnvironment() {
+ Map env = Map.of(
+ "OTHER", "value",
+ "PLANETILEROTHER", "VALUE",
+ "PLANETILER_MBTILES_KEY1", "value1",
+ "PLANETILER_PMTILES_KEY2", "value2"
+ );
+ Arguments args = Arguments.fromEnvironment(env::get, env::keySet).withPrefix("mbtiles");
+ assertEquals(Map.of(
+ "key1", "value1"
+ ), args.toMap());
+ assertEquals("value1", args.getArg("key1"));
+ }
+
+ @Test
+ void testSubset() {
+ var args = Arguments.of(Map.of(
+ "key_1", "val_1",
+ "key-2", "val_2",
+ "key-3", "val_3"
+ )).subset("key-1", "key-2");
+ assertEquals(Map.of(
+ "key_1", "val_1",
+ "key_2", "val_2"
+ ), args.toMap());
+ assertEquals("val_1", args.getArg("key-1"));
+ assertNull(args.getArg("key-3"));
+ }
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java
index 696947b3..9ac149e1 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java
@@ -5,24 +5,27 @@ import static org.junit.jupiter.api.Assertions.*;
import com.google.common.math.IntMath;
import com.onthegomap.planetiler.TestUtils;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileEncodingResult;
-import com.onthegomap.planetiler.geo.GeoUtils;
+import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.util.LayerStats;
import java.io.IOException;
import java.math.RoundingMode;
+import java.nio.file.Path;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;
-import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import org.locationtech.jts.geom.CoordinateXY;
import org.locationtech.jts.geom.Envelope;
class MbtilesTest {
@@ -33,11 +36,12 @@ class MbtilesTest {
private static final int TILES_DATA_BATCH = MAX_PARAMETERS_IN_PREPARED_STATEMENT / 2;
- private static final
-
- void testWriteTiles(int howMany, boolean skipIndexCreation, boolean optimize, boolean compactDb)
- throws IOException, SQLException {
- try (Mbtiles db = Mbtiles.newInMemoryDatabase(compactDb)) {
+ private static void testWriteTiles(Path path, int howMany, boolean skipIndexCreation, boolean optimize,
+ boolean compactDb) throws IOException, SQLException {
+ var options = Arguments.of("compact", Boolean.toString(compactDb));
+ try (
+ Mbtiles db = path == null ? Mbtiles.newInMemoryDatabase(options) : Mbtiles.newWriteToFileDatabase(path, options)
+ ) {
if (skipIndexCreation) {
db.createTablesWithoutIndexes();
} else {
@@ -84,24 +88,42 @@ class MbtilesTest {
@ParameterizedTest
@ValueSource(ints = {0, 1, TILES_BATCH, TILES_BATCH + 1, 2 * TILES_BATCH, 2 * TILES_BATCH + 1})
void testWriteTilesDifferentSizeInNonCompactMode(int howMany) throws IOException, SQLException {
- testWriteTiles(howMany, false, false, false);
+ testWriteTiles(null, howMany, false, false, false);
}
@ParameterizedTest
@ValueSource(ints = {0, 1, TILES_DATA_BATCH, TILES_DATA_BATCH + 1, 2 * TILES_DATA_BATCH, 2 * TILES_DATA_BATCH + 1,
TILES_SHALLOW_BATCH, TILES_SHALLOW_BATCH + 1, 2 * TILES_SHALLOW_BATCH, 2 * TILES_SHALLOW_BATCH + 1})
void testWriteTilesDifferentSizeInCompactMode(int howMany) throws IOException, SQLException {
- testWriteTiles(howMany, false, false, true);
+ testWriteTiles(null, howMany, false, false, true);
}
@Test
void testSkipIndexCreation() throws IOException, SQLException {
- testWriteTiles(10, true, false, false);
+ testWriteTiles(null, 10, true, false, false);
}
@Test
void testVacuumAnalyze() throws IOException, SQLException {
- testWriteTiles(10, false, true, false);
+ testWriteTiles(null, 10, false, true, false);
+ }
+
+ @Test
+ void testWriteToFile(@TempDir Path tmpDir) throws IOException, SQLException {
+ testWriteTiles(tmpDir.resolve("archive.mbtiles"), 10, false, false, true);
+ }
+
+ @Test
+ void testCustomPragma() throws IOException, SQLException {
+ try (
+ Mbtiles db = Mbtiles.newInMemoryDatabase(Arguments.of(
+ "cache-size", "123",
+ "garbage", "456"
+ ));
+ ) {
+ int result = db.connection().createStatement().executeQuery("pragma cache_size").getInt(1);
+ assertEquals(123, result);
+ }
}
@ParameterizedTest
@@ -121,71 +143,47 @@ class MbtilesTest {
}
@Test
- void testAddMetadata() throws IOException {
- Map expected = new TreeMap<>();
- try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
- var metadata = db.createTablesWithoutIndexes().metadata();
- metadata.setName("name value");
- expected.put("name", "name value");
-
- metadata.setFormat("pbf");
- expected.put("format", "pbf");
-
- metadata.setAttribution("attribution value");
- expected.put("attribution", "attribution value");
-
- metadata.setBoundsAndCenter(GeoUtils.toLatLonBoundsBounds(new Envelope(0.25, 0.75, 0.25, 0.75)));
- expected.put("bounds", "-90,-66.51326,90,66.51326");
- expected.put("center", "0,0,1");
-
- metadata.setDescription("description value");
- expected.put("description", "description value");
-
- metadata.setMinzoom(1);
- expected.put("minzoom", "1");
-
- metadata.setMaxzoom(13);
- expected.put("maxzoom", "13");
-
- metadata.setVersion("1.2.3");
- expected.put("version", "1.2.3");
-
- metadata.setTypeIsBaselayer();
- expected.put("type", "baselayer");
-
- assertEquals(expected, metadata.getAll());
- }
+ void testRoundTripMetadata() throws IOException {
+ roundTripMetadata(new TileArchiveMetadata(
+ "MyName",
+ "MyDescription",
+ "MyAttribution",
+ "MyVersion",
+ "baselayer",
+ TileArchiveMetadata.MVT_FORMAT,
+ new Envelope(1, 2, 3, 4),
+ new CoordinateXY(5, 6),
+ 7d,
+ 8,
+ 9,
+ List.of(new LayerStats.VectorLayer("MyLayer", Map.of())),
+ Map.of("other key", "other value")
+ ));
}
@Test
- void testAddMetadataWorldBounds() throws IOException {
- Map expected = new TreeMap<>();
+ void testRoundTripMinimalMetadata() throws IOException {
+ var empty =
+ new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, Map.of());
+ roundTripMetadata(empty);
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
- var metadata = db.createTablesWithoutIndexes().metadata();
- metadata.setBoundsAndCenter(GeoUtils.WORLD_LAT_LON_BOUNDS);
- expected.put("bounds", "-180,-85.05113,180,85.05113");
- expected.put("center", "0,0,0");
-
- assertEquals(expected, metadata.getAll());
+ db.createTablesWithoutIndexes();
+ assertEquals(empty, db.metadata());
}
}
- @Test
- void testAddMetadataSmallBounds() throws IOException {
- Map expected = new TreeMap<>();
+ private static void roundTripMetadata(TileArchiveMetadata metadata) throws IOException {
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
- var metadata = db.createTablesWithoutIndexes().metadata();
- metadata.setBoundsAndCenter(new Envelope(-73.6632, -69.7598, 41.1274, 43.0185));
- expected.put("bounds", "-73.6632,41.1274,-69.7598,43.0185");
- expected.put("center", "-71.7115,42.07295,7");
-
- assertEquals(expected, metadata.getAll());
+ db.createTablesWithoutIndexes();
+ var metadataTable = db.metadataTable();
+ metadataTable.set(metadata);
+ assertEquals(metadata, metadataTable.get());
}
}
private void testMetadataJson(Mbtiles.MetadataJson object, String expected) throws IOException {
try (Mbtiles db = Mbtiles.newInMemoryDatabase()) {
- var metadata = db.createTablesWithoutIndexes().metadata();
+ var metadata = db.createTablesWithoutIndexes().metadataTable();
metadata.setJson(object);
var actual = metadata.getAll().get("json");
assertSameJson(expected, actual);
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java
index 9324add2..52d6ba56 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java
@@ -45,7 +45,7 @@ class VerifyTest {
@Test
void testValidWithNameAndOneTile() throws IOException {
mbtiles.createTablesWithIndexes();
- mbtiles.metadata().setName("name");
+ mbtiles.metadataTable().setMetadata("name", "name");
try (var writer = mbtiles.newTileWriter()) {
VectorTile tile = new VectorTile();
tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
@@ -62,7 +62,7 @@ class VerifyTest {
@Test
void testInvalidGeometry() throws IOException {
mbtiles.createTablesWithIndexes();
- mbtiles.metadata().setName("name");
+ mbtiles.metadataTable().setMetadata("name", "name");
try (var writer = mbtiles.newTileWriter()) {
VectorTile tile = new VectorTile();
tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
index 122c701e..d5b5f008 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
@@ -15,15 +15,16 @@ import com.onthegomap.planetiler.util.SeekableInMemoryByteChannel;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.Envelope;
class PmtilesTest {
@@ -181,11 +182,12 @@ class PmtilesTest {
var in = WriteablePmtiles.newWriteToMemory(bytes);
var config = PlanetilerConfig.defaults();
- in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
+ var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
+ in.initialize(metadata);
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.empty()));
- in.finish(config);
+ in.finish(metadata);
var reader = new ReadablePmtiles(bytes);
var header = reader.getHeader();
assertEquals(1, header.numAddressedTiles());
@@ -200,28 +202,59 @@ class PmtilesTest {
}
@Test
- void testWritePmtilesToFileWithMetadata(@TempDir Path tempDir) throws IOException {
+ void testRoundtripMetadata() throws IOException {
+ roundTripMetadata(new TileArchiveMetadata(
+ "MyName",
+ "MyDescription",
+ "MyAttribution",
+ "MyVersion",
+ "baselayer",
+ TileArchiveMetadata.MVT_FORMAT,
+ new Envelope(1, 2, 3, 4),
+ new CoordinateXY(5, 6),
+ 7d,
+ 8,
+ 9,
+ List.of(new LayerStats.VectorLayer("MyLayer", Map.of())),
+ Map.of("other key", "other value")
+ ));
+ }
- try (var in = WriteablePmtiles.newWriteToFile(tempDir.resolve("tmp.pmtiles"))) {
- var config = PlanetilerConfig.defaults();
- in.initialize(config,
- new TileArchiveMetadata("MyName", "MyDescription", "MyAttribution", "MyVersion", "baselayer", new HashMap<>()),
- new LayerStats());
+ @Test
+ void testRoundtripMetadataMinimal() throws IOException {
+ roundTripMetadata(
+ new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, Map.of()),
+ new TileArchiveMetadata(null, null, null, null, null, null,
+ new Envelope(-180, 180, -85.0511287, 85.0511287),
+ new CoordinateXY(0, 0),
+ 0d,
+ 0,
+ 15,
+ null,
+ Map.of()
+ )
+ );
+ }
+
+ private static void roundTripMetadata(TileArchiveMetadata metadata) throws IOException {
+ roundTripMetadata(metadata, metadata);
+ }
+
+ private static void roundTripMetadata(TileArchiveMetadata input, TileArchiveMetadata output) throws IOException {
+ try (
+ var channel = new SeekableInMemoryByteChannel(0);
+ var in = WriteablePmtiles.newWriteToMemory(channel)
+ ) {
+ in.initialize(input);
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.empty()));
- in.finish(config);
+ in.finish(input);
+ var reader = new ReadablePmtiles(channel);
+ assertArrayEquals(new byte[]{0xa, 0x2}, reader.getTile(0, 0, 0));
+
+ assertEquals(output, reader.metadata());
}
-
- var reader = new ReadablePmtiles(FileChannel.open(tempDir.resolve("tmp.pmtiles")));
- assertArrayEquals(new byte[]{0xa, 0x2}, reader.getTile(0, 0, 0));
-
- var metadata = reader.getJsonMetadata();
- assertEquals("MyName", metadata.otherMetadata().get("name"));
- assertEquals("MyDescription", metadata.otherMetadata().get("description"));
- assertEquals("MyAttribution", metadata.otherMetadata().get("attribution"));
- assertEquals("MyVersion", metadata.otherMetadata().get("version"));
- assertEquals("baselayer", metadata.otherMetadata().get("type"));
}
@Test
@@ -250,13 +283,14 @@ class PmtilesTest {
var in = WriteablePmtiles.newWriteToMemory(bytes);
var config = PlanetilerConfig.defaults();
- in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
+ var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
+ in.initialize(metadata);
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 2), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
- in.finish(config);
+ in.finish(metadata);
var reader = new ReadablePmtiles(bytes);
var header = reader.getHeader();
assertEquals(3, header.numAddressedTiles());
@@ -276,12 +310,13 @@ class PmtilesTest {
var in = WriteablePmtiles.newWriteToMemory(bytes);
var config = PlanetilerConfig.defaults();
- in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
+ var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
+ in.initialize(metadata);
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
- in.finish(config);
+ in.finish(metadata);
var reader = new ReadablePmtiles(bytes);
var header = reader.getHeader();
assertEquals(2, header.numAddressedTiles());
@@ -301,7 +336,8 @@ class PmtilesTest {
var in = WriteablePmtiles.newWriteToMemory(bytes);
var config = PlanetilerConfig.defaults();
- in.initialize(config, new TileArchiveMetadata(new Profile.NullProfile()), new LayerStats());
+ var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
+ in.initialize(metadata);
var writer = in.newTileWriter();
int ENTRIES = 20000;
@@ -311,7 +347,7 @@ class PmtilesTest {
OptionalLong.empty()));
}
- in.finish(config);
+ in.finish(metadata);
var reader = new ReadablePmtiles(bytes);
var header = reader.getHeader();
assertEquals(ENTRIES, header.numAddressedTiles());
diff --git a/planetiler-custommap/README.md b/planetiler-custommap/README.md
index 7da4d729..f62d2808 100644
--- a/planetiler-custommap/README.md
+++ b/planetiler-custommap/README.md
@@ -151,8 +151,6 @@ cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.proper
- `minzoom` - Minimum tile zoom level to emit
- `maxzoom` - Maximum tile zoom level to emit
- `render_maxzoom` - Maximum rendering zoom level up to
-- `skip_mbtiles_index_creation` - Skip adding index to mbtiles file
-- `optimize_db` - Vacuum analyze mbtiles file after writing
- `force` - Overwriting output file and ignore warnings
- `gzip_temp` - Gzip temporary feature storage (uses more CPU, but less disk space)
- `mmap_temp` - Use memory-mapped IO for temp feature files
@@ -175,7 +173,6 @@ cat planetiler-custommap/planetiler.schema.json | jq -r '.properties.args.proper
maximum zoom level to allow for overzooming
- `simplify_tolerance` - Default value for the tile pixel tolerance to use when simplifying features below the maximum
zoom level
-- `compact_db` - Reduce the DB size by separating and deduping the tile data
- `skip_filled_tiles` - Skip writing tiles containing only polygon fills to the output
- `tile_warning_size_mb` - Maximum size in megabytes of a tile to emit a warning about
diff --git a/planetiler-custommap/planetiler.schema.json b/planetiler-custommap/planetiler.schema.json
index 99a8c541..e8b6f2a6 100644
--- a/planetiler-custommap/planetiler.schema.json
+++ b/planetiler-custommap/planetiler.schema.json
@@ -138,28 +138,6 @@
"render_maxzoom": {
"description": "Maximum rendering zoom level up to"
},
- "skip_mbtiles_index_creation": {
- "description": "Skip adding index to mbtiles file",
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "boolean"
- }
- ]
- },
- "optimize_db": {
- "description": "Vacuum analyze mbtiles file after writing",
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "boolean"
- }
- ]
- },
"force": {
"description": "Overwriting output file and ignore warnings",
"anyOf": [
@@ -294,17 +272,6 @@
"simplify_tolerance": {
"description": "Default value for the tile pixel tolerance to use when simplifying features below the maximum zoom level"
},
- "compact_db": {
- "description": "Reduce the DB size by separating and deduping the tile data",
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "boolean"
- }
- ]
- },
"skip_filled_tiles": {
"description": "Skip writing tiles containing only polygon fills to the output",
"anyOf": [
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java
index 0e0677e2..23283e53 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java
@@ -10,7 +10,7 @@ import java.nio.file.Path;
/**
* Main driver to create maps configured by a YAML file.
- *
+ *
* Parses the config file into a {@link ConfiguredProfile}, loads sources into {@link Planetiler} runner and kicks off
* the map generation process.
*/
@@ -54,7 +54,7 @@ public class ConfiguredMapMain {
configureSource(planetiler, sourcesDir, source);
}
- planetiler.overwriteOutput("mbtiles", Path.of("data", "output.mbtiles")).run();
+ planetiler.overwriteOutput(Path.of("data", "output.mbtiles")).run();
}
private static void configureSource(Planetiler planetiler, Path sourcesDir, Source source) {
diff --git a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
index 9cb6e08e..57ca78c9 100644
--- a/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
+++ b/planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/Contexts.java
@@ -187,8 +187,6 @@ public class Contexts {
argumentValues.put("minzoom", config.minzoom());
argumentValues.put("maxzoom", config.maxzoom());
argumentValues.put("render_maxzoom", config.maxzoomForRendering());
- argumentValues.put("skip_mbtiles_index_creation", config.skipIndexCreation());
- argumentValues.put("optimize_db", config.optimizeDb());
argumentValues.put("force", config.force());
argumentValues.put("gzip_temp", config.gzipTempStorage());
argumentValues.put("mmap_temp", config.mmapTempStorage());
@@ -209,7 +207,6 @@ public class Contexts {
argumentValues.put("min_feature_size", config.minFeatureSizeBelowMaxZoom());
argumentValues.put("simplify_tolerance_at_max_zoom", config.simplifyToleranceAtMaxZoom());
argumentValues.put("simplify_tolerance", config.simplifyToleranceBelowMaxZoom());
- argumentValues.put("compact_db", config.compactDb());
argumentValues.put("skip_filled_tiles", config.skipFilledTiles());
argumentValues.put("tile_warning_size_mb", config.tileWarningSizeBytes());
builtInArgs = Set.copyOf(argumentValues.keySet());
diff --git a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java
index fe179519..d7c8527a 100644
--- a/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java
+++ b/planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredMapTest.java
@@ -47,7 +47,7 @@ class ConfiguredMapTest {
"--tmp=" + tmpDir,
// Override output location
- "--mbtiles=" + dbPath
+ "--output=" + dbPath
);
mbtiles = Mbtiles.newReadOnlyDatabase(dbPath);
}
@@ -59,7 +59,7 @@ class ConfiguredMapTest {
@Test
void testMetadata() {
- Map metadata = mbtiles.metadata().getAll();
+ Map metadata = mbtiles.metadataTable().getAll();
assertEquals("OWG Simple Schema", metadata.get("name"));
assertEquals("0", metadata.get("minzoom"));
assertEquals("14", metadata.get("maxzoom"));
diff --git a/planetiler-examples/README.md b/planetiler-examples/README.md
index 3978b59d..c6e39225 100644
--- a/planetiler-examples/README.md
+++ b/planetiler-examples/README.md
@@ -105,7 +105,7 @@ java -cp target/*-with-deps.jar com.onthegomap.planetiler.examples.MyProfile
Then, to inspect the tiles:
```bash
-tileserver-gl-light --mbtiles data/toilets.mbtiles
+tileserver-gl-light data/toilets.mbtiles
```
Finally, open http://localhost:8080 to see your tiles.
@@ -143,7 +143,7 @@ public void integrationTest(@TempDir Path tmpDir) throws Exception {
MyProfile.main(
"--osm_path=" + TestUtils.pathToResource("monaco-latest.osm.pbf"),
"--tmp=" + tmpDir,
- "--mbtiles=" + mbtilesPath,
+ "--output=" + mbtilesPath,
));
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(mbtilesPath)) {
Map metadata = mbtiles.metadata().getAll();
diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java
index 5fed1516..719b5629 100644
--- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java
+++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/BikeRouteOverlay.java
@@ -22,7 +22,7 @@ import java.util.List;
* then build the examples: {@code mvn clean package}
* then run this example:
* {@code java -cp target/*-with-deps.jar com.onthegomap.planetiler.examples.BikeRouteOverlay osm_path="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}
- * then run the demo tileserver: {@code tileserver-gl-light --mbtiles data/bikeroutes.mbtiles}
+ * then run the demo tileserver: {@code tileserver-gl-light data/bikeroutes.mbtiles}
* and view the output at localhost:8080
*
*/
@@ -175,7 +175,7 @@ public class BikeRouteOverlay implements Profile {
// override this default with osm_path="path/to/data.osm.pbf"
.addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area)
// override this default with mbtiles="path/to/output.mbtiles"
- .overwriteOutput("mbtiles", Path.of("data", "bikeroutes.mbtiles"))
+ .overwriteOutput(Path.of("data", "bikeroutes.mbtiles"))
.run();
}
}
diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java
index 44df7001..4b149034 100644
--- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java
+++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/OsmQaTiles.java
@@ -54,7 +54,7 @@ public class OsmQaTiles implements Profile {
Path.of("data", "sources", area + ".osm.pbf"),
"planet".equalsIgnoreCase(area) ? "aws:latest" : ("geofabrik:" + area)
)
- .overwriteOutput("mbtiles", Path.of("data", "qa.mbtiles"))
+ .overwriteOutput(Path.of("data", "qa.mbtiles"))
.run();
}
diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java
index 35fe50b7..7763a40b 100644
--- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java
+++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlay.java
@@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* then build the examples: {@code mvn clean package}
* then run this example:
* {@code java -cp target/*-fatjar.jar com.onthegomap.planetiler.examples.ToiletsOverlay osm_path="path/to/data.osm.pbf" mbtiles="data/output.mbtiles"}
- * then run the demo tileserver: {@code tileserver-gl-light --mbtiles=data/output.mbtiles}
+ * then run the demo tileserver: {@code tileserver-gl-light data/output.mbtiles}
* and view the output at localhost:8080
*
*/
@@ -103,7 +103,7 @@ public class ToiletsOverlay implements Profile {
// override this default with osm_path="path/to/data.osm.pbf"
.addOsmSource("osm", Path.of("data", "sources", area + ".osm.pbf"), "geofabrik:" + area)
// override this default with mbtiles="path/to/output.mbtiles"
- .overwriteOutput("mbtiles", Path.of("data", "toilets.mbtiles"))
+ .overwriteOutput(Path.of("data", "toilets.mbtiles"))
.run();
}
}
diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java
index 1c5a8e61..801489af 100644
--- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java
+++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java
@@ -4,13 +4,14 @@ import com.onthegomap.planetiler.Planetiler;
import com.onthegomap.planetiler.Profile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileArchiveWriter;
+import com.onthegomap.planetiler.archive.TileArchives;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
import com.onthegomap.planetiler.collection.FeatureGroup;
import com.onthegomap.planetiler.collection.LongLongMap;
import com.onthegomap.planetiler.collection.LongLongMultimap;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.geo.TileOrder;
-import com.onthegomap.planetiler.mbtiles.Mbtiles;
import com.onthegomap.planetiler.reader.osm.OsmInputFile;
import com.onthegomap.planetiler.reader.osm.OsmReader;
import com.onthegomap.planetiler.stats.Stats;
@@ -31,7 +32,7 @@ import org.slf4j.LoggerFactory;
* then build the examples: {@code mvn clean package}
* then run this example:
* {@code java -cp target/*-fatjar.jar com.onthegomap.planetiler.examples.ToiletsOverlayLowLevelApi}
- * then run the demo tileserver: {@code tileserver-gl-light --mbtiles=data/toilets.mbtiles}
+ * then run the demo tileserver: {@code tileserver-gl-light data/toilets.mbtiles}
* and view the output at localhost:8080
*
*/
@@ -57,7 +58,7 @@ public class ToiletsOverlayLowLevelApi {
PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromJvmProperties());
// extract mbtiles metadata from profile
- TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile);
+ TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile, config);
// overwrite output each time
FileUtils.deleteFile(output);
@@ -112,7 +113,7 @@ public class ToiletsOverlayLowLevelApi {
// then process rendered features, grouped by tile, encoding them into binary vector tile format
// and writing to the output mbtiles file.
- try (Mbtiles db = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+ try (WriteableTileArchive db = TileArchives.newWriter(output, config)) {
TileArchiveWriter.writeOutput(featureGroup, db, () -> FileUtils.fileSize(output), tileArchiveMetadata, config,
stats);
} catch (IOException e) {
diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java
index 525c1278..7dbc25ef 100644
--- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java
+++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/BikeRouteOverlayTest.java
@@ -98,10 +98,10 @@ class BikeRouteOverlayTest {
// Override temp dir location
"tmp", tmpDir.toString(),
// Override output location
- "mbtiles", dbPath.toString()
+ "output", dbPath.toString()
));
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
- Map metadata = mbtiles.metadata().getAll();
+ Map metadata = mbtiles.metadataTable().getAll();
assertEquals("Bike Paths Overlay", metadata.get("name"));
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java
index a1b62eaa..65669508 100644
--- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java
+++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/OsmQaTilesTest.java
@@ -93,10 +93,10 @@ class OsmQaTilesTest {
// Override temp dir location
"tmp", tmpDir.toString(),
// Override output location
- "mbtiles", dbPath.toString()
+ "output", dbPath.toString()
));
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
- Map metadata = mbtiles.metadata().getAll();
+ Map metadata = mbtiles.metadataTable().getAll();
assertEquals("osm qa", metadata.get("name"));
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java
index 5b9cc15e..16c87280 100644
--- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java
+++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApiTest.java
@@ -27,7 +27,7 @@ class ToiletsOverlayLowLevelApiTest {
dbPath
);
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
- Map metadata = mbtiles.metadata().getAll();
+ Map metadata = mbtiles.metadata().toMap();
assertEquals("Toilets Overlay", metadata.get("name"));
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
diff --git a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java
index 51c69997..691cbf99 100644
--- a/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java
+++ b/planetiler-examples/src/test/java/com/onthegomap/planetiler/examples/ToiletsProfileTest.java
@@ -58,10 +58,10 @@ class ToiletsProfileTest {
// Override temp dir location
"tmp", tmpDir.toString(),
// Override output location
- "mbtiles", dbPath.toString()
+ "output", dbPath.toString()
));
try (Mbtiles mbtiles = Mbtiles.newReadOnlyDatabase(dbPath)) {
- Map metadata = mbtiles.metadata().getAll();
+ Map metadata = mbtiles.metadata().toMap();
assertEquals("Toilets Overlay", metadata.get("name"));
assertContains("openstreetmap.org/copyright", metadata.get("attribution"));
diff --git a/planetiler-openmaptiles b/planetiler-openmaptiles
index 62de454c..292611de 160000
--- a/planetiler-openmaptiles
+++ b/planetiler-openmaptiles
@@ -1 +1 @@
-Subproject commit 62de454cf769e5bf2832f32d6b1f707860d442cf
+Subproject commit 292611de84b69f0ddbd4b603ec1d0f5d13257c33
diff --git a/quickstart.sh b/quickstart.sh
index de0911f9..82fe78e2 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -102,9 +102,10 @@ if [ "$DRY_RUN" == "true" ]; then
fi
function run() {
- echo "$ $*"
+ command="${*//&/\&}"
+ echo "$ $command"
if [ "$DRY_RUN" != "true" ]; then
- eval "$*"
+ eval "$command"
fi
}
diff --git a/scripts/test-release.sh b/scripts/test-release.sh
index a8aab2bf..179c75b7 100755
--- a/scripts/test-release.sh
+++ b/scripts/test-release.sh
@@ -15,23 +15,23 @@ fi
echo "Test java build"
echo "::group::OpenMapTiles monaco (java)"
rm -f data/out.mbtiles
-java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --mbtiles=data/out.mbtiles
+java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/out.mbtiles
./scripts/check-monaco.sh data/out.mbtiles
echo "::endgroup::"
echo "::group::Example (java)"
rm -f data/out.mbtiles
-java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --mbtiles=data/out.mbtiles
+java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/out.mbtiles
./scripts/check-mbtiles.sh data/out.mbtiles
echo "::endgroup::"
echo "::endgroup::"
echo "::group::OpenMapTiles monaco (docker)"
rm -f data/out.mbtiles
-docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --mbtiles=data/out.mbtiles
+docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/out.mbtiles
./scripts/check-monaco.sh data/out.mbtiles
echo "::endgroup::"
echo "::group::Example (docker)"
rm -f data/out.mbtiles
-docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --mbtiles=data/out.mbtiles
+docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/out.mbtiles
./scripts/check-mbtiles.sh data/out.mbtiles
echo "::endgroup::"