diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 57b8ffa8..f10f4fec 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -8,7 +8,7 @@ Planetiler builds a map in 3 phases: intermediate files on disk 2. [Sort Features](#2-sort-features) by tile ID 3. [Emit Vector Tiles](#3-emit-vector-tiles) by iterating through sorted features to group by tile ID, encoding, and - writing to the output MBTiles file + writing to the output tile archive User-defined [profiles](#profiles) customize the behavior of each part of this pipeline. @@ -96,7 +96,7 @@ of the intermediate features using a worker thread per core: ## 3) Emit Vector Tiles -[MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) is the main driver. +[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver. First, a single-threaded reader reads features from disk: - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits @@ -104,7 +104,7 @@ First, a single-threaded reader reads features from disk: - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit to limit point label density -- Then [MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) groups tiles +- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the writer thread waits for finished tiles in order) @@ -116,9 +116,9 @@ Then, process tile batches in worker threads (default 1 per core): - gzip each encoded tile - Pass the batch of encoded vector tiles to the writer thread -Finally, a single-threaded writer writes encoded vector tiles to the output MBTiles file: +Finally, a single-threaded writer writes encoded vector tiles to the output archive format: -- Create the largest prepared statement supported by SQLite (999 parameters) +- For MBTiles, create the largest prepared statement supported by SQLite (999 parameters) - Iterate through finished vector tile batches until the prepared statement is full, flush to disk, then repeat - Then flush any remaining tiles at the end diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java index 30d0ed8e..b53777b1 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java @@ -5,8 +5,8 @@ import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.mbtiles.Mbtiles.BatchedTileWriter; -import com.onthegomap.planetiler.mbtiles.TileEncodingResult; +import com.onthegomap.planetiler.writer.TileArchive.TileWriter; +import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -74,7 +74,7 @@ public class BenchmarkMbtilesWriter { mbtiles.createTablesWithIndexes(); } - try (var writer = mbtiles.newBatchedTileWriter()) { + try (var writer = mbtiles.newTileWriter()) { Stopwatch sw = Stopwatch.createStarted(); writeTiles(writer, tilesToWrite, distinctTilesInPercent, distinctTileData, dupeTileData, dupeSpreadInPercent); sw.stop(); @@ -92,7 +92,7 @@ public class BenchmarkMbtilesWriter { } - private static void writeTiles(BatchedTileWriter writer, int tilesToWrite, int distinctTilesInPercent, + private static void writeTiles(TileWriter writer, int tilesToWrite, int distinctTilesInPercent, byte[] distinctTileData, byte[] dupeTileData, int dupeSpreadInPercent) { int dupesToWrite = (int) Math.round(tilesToWrite * (100 - distinctTilesInPercent) / 100.0); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java index 1d6bf9c8..46b3b8ae 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java @@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory; /** * A collection of utilities for merging features with the same attributes in a rendered tile from * {@link Profile#postProcessLayerFeatures(String, int, List)} immediately before a tile is written to the output - * mbtiles file. + * archive. *

* Unlike postgis-based solutions that have a full view of all features after they are loaded into the database, the * planetiler engine only sees a single input feature at a time while processing source features, then only has diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index 0ba01557..34e6fc0b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -4,9 +4,8 @@ import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.MbtilesMetadata; import com.onthegomap.planetiler.config.PlanetilerConfig; -import com.onthegomap.planetiler.mbtiles.MbtilesWriter; +import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.reader.GeoPackageReader; import com.onthegomap.planetiler.reader.NaturalEarthReader; import com.onthegomap.planetiler.reader.ShapefileReader; @@ -27,6 +26,10 @@ import com.onthegomap.planetiler.util.ResourceUsage; import com.onthegomap.planetiler.util.Translations; import com.onthegomap.planetiler.util.Wikidata; import com.onthegomap.planetiler.worker.RunnableThatThrows; +import com.onthegomap.planetiler.writer.TileArchive; +import com.onthegomap.planetiler.writer.TileArchiveMetadata; +import com.onthegomap.planetiler.writer.TileArchiveWriter; +import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; @@ -94,7 +97,7 @@ public class Planetiler { private boolean useWikidata = false; private boolean onlyFetchWikidata = false; private boolean fetchWikidata = false; - private MbtilesMetadata mbtilesMetadata; + private TileArchiveMetadata tileArchiveMetadata; private Planetiler(Arguments arguments) { this.arguments = arguments; @@ -176,9 +179,10 @@ public class Planetiler { ), ifSourceUsed(name, () -> { var header = osmInputFile.getHeader(); - mbtilesMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant()); - mbtilesMetadata.set("planetiler:" + name + ":osmosisreplicationseq", header.osmosisReplicationSequenceNumber()); - mbtilesMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl()); + tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant()); + tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationseq", + header.osmosisReplicationSequenceNumber()); + tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl()); try ( var nodeLocations = LongLongMap.from(config.nodeMapType(), config.nodeMapStorage(), nodeDbPath, config.nodeMapMadvise()); @@ -489,30 +493,29 @@ public class Planetiler { } /** - * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Fails if the file already exists. + * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists. *

- * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments. + * To override the location of the file, set {@code argument=newpath} in the arguments. * * @param argument the argument key to check for an override to {@code fallback} * @param fallback the fallback value if {@code argument} is not set in arguments * @return this runner instance for chaining - * @see MbtilesWriter + * @see TileArchiveWriter */ public Planetiler setOutput(String argument, Path fallback) { - this.output = arguments.file(argument, "mbtiles output file", fallback); + this.output = arguments.file(argument, "output tile archive", fallback); return this; } /** - * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Overwrites file if it already - * exists. + * Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists. *

- * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments. + * To override the location of the file, set {@code argument=newpath} in the arguments. * * @param argument the argument key to check for an override to {@code fallback} * @param fallback the fallback value if {@code argument} is not set in arguments * @return this runner instance for chaining - * @see MbtilesWriter + * @see TileArchiveWriter */ public Planetiler overwriteOutput(String argument, Path fallback) { this.overwrite = true; @@ -521,7 +524,7 @@ public class Planetiler { /** * Reads all elements from all sourced that have been added, generates map features according to the profile, and - * writes the rendered tiles to the output mbtiles file. + * writes the rendered tiles to the output archive. * * @throws IllegalArgumentException if expected inputs have not been provided * @throws Exception if an error occurs while processing @@ -550,7 +553,7 @@ public class Planetiler { throw new IllegalArgumentException("Can only run once"); } ran = true; - mbtilesMetadata = new MbtilesMetadata(profile, config.arguments()); + tileArchiveMetadata = new TileArchiveMetadata(profile, config.arguments()); if (arguments.getBoolean("help", "show arguments then exit", false)) { System.exit(0); @@ -579,7 +582,7 @@ public class Planetiler { } } LOGGER.info(" sort: Sort rendered features by tile ID"); - LOGGER.info(" mbtiles: Encode each tile and write to {}", output); + LOGGER.info(" archive: Encode each tile and write to {}", output); } // in case any temp files are left from a previous run... @@ -616,7 +619,7 @@ public class Planetiler { stats.monitorFile("nodes", nodeDbPath); stats.monitorFile("features", featureDbPath); stats.monitorFile("multipolygons", multipolygonPath); - stats.monitorFile("mbtiles", output); + stats.monitorFile("archive", output); for (Stage stage : stages) { stage.task.run(); @@ -633,7 +636,13 @@ public class Planetiler { featureGroup.prepare(); - MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats); + try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + TileArchiveWriter.writeOutput(featureGroup, archive, () -> FileUtils.fileSize(output), tileArchiveMetadata, + config, + stats); + } catch (IOException e) { + throw new IllegalStateException("Unable to write to " + output, e); + } overallTimer.stop(); LOGGER.info("FINISHED!"); @@ -659,7 +668,7 @@ public class Planetiler { readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage"); writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage"); // output only needed during write phase - writePhase.addDisk(output, outputSize, "mbtiles output"); + writePhase.addDisk(output, outputSize, "archive output"); // if the user opts to remove an input source after reading to free up additional space for the output... for (var input : inputPaths) { if (input.freeAfterReading()) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java index 2a30a683..8bb2977b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java @@ -31,7 +31,7 @@ import java.util.function.Consumer; * (i.e. one per layer) and forwarding each element/event to the handlers that care about it. */ public interface Profile { - // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and MbtilesMetadata) can use too + // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and TileArchiveMetadata) can use too /** * Allows profile to extract any information it needs from a {@link OsmElement.Node} during the first pass through OSM diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index e0fbae09..4557a6c6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -529,7 +529,7 @@ public class VectorTile { *

* Deduplication code is aiming for a balance between filtering-out all duplicates and not spending too much CPU on * hash calculations: calculating hashes for all tiles costs too much CPU, not calculating hashes at all means - * generating mbtiles which are too big. This method is responsible for achieving that balance. + * generating archives which are too big. This method is responsible for achieving that balance. *

* Current understanding is, that for the whole planet, there are 267m total tiles and 38m unique tiles. The * {@link #containsOnlyFillsOrEdges()} heuristic catches >99.9% of repeated tiles and cuts down the number of tile @@ -561,7 +561,7 @@ public class VectorTile { * To encode extra precision in intermediate feature geometries, the geometry contained in {@code commands} is scaled * to a tile extent of {@code EXTENT * 2^scale}, so when the {@code scale == 0} the extent is {@link #EXTENT} and when * {@code scale == 2} the extent is 4x{@link #EXTENT}. Geometries must be scaled back to 0 using {@link #unscale()} - * before outputting to mbtiles. + * before outputting to the archive. */ public record VectorGeometry(int[] commands, GeometryType geomType, int scale) { @@ -627,7 +627,7 @@ public class VectorTile { return decodeCommands(geomType, commands, scale); } - /** Returns this encoded geometry, scaled back to 0, so it is safe to emit to mbtiles output. */ + /** Returns this encoded geometry, scaled back to 0, so it is safe to emit to archive output. */ public VectorGeometry unscale() { return scale == 0 ? this : new VectorGeometry(VectorTile.unscale(commands, scale, geomType), geomType, 0); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index eb80dd9c..41b2ae66 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -7,10 +7,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.util.Format; -import java.io.Closeable; +import com.onthegomap.planetiler.util.LayerStats; +import com.onthegomap.planetiler.writer.TileArchive; +import com.onthegomap.planetiler.writer.TileArchiveMetadata; +import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.nio.file.Path; import java.sql.Connection; @@ -46,7 +50,7 @@ import org.sqlite.SQLiteConfig; * * @see MBTiles Specification */ -public final class Mbtiles implements Closeable { +public final class Mbtiles implements TileArchive { // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt private static final int MBTILES_APPLICATION_ID = 0x4d504258; @@ -145,6 +149,42 @@ public final class Mbtiles implements Closeable { } } + @Override + public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) { + if (config.skipIndexCreation()) { + createTablesWithoutIndexes(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Skipping index creation. Add later by executing: {}", + String.join(" ; ", getManualIndexCreationStatements())); + } + } else { + createTablesWithIndexes(); + } + + var metadata = metadata() + .setName(tileArchiveMetadata.name()) + .setFormat("pbf") + .setDescription(tileArchiveMetadata.description()) + .setAttribution(tileArchiveMetadata.attribution()) + .setVersion(tileArchiveMetadata.version()) + .setType(tileArchiveMetadata.type()) + .setBoundsAndCenter(config.bounds().latLon()) + .setMinzoom(config.minzoom()) + .setMaxzoom(config.maxzoom()) + .setJson(layerStats.getTileStats()); + + for (var entry : tileArchiveMetadata.planetilerSpecific().entrySet()) { + metadata.setMetadata(entry.getKey(), entry.getValue()); + } + } + + @Override + public void finish(PlanetilerConfig config) { + if (config.optimizeDb()) { + vacuumAnalyze(); + } + } + @Override public void close() throws IOException { try { @@ -281,7 +321,7 @@ public final class Mbtiles implements Closeable { } /** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */ - public BatchedTileWriter newBatchedTileWriter() { + public TileArchive.TileWriter newTileWriter() { if (compactDb) { return new BatchedCompactTileWriter(); } else { @@ -289,6 +329,11 @@ public final class Mbtiles implements Closeable { } } + // TODO: exists for compatibility purposes + public TileArchive.TileWriter newBatchedTileWriter() { + return newTileWriter(); + } + /** Returns the contents of the metadata table. */ public Metadata metadata() { return new Metadata(); @@ -659,20 +704,7 @@ public final class Mbtiles implements Closeable { } } - - /** - * A high-throughput writer that accepts new tiles and queues up the writes to execute them in fewer large-batches. - */ - public interface BatchedTileWriter extends AutoCloseable { - void write(TileEncodingResult encodingResult); - - @Override - void close(); - - default void printStats() {} - } - - private class BatchedNonCompactTileWriter implements BatchedTileWriter { + private class BatchedNonCompactTileWriter implements TileWriter { private final BatchedTileTableWriter tableWriter = new BatchedTileTableWriter(); @@ -681,6 +713,12 @@ public final class Mbtiles implements Closeable { tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData())); } + // TODO: exists for compatibility purposes + @Override + public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { + tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData())); + } + @Override public void close() { tableWriter.close(); @@ -688,7 +726,7 @@ public final class Mbtiles implements Closeable { } - private class BatchedCompactTileWriter implements BatchedTileWriter { + private class BatchedCompactTileWriter implements TileWriter { private final BatchedTileShallowTableWriter batchedTileShallowTableWriter = new BatchedTileShallowTableWriter(); private final BatchedTileDataTableWriter batchedTileDataTableWriter = new BatchedTileDataTableWriter(); @@ -722,6 +760,12 @@ public final class Mbtiles implements Closeable { batchedTileShallowTableWriter.write(new TileShallowEntry(encodingResult.coord(), tileDataId)); } + // TODO: exists for compatibility purposes + @Override + public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { + write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash())); + } + @Override public void close() { batchedTileShallowTableWriter.close(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/TileEncodingResult.java index a8504f2e..54bdd1f7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/TileEncodingResult.java @@ -5,6 +5,7 @@ import java.util.Arrays; import java.util.Objects; import java.util.OptionalLong; +// TODO: exists for compatibility reasons public record TileEncodingResult( TileCoord coord, byte[] tileData, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java index d0a07685..189ccb8c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java @@ -239,7 +239,7 @@ public class FeatureRenderer implements Consumer, Clos // Store lines with extra precision (2^scale) in intermediate feature storage so that // rounding does not introduce artificial endpoint intersections and confuse line merge // post-processing. Features need to be "unscaled" in FeatureGroup after line merging, - // and before emitting to output mbtiles. + // and before emitting to the output archive. scale = Math.max(config.maxzoom(), 14) - zoom; // need 14 bits to represent tile coordinates (4096 * 2 for buffer * 2 for zigzag encoding) // so cap the scale factor to avoid overflowing 32-bit integer space diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java index 1c38f66e..c798db87 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java @@ -152,7 +152,7 @@ class PrometheusStats implements Stats { } private final Histogram tilesWrittenBytes = Histogram - .build(BASE + "mbtiles_tile_written_bytes", "Written tile sizes by zoom level") + .build(BASE + "archive_tile_written_bytes", "Written tile sizes by zoom level") .buckets(1_000, 10_000, 100_000, 500_000) .labelNames("zoom") .register(registry); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java index 3ac15efc..75e55405 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java @@ -99,7 +99,7 @@ public interface Stats extends AutoCloseable { /** Records that an input element was processed and emitted some output features in {@code layer}. */ void processedElement(String elemType, String layer); - /** Records that a tile has been written to the mbtiles output where compressed size is {@code bytes}. */ + /** Records that a tile has been written to the archive output where compressed size is {@code bytes}. */ void wroteTile(int zoom, int bytes); /** Returns the timers for all stages started with {@link #startStage(String)}. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java new file mode 100644 index 00000000..d5cca46f --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java @@ -0,0 +1,44 @@ +package com.onthegomap.planetiler.writer; + +import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.util.LayerStats; +import java.io.Closeable; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive + * format. + */ +@NotThreadSafe +public interface TileArchive extends Closeable { + interface TileWriter extends Closeable { + void write(TileEncodingResult encodingResult); + + // TODO: exists for compatibility reasons + void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult); + + @Override + void close(); + + default void printStats() {} + } + + + /** + * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any + * required state here. + */ + void initialize(PlanetilerConfig config, TileArchiveMetadata metadata, LayerStats layerStats); + + /** + * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might + * depend on {@link PlanetilerConfig}. + */ + TileWriter newTileWriter(); + + /** + * Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on + * disk. + */ + void finish(PlanetilerConfig config); +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/MbtilesMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java similarity index 67% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/config/MbtilesMetadata.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java index 56fde04b..271ce5a9 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/MbtilesMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java @@ -1,13 +1,13 @@ -package com.onthegomap.planetiler.config; +package com.onthegomap.planetiler.writer; import com.onthegomap.planetiler.Profile; -import com.onthegomap.planetiler.mbtiles.MbtilesWriter; +import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.util.BuildInfo; import java.util.LinkedHashMap; import java.util.Map; -/** Controls information that {@link MbtilesWriter} will write to the mbtiles metadata table. */ -public record MbtilesMetadata( +/** Controls information that {@link TileArchiveWriter} will write to the archive metadata. */ +public record TileArchiveMetadata( String name, String description, String attribution, @@ -16,7 +16,7 @@ public record MbtilesMetadata( Map planetilerSpecific ) { - public MbtilesMetadata(Profile profile) { + public TileArchiveMetadata(Profile profile) { this( profile.name(), profile.description(), @@ -27,13 +27,13 @@ public record MbtilesMetadata( ); } - public MbtilesMetadata(Profile profile, Arguments args) { + public TileArchiveMetadata(Profile profile, Arguments args) { this( - args.getString("mbtiles_name", "'name' attribute for mbtiles metadata", profile.name()), - args.getString("mbtiles_description", "'description' attribute for mbtiles metadata", profile.description()), - args.getString("mbtiles_attribution", "'attribution' attribute for mbtiles metadata", profile.attribution()), - args.getString("mbtiles_version", "'version' attribute for mbtiles metadata", profile.version()), - args.getString("mbtiles_type", "'type' attribute for mbtiles metadata", + args.getString("mbtiles_name", "'name' attribute for tileset metadata", profile.name()), + args.getString("mbtiles_description", "'description' attribute for tileset metadata", profile.description()), + args.getString("mbtiles_attribution", "'attribution' attribute for tileset metadata", profile.attribution()), + args.getString("mbtiles_version", "'version' attribute for tileset metadata", profile.version()), + args.getString("mbtiles_type", "'type' attribute for tileset metadata", profile.isOverlay() ? "overlay" : "baselayer"), mapWithBuildInfo() ); @@ -56,7 +56,7 @@ public record MbtilesMetadata( return result; } - public MbtilesMetadata set(String key, Object value) { + public TileArchiveMetadata set(String key, Object value) { if (key != null && value != null) { planetilerSpecific.put(key, value.toString()); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java similarity index 82% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java index 0f523937..1939e82b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java @@ -1,11 +1,10 @@ -package com.onthegomap.planetiler.mbtiles; +package com.onthegomap.planetiler.writer; import static com.onthegomap.planetiler.util.Gzip.gzip; import static com.onthegomap.planetiler.worker.Worker.joinFutures; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.collection.FeatureGroup; -import com.onthegomap.planetiler.config.MbtilesMetadata; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.stats.Counter; @@ -14,7 +13,6 @@ import com.onthegomap.planetiler.stats.ProgressLoggers; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.stats.Timer; import com.onthegomap.planetiler.util.DiskBacked; -import com.onthegomap.planetiler.util.FileUtils; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.Hashing; import com.onthegomap.planetiler.util.LayerStats; @@ -22,7 +20,6 @@ import com.onthegomap.planetiler.worker.WorkQueue; import com.onthegomap.planetiler.worker.Worker; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -42,17 +39,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to an - * {@link Mbtiles} file. + * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a + * {@link TileArchive}. */ -public class MbtilesWriter { +public class TileArchiveWriter { - private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TileArchiveWriter.class); private static final long MAX_FEATURES_PER_BATCH = 10_000; private static final long MAX_TILES_PER_BATCH = 1_000; private final Counter.Readable featuresProcessed; private final Counter memoizedTiles; - private final Mbtiles db; + private final TileArchive archive; private final PlanetilerConfig config; private final Stats stats; private final LayerStats layerStats; @@ -61,14 +58,15 @@ public class MbtilesWriter { private final LongAccumulator[] maxTileSizesByZoom; private final Iterable inputTiles; private final AtomicReference lastTileWritten = new AtomicReference<>(); - private final MbtilesMetadata mbtilesMetadata; + private final TileArchiveMetadata tileArchiveMetadata; - private MbtilesWriter(Iterable inputTiles, Mbtiles db, PlanetilerConfig config, - MbtilesMetadata mbtilesMetadata, Stats stats, LayerStats layerStats) { + private TileArchiveWriter(Iterable inputTiles, TileArchive archive, + PlanetilerConfig config, + TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) { this.inputTiles = inputTiles; - this.db = db; + this.archive = archive; this.config = config; - this.mbtilesMetadata = mbtilesMetadata; + this.tileArchiveMetadata = tileArchiveMetadata; this.stats = stats; this.layerStats = layerStats; tilesByZoom = IntStream.rangeClosed(0, config.maxzoom()) @@ -80,29 +78,19 @@ public class MbtilesWriter { maxTileSizesByZoom = IntStream.rangeClosed(0, config.maxzoom()) .mapToObj(i -> new LongAccumulator(Long::max, 0)) .toArray(LongAccumulator[]::new); - memoizedTiles = stats.longCounter("mbtiles_memoized_tiles"); - featuresProcessed = stats.longCounter("mbtiles_features_processed"); + memoizedTiles = stats.longCounter("archive_memoized_tiles"); + featuresProcessed = stats.longCounter("archive_features_processed"); Map countsByZoom = new LinkedHashMap<>(); for (int zoom = config.minzoom(); zoom <= config.maxzoom(); zoom++) { countsByZoom.put(Integer.toString(zoom), tilesByZoom[zoom]); } - stats.counter("mbtiles_tiles_written", "zoom", () -> countsByZoom); - } - - /** Reads all {@code features}, encodes them in parallel, and writes to {@code outputPath}. */ - public static void writeOutput(FeatureGroup features, Path outputPath, MbtilesMetadata mbtilesMetadata, - PlanetilerConfig config, Stats stats) { - try (Mbtiles output = Mbtiles.newWriteToFileDatabase(outputPath, config.compactDb())) { - writeOutput(features, output, () -> FileUtils.fileSize(outputPath), mbtilesMetadata, config, stats); - } catch (IOException e) { - throw new IllegalStateException("Unable to write to " + outputPath, e); - } + stats.counter("archive_tiles_written", "zoom", () -> countsByZoom); } /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */ - public static void writeOutput(FeatureGroup features, Mbtiles output, DiskBacked fileSize, - MbtilesMetadata mbtilesMetadata, PlanetilerConfig config, Stats stats) { - var timer = stats.startStage("mbtiles"); + public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize, + TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) { + var timer = stats.startStage("archive"); int readThreads = config.featureReadThreads(); int threads = config.threads(); @@ -123,10 +111,10 @@ public class MbtilesWriter { readWorker = reader.readWorker(); } - MbtilesWriter writer = new MbtilesWriter(inputTiles, output, config, mbtilesMetadata, stats, + TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats, features.layerStats()); - var pipeline = WorkerPipeline.start("mbtiles", stats); + var pipeline = WorkerPipeline.start("archive", stats); // a larger tile queue size helps keep cores busy, but needs a lot of RAM // 5k works fine with 100GB of RAM, so adjust the queue size down from there @@ -143,7 +131,7 @@ public class MbtilesWriter { * waits on them to be encoded in the order they were received, and the encoder processes them in parallel. * One batch might take a long time to process, so make the queues very big to avoid idle encoding CPUs. */ - WorkQueue writerQueue = new WorkQueue<>("mbtiles_writer_queue", queueSize, 1, stats); + WorkQueue writerQueue = new WorkQueue<>("archive_writer_queue", queueSize, 1, stats); encodeBranch = pipeline .fromGenerator(secondStageName, next -> { var writerEnqueuer = writerQueue.threadLocalWriter(); @@ -317,36 +305,12 @@ public class MbtilesWriter { private void tileWriter(Iterable tileBatches) throws ExecutionException, InterruptedException { - if (config.skipIndexCreation()) { - db.createTablesWithoutIndexes(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Skipping index creation. Add later by executing: {}", - String.join(" ; ", db.getManualIndexCreationStatements())); - } - } else { - db.createTablesWithIndexes(); - } - - var metadata = db.metadata() - .setName(mbtilesMetadata.name()) - .setFormat("pbf") - .setDescription(mbtilesMetadata.description()) - .setAttribution(mbtilesMetadata.attribution()) - .setVersion(mbtilesMetadata.version()) - .setType(mbtilesMetadata.type()) - .setBoundsAndCenter(config.bounds().latLon()) - .setMinzoom(config.minzoom()) - .setMaxzoom(config.maxzoom()) - .setJson(layerStats.getTileStats()); - - for (var entry : mbtilesMetadata.planetilerSpecific().entrySet()) { - metadata.setMetadata(entry.getKey(), entry.getValue()); - } + archive.initialize(config, tileArchiveMetadata, layerStats); TileCoord lastTile = null; Timer time = null; int currentZ = Integer.MIN_VALUE; - try (var batchedTileWriter = db.newBatchedTileWriter()) { + try (var tileWriter = archive.newTileWriter()) { for (TileBatch batch : tileBatches) { Queue encodedTiles = batch.out.get(); TileEncodingResult encodedTile; @@ -365,23 +329,22 @@ public class MbtilesWriter { time = Timer.start(); currentZ = z; } - batchedTileWriter.write(encodedTile); + tileWriter.write(encodedTile); stats.wroteTile(z, encodedTile.tileData() == null ? 0 : encodedTile.tileData().length); tilesByZoom[z].inc(); } lastTileWritten.set(lastTile); } - batchedTileWriter.printStats(); + tileWriter.printStats(); } if (time != null) { LOGGER.info("Finished z{} in {}", currentZ, time.stop()); } - if (config.optimizeDb()) { - db.vacuumAnalyze(); - } + + archive.finish(config); } private void printTileStats() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java new file mode 100644 index 00000000..39b9a08d --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java @@ -0,0 +1,43 @@ +package com.onthegomap.planetiler.writer; + +import com.onthegomap.planetiler.geo.TileCoord; +import java.util.Arrays; +import java.util.Objects; +import java.util.OptionalLong; + +public record TileEncodingResult( + TileCoord coord, + byte[] tileData, + /** will always be empty in non-compact mode and might also be empty in compact mode */ + OptionalLong tileDataHash +) { + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(tileData); + result = prime * result + Objects.hash(coord, tileDataHash); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TileEncodingResult)) { + return false; + } + TileEncodingResult other = (TileEncodingResult) obj; + return Objects.equals(coord, other.coord) && Arrays.equals(tileData, other.tileData) && + Objects.equals(tileDataHash, other.tileDataHash); + } + + @Override + public String toString() { + return "TileEncodingResult [coord=" + coord + ", tileData=" + Arrays.toString(tileData) + ", tileDataHash=" + + tileDataHash + "]"; + } + +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 1681c22f..d858721d 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -7,13 +7,11 @@ import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.MbtilesMetadata; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.mbtiles.MbtilesWriter; import com.onthegomap.planetiler.reader.SimpleFeature; import com.onthegomap.planetiler.reader.SimpleReader; import com.onthegomap.planetiler.reader.SourceFeature; @@ -24,6 +22,8 @@ import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.BuildInfo; +import com.onthegomap.planetiler.writer.TileArchiveMetadata; +import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -141,7 +141,8 @@ class PlanetilerTests { runner.run(featureGroup, profile, config); featureGroup.prepare(); try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.compactDb())) { - MbtilesWriter.writeOutput(featureGroup, db, () -> 0L, new MbtilesMetadata(profile, config.arguments()), config, + TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config.arguments()), + config, stats); var tileMap = TestUtils.getTileMap(db); tileMap.values().forEach(fs -> fs.forEach(f -> f.geometry().validate())); @@ -267,11 +268,11 @@ class PlanetilerTests { void testOverrideMetadata() throws Exception { var results = runWithReaderFeatures( Map.of( - "mbtiles_name", "mbtiles_name", - "mbtiles_description", "mbtiles_description", - "mbtiles_attribution", "mbtiles_attribution", - "mbtiles_version", "mbtiles_version", - "mbtiles_type", "mbtiles_type" + "mbtiles_name", "override_name", + "mbtiles_description", "override_description", + "mbtiles_attribution", "override_attribution", + "mbtiles_version", "override_version", + "mbtiles_type", "override_type" ), List.of(), (sourceFeature, features) -> { @@ -279,11 +280,11 @@ class PlanetilerTests { ); assertEquals(Map.of(), results.tiles); assertSubmap(Map.of( - "name", "mbtiles_name", - "description", "mbtiles_description", - "attribution", "mbtiles_attribution", - "version", "mbtiles_version", - "type", "mbtiles_type" + "name", "override_name", + "description", "override_description", + "attribution", "override_attribution", + "version", "override_version", + "type", "override_type" ), results.metadata); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java index a5f89f1f..2808730a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java @@ -12,11 +12,11 @@ import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.TileCoord; -import com.onthegomap.planetiler.mbtiles.MbtilesWriter; import com.onthegomap.planetiler.render.RenderedFeature; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.CloseableConusmer; import com.onthegomap.planetiler.util.Gzip; +import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -365,10 +365,10 @@ class FeatureGroupTest { put(args1); sorter.sort(); var iter = features.iterator(); - var tileHash0 = MbtilesWriter.generateContentHash( + var tileHash0 = TileArchiveWriter.generateContentHash( Gzip.gzip(iter.next().getVectorTileEncoder().encode()) ); - var tileHash1 = MbtilesWriter.generateContentHash( + var tileHash1 = TileArchiveWriter.generateContentHash( Gzip.gzip(iter.next().getVectorTileEncoder().encode()) ); if (expectSame) { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java index 605c8e0a..208cdc49 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java @@ -7,6 +7,7 @@ import com.google.common.math.IntMath; import com.onthegomap.planetiler.TestUtils; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.math.RoundingMode; import java.sql.SQLException; @@ -45,7 +46,7 @@ class MbtilesTest { assertNull(db.getTile(0, 0, 0)); Set expected = new TreeSet<>(); - try (var writer = db.newBatchedTileWriter()) { + try (var writer = db.newTileWriter()) { for (int i = 0; i < howMany; i++) { var dataHash = i - (i % 2); var dataBase = howMany + dataHash; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java index 843ce778..cd0c607b 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.util.List; import java.util.Map; @@ -45,7 +46,7 @@ class VerifyTest { void testValidWithNameAndOneTile() throws IOException { mbtiles.createTablesWithIndexes(); mbtiles.metadata().setName("name"); - try (var writer = mbtiles.newBatchedTileWriter()) { + try (var writer = mbtiles.newTileWriter()) { VectorTile tile = new VectorTile(); tile.addLayerFeatures("layer", List.of(new VectorTile.Feature( "layer", @@ -62,7 +63,7 @@ class VerifyTest { void testInvalidGeometry() throws IOException { mbtiles.createTablesWithIndexes(); mbtiles.metadata().setName("name"); - try (var writer = mbtiles.newBatchedTileWriter()) { + try (var writer = mbtiles.newTileWriter()) { VectorTile tile = new VectorTile(); tile.addLayerFeatures("layer", List.of(new VectorTile.Feature( "layer", diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stats/PrometheusStatsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stats/PrometheusStatsTest.java index ba2edede..6d9658c1 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stats/PrometheusStatsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stats/PrometheusStatsTest.java @@ -91,8 +91,8 @@ class PrometheusStatsTest { PrometheusStats stats = new PrometheusStats("job"); stats.wroteTile(0, 10); stats.wroteTile(0, 10_000); - assertContainsStat("^planetiler_mbtiles_tile_written_bytes_bucket\\{.*le=\"1000\\..* 1", stats); - assertContainsStat("^planetiler_mbtiles_tile_written_bytes_bucket\\{.*le=\"10000\\..* 2", stats); + assertContainsStat("^planetiler_archive_tile_written_bytes_bucket\\{.*le=\"1000\\..* 1", stats); + assertContainsStat("^planetiler_archive_tile_written_bytes_bucket\\{.*le=\"10000\\..* 2", stats); } @Test diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java index bf2f9890..630abd2e 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java @@ -6,13 +6,14 @@ import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.MbtilesMetadata; import com.onthegomap.planetiler.config.PlanetilerConfig; -import com.onthegomap.planetiler.mbtiles.MbtilesWriter; +import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.reader.osm.OsmInputFile; import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.FileUtils; +import com.onthegomap.planetiler.writer.TileArchiveMetadata; +import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -55,7 +56,7 @@ public class ToiletsOverlayLowLevelApi { PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromJvmProperties()); // extract mbtiles metadata from profile - MbtilesMetadata mbtilesMetadata = new MbtilesMetadata(profile); + TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile); // overwrite output each time FileUtils.deleteFile(output); @@ -109,7 +110,12 @@ public class ToiletsOverlayLowLevelApi { // then process rendered features, grouped by tile, encoding them into binary vector tile format // and writing to the output mbtiles file. - MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats); + try (Mbtiles db = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + TileArchiveWriter.writeOutput(featureGroup, db, () -> FileUtils.fileSize(output), tileArchiveMetadata, config, + stats); + } catch (IOException e) { + throw new IllegalStateException("Unable to write to " + output, e); + } // dump recorded timings at the end stats.printSummary();