diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f10f4fec..ddd7ba9f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -96,7 +96,8 @@ of the intermediate features using a worker thread per core: ## 3) Emit Vector Tiles -[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver. +[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) is the main +driver. First, a single-threaded reader reads features from disk: - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits @@ -104,7 +105,8 @@ First, a single-threaded reader reads features from disk: - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit to limit point label density -- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles +- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) + groups tiles into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the writer thread waits for finished tiles in order) diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java index b53777b1..47379b9f 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java @@ -1,12 +1,12 @@ package com.onthegomap.planetiler.benchmarks; import com.google.common.base.Stopwatch; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive.TileWriter; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.writer.TileArchive.TileWriter; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.File; import java.io.IOException; import java.nio.file.Files; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index cd30e956..718e0c12 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -1,5 +1,8 @@ package com.onthegomap.planetiler; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -26,9 +29,6 @@ import com.onthegomap.planetiler.util.ResourceUsage; import com.onthegomap.planetiler.util.Translations; import com.onthegomap.planetiler.util.Wikidata; import com.onthegomap.planetiler.worker.RunnableThatThrows; -import com.onthegomap.planetiler.writer.TileArchive; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -658,7 +658,7 @@ public class Planetiler { bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats)); } - try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { featureGroup = FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats); stats.monitorFile("nodes", nodeDbPath); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java new file mode 100644 index 00000000..2bc9960c --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java @@ -0,0 +1,44 @@ +package com.onthegomap.planetiler.archive; + +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.Closeable; + +/** + * Read API for on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive + * format. + *
+ * See {@link WriteableTileArchive} for the write API. + */ +public interface ReadableTileArchive extends Closeable { + + /** Returns the raw tile data associated with the tile at {@code coord}. */ + default byte[] getTile(TileCoord coord) { + return getTile(coord.x(), coord.y(), coord.z()); + } + + /** Returns the raw tile data associated with the tile at coordinate {@code x, y, z}. */ + byte[] getTile(int x, int y, int z); + + /** + * Returns an iterator over the coordinates of tiles in this archive. + *
+ * The order should respect {@link WriteableTileArchive#tileOrder()} of the corresponding writer. + *
+ * Clients should be sure to close the iterator after iterating through it, for example: + * + *
+ * {@code + * try (var iter = archive.getAllTileCoords()) { + * while (iter.hasNext()) { + * var coord = iter.next(); + * ... + * } + * } + * } + *+ */ + CloseableIterator
+ * See {@link ReadableTileArchive} for the read API.
*/
@NotThreadSafe
-public interface TileArchive extends Closeable {
+public interface WriteableTileArchive extends Closeable {
+
interface TileWriter extends Closeable {
+
void write(TileEncodingResult encodingResult);
// TODO: exists for compatibility reasons
- void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult);
+ default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
+ write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
+ }
@Override
void close();
@@ -47,4 +53,6 @@ public interface TileArchive extends Closeable {
* disk.
*/
void finish(PlanetilerConfig config);
+
+ // TODO update archive metadata
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java
index be34388f..70658d27 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java
@@ -1,13 +1,14 @@
package com.onthegomap.planetiler.geo;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
import java.util.function.IntFunction;
import java.util.function.ToDoubleBiFunction;
import java.util.function.ToIntFunction;
/**
* Controls the sort order of {@link com.onthegomap.planetiler.collection.FeatureGroup}, which determines the ordering
- * of {@link com.onthegomap.planetiler.writer.TileEncodingResult}s when written to
- * {@link com.onthegomap.planetiler.writer.TileArchive.TileWriter}.
+ * of {@link com.onthegomap.planetiler.archive.TileEncodingResult}s when written to
+ * {@link WriteableTileArchive.TileWriter}.
*/
public enum TileOrder {
TMS(TileCoord::encoded, TileCoord::decode, TileCoord::progressOnLevel),
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
index 5a314922..a2cc353c 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
@@ -7,15 +7,18 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.onthegomap.planetiler.archive.ReadableTileArchive;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.geo.TileOrder;
+import com.onthegomap.planetiler.reader.FileFormatException;
+import com.onthegomap.planetiler.util.CloseableIterator;
import com.onthegomap.planetiler.util.Format;
import com.onthegomap.planetiler.util.LayerStats;
-import com.onthegomap.planetiler.writer.TileArchive;
-import com.onthegomap.planetiler.writer.TileArchiveMetadata;
-import com.onthegomap.planetiler.writer.TileEncodingResult;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Connection;
@@ -31,6 +34,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.OptionalLong;
import java.util.TreeMap;
@@ -48,7 +52,7 @@ import org.sqlite.SQLiteConfig;
*
* @see MBTiles Specification
*/
-public final class Mbtiles implements TileArchive {
+public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive {
// https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
private static final int MBTILES_APPLICATION_ID = 0x4d504258;
@@ -88,13 +92,8 @@ public final class Mbtiles implements TileArchive {
}
private final Connection connection;
- private PreparedStatement getTileStatement = null;
private final boolean compactDb;
-
- @Override
- public TileOrder tileOrder() {
- return TileOrder.TMS;
- }
+ private PreparedStatement getTileStatement = null;
private Mbtiles(Connection connection, boolean compactDb) {
this.connection = connection;
@@ -152,6 +151,11 @@ public final class Mbtiles implements TileArchive {
}
}
+ @Override
+ public TileOrder tileOrder() {
+ return TileOrder.TMS;
+ }
+
@Override
public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
if (config.skipIndexCreation()) {
@@ -324,7 +328,7 @@ public final class Mbtiles implements TileArchive {
}
/** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */
- public TileArchive.TileWriter newTileWriter() {
+ public WriteableTileArchive.TileWriter newTileWriter() {
if (compactDb) {
return new BatchedCompactTileWriter();
} else {
@@ -333,7 +337,7 @@ public final class Mbtiles implements TileArchive {
}
// TODO: exists for compatibility purposes
- public TileArchive.TileWriter newBatchedTileWriter() {
+ public WriteableTileArchive.TileWriter newBatchedTileWriter() {
return newTileWriter();
}
@@ -356,10 +360,7 @@ public final class Mbtiles implements TileArchive {
return getTileStatement;
}
- public byte[] getTile(TileCoord coord) {
- return getTile(coord.x(), coord.y(), coord.z());
- }
-
+ @Override
public byte[] getTile(int x, int y, int z) {
try {
PreparedStatement stmt = getTileStatement();
@@ -374,22 +375,9 @@ public final class Mbtiles implements TileArchive {
}
}
- public List
- * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link TileArchive} formats. To
- * minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
+ * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link WriteableTileArchive} formats.
+ * To minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
* thread-local handler that can update stats without contention.
*