From 9a704e773ef7a26340ebd4f251fb36d8fdcf86a0 Mon Sep 17 00:00:00 2001 From: Michael Barry Date: Sun, 5 Feb 2023 14:16:05 -0500 Subject: [PATCH] Convert get all tile coords to iterator (#463) --- ARCHITECTURE.md | 6 +- .../benchmarks/BenchmarkMbtilesWriter.java | 4 +- .../com/onthegomap/planetiler/Planetiler.java | 8 +- .../archive/ReadableTileArchive.java | 44 +++++++ .../TileArchiveMetadata.java | 2 +- .../TileArchiveWriter.java | 54 ++++---- .../TileEncodingResult.java | 2 +- .../WriteableTileArchive.java} | 16 ++- .../onthegomap/planetiler/geo/TileOrder.java | 5 +- .../planetiler/mbtiles/Mbtiles.java | 124 +++++++++++------- .../onthegomap/planetiler/mbtiles/Verify.java | 25 ++-- .../planetiler/util/CloseableIterator.java | 17 +++ .../planetiler/util/LayerStats.java | 6 +- .../planetiler/PlanetilerTests.java | 4 +- .../collection/FeatureGroupTest.java | 2 +- .../planetiler/mbtiles/MbtilesTest.java | 5 +- .../planetiler/mbtiles/VerifyTest.java | 2 +- .../examples/ToiletsOverlayLowLevelApi.java | 4 +- 18 files changed, 219 insertions(+), 111 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer => archive}/TileArchiveMetadata.java (98%) rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer => archive}/TileArchiveWriter.java (91%) rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer => archive}/TileEncodingResult.java (96%) rename planetiler-core/src/main/java/com/onthegomap/planetiler/{writer/TileArchive.java => archive/WriteableTileArchive.java} (70%) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f10f4fec..ddd7ba9f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -96,7 +96,8 @@ of the intermediate features using a worker thread per core: ## 3) Emit Vector Tiles -[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver. +[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) is the main +driver. First, a single-threaded reader reads features from disk: - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits @@ -104,7 +105,8 @@ First, a single-threaded reader reads features from disk: - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the grouping limit to limit point label density -- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles +- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) + groups tiles into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy while the writer thread waits for finished tiles in order) diff --git a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java index b53777b1..47379b9f 100644 --- a/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java +++ b/planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java @@ -1,12 +1,12 @@ package com.onthegomap.planetiler.benchmarks; import com.google.common.base.Stopwatch; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive.TileWriter; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.writer.TileArchive.TileWriter; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.File; import java.io.IOException; import java.nio.file.Files; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index cd30e956..718e0c12 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -1,5 +1,8 @@ package com.onthegomap.planetiler; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -26,9 +29,6 @@ import com.onthegomap.planetiler.util.ResourceUsage; import com.onthegomap.planetiler.util.Translations; import com.onthegomap.planetiler.util.Wikidata; import com.onthegomap.planetiler.worker.RunnableThatThrows; -import com.onthegomap.planetiler.writer.TileArchive; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -658,7 +658,7 @@ public class Planetiler { bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats)); } - try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { + try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) { featureGroup = FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats); stats.monitorFile("nodes", nodeDbPath); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java new file mode 100644 index 00000000..2bc9960c --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java @@ -0,0 +1,44 @@ +package com.onthegomap.planetiler.archive; + +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.Closeable; + +/** + * Read API for on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive + * format. + *

+ * See {@link WriteableTileArchive} for the write API. + */ +public interface ReadableTileArchive extends Closeable { + + /** Returns the raw tile data associated with the tile at {@code coord}. */ + default byte[] getTile(TileCoord coord) { + return getTile(coord.x(), coord.y(), coord.z()); + } + + /** Returns the raw tile data associated with the tile at coordinate {@code x, y, z}. */ + byte[] getTile(int x, int y, int z); + + /** + * Returns an iterator over the coordinates of tiles in this archive. + *

+ * The order should respect {@link WriteableTileArchive#tileOrder()} of the corresponding writer. + *

+ * Clients should be sure to close the iterator after iterating through it, for example: + * + * 

+   * {@code
+   * try (var iter = archive.getAllTileCoords()) {
+   *   while (iter.hasNext()) {
+   *     var coord = iter.next();
+   *     ...
+   *   }
+   * }
+   * }
+   *
+ */ + CloseableIterator getAllTileCoords(); + + // TODO access archive metadata +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java similarity index 98% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java index 271ce5a9..053dc3c1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.config.Arguments; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java similarity index 91% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index fad9f06b..ad95e1c1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import static com.onthegomap.planetiler.util.Gzip.gzip; import static com.onthegomap.planetiler.worker.Worker.joinFutures; @@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory; /** * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a - * {@link TileArchive}. + * {@link WriteableTileArchive}. */ public class TileArchiveWriter { @@ -49,7 +49,7 @@ public class TileArchiveWriter { private static final long MAX_TILES_PER_BATCH = 1_000; private final Counter.Readable featuresProcessed; private final Counter memoizedTiles; - private final TileArchive archive; + private final WriteableTileArchive archive; private final PlanetilerConfig config; private final Stats stats; private final LayerStats layerStats; @@ -60,7 +60,7 @@ public class TileArchiveWriter { private final AtomicReference lastTileWritten = new AtomicReference<>(); private final TileArchiveMetadata tileArchiveMetadata; - private TileArchiveWriter(Iterable inputTiles, TileArchive archive, + private TileArchiveWriter(Iterable inputTiles, WriteableTileArchive archive, PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) { this.inputTiles = inputTiles; @@ -88,7 +88,7 @@ public class TileArchiveWriter { } /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */ - public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize, + public static void writeOutput(FeatureGroup features, WriteableTileArchive output, DiskBacked fileSize, TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) { var timer = stats.startStage("archive"); @@ -335,28 +335,30 @@ public class TileArchiveWriter { } private void printTileStats() { - Format format = Format.defaultInstance(); - LOGGER.debug("Tile stats:"); - long sumSize = 0; - long sumCount = 0; - long maxMax = 0; - for (int z = config.minzoom(); z <= config.maxzoom(); z++) { - long totalCount = tilesByZoom[z].get(); - long totalSize = totalTileSizesByZoom[z].get(); - sumSize += totalSize; - sumCount += totalCount; - long maxSize = maxTileSizesByZoom[z].get(); - maxMax = Math.max(maxMax, maxSize); - LOGGER.debug("z{} avg:{} max:{}", - z, - format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false), - format.storage(maxSize, false)); + if (LOGGER.isDebugEnabled()) { + Format format = Format.defaultInstance(); + LOGGER.debug("Tile stats:"); + long sumSize = 0; + long sumCount = 0; + long maxMax = 0; + for (int z = config.minzoom(); z <= config.maxzoom(); z++) { + long totalCount = tilesByZoom[z].get(); + long totalSize = totalTileSizesByZoom[z].get(); + sumSize += totalSize; + sumCount += totalCount; + long maxSize = maxTileSizesByZoom[z].get(); + maxMax = Math.max(maxMax, maxSize); + LOGGER.debug("z{} avg:{} max:{}", + z, + format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false), + format.storage(maxSize, false)); + } + LOGGER.debug("all avg:{} max:{}", + format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false), + format.storage(maxMax, false)); + LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get())); + LOGGER.debug(" # tiles: {}", format.integer(this.tilesEmitted())); } - LOGGER.debug("all avg:{} max:{}", - format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false), - format.storage(maxMax, false)); - LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get())); - LOGGER.debug(" # tiles: {}", format.integer(this.tilesEmitted())); } private long tilesEmitted() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java similarity index 96% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java index 39b9a08d..5df7c0a4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.geo.TileCoord; import java.util.Arrays; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java similarity index 70% rename from planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java rename to planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index 30126489..7449c0b1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.writer; +package com.onthegomap.planetiler.archive; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.TileOrder; @@ -7,16 +7,22 @@ import java.io.Closeable; import javax.annotation.concurrent.NotThreadSafe; /** - * A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive + * Write API for an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive * format. + *

+ * See {@link ReadableTileArchive} for the read API. */ @NotThreadSafe -public interface TileArchive extends Closeable { +public interface WriteableTileArchive extends Closeable { + interface TileWriter extends Closeable { + void write(TileEncodingResult encodingResult); // TODO: exists for compatibility reasons - void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult); + default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { + write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash())); + } @Override void close(); @@ -47,4 +53,6 @@ public interface TileArchive extends Closeable { * disk. */ void finish(PlanetilerConfig config); + + // TODO update archive metadata } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java index be34388f..70658d27 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java @@ -1,13 +1,14 @@ package com.onthegomap.planetiler.geo; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import java.util.function.IntFunction; import java.util.function.ToDoubleBiFunction; import java.util.function.ToIntFunction; /** * Controls the sort order of {@link com.onthegomap.planetiler.collection.FeatureGroup}, which determines the ordering - * of {@link com.onthegomap.planetiler.writer.TileEncodingResult}s when written to - * {@link com.onthegomap.planetiler.writer.TileArchive.TileWriter}. + * of {@link com.onthegomap.planetiler.archive.TileEncodingResult}s when written to + * {@link WriteableTileArchive.TileWriter}. */ public enum TileOrder { TMS(TileCoord::encoded, TileCoord::decode, TileCoord::progressOnLevel), diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index 5a314922..a2cc353c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -7,15 +7,18 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.reader.FileFormatException; +import com.onthegomap.planetiler.util.CloseableIterator; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.LayerStats; -import com.onthegomap.planetiler.writer.TileArchive; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.nio.file.Path; import java.sql.Connection; @@ -31,6 +34,7 @@ import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.OptionalLong; import java.util.TreeMap; @@ -48,7 +52,7 @@ import org.sqlite.SQLiteConfig; * * @see MBTiles Specification */ -public final class Mbtiles implements TileArchive { +public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive { // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt private static final int MBTILES_APPLICATION_ID = 0x4d504258; @@ -88,13 +92,8 @@ public final class Mbtiles implements TileArchive { } private final Connection connection; - private PreparedStatement getTileStatement = null; private final boolean compactDb; - - @Override - public TileOrder tileOrder() { - return TileOrder.TMS; - } + private PreparedStatement getTileStatement = null; private Mbtiles(Connection connection, boolean compactDb) { this.connection = connection; @@ -152,6 +151,11 @@ public final class Mbtiles implements TileArchive { } } + @Override + public TileOrder tileOrder() { + return TileOrder.TMS; + } + @Override public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) { if (config.skipIndexCreation()) { @@ -324,7 +328,7 @@ public final class Mbtiles implements TileArchive { } /** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */ - public TileArchive.TileWriter newTileWriter() { + public WriteableTileArchive.TileWriter newTileWriter() { if (compactDb) { return new BatchedCompactTileWriter(); } else { @@ -333,7 +337,7 @@ public final class Mbtiles implements TileArchive { } // TODO: exists for compatibility purposes - public TileArchive.TileWriter newBatchedTileWriter() { + public WriteableTileArchive.TileWriter newBatchedTileWriter() { return newTileWriter(); } @@ -356,10 +360,7 @@ public final class Mbtiles implements TileArchive { return getTileStatement; } - public byte[] getTile(TileCoord coord) { - return getTile(coord.x(), coord.y(), coord.z()); - } - + @Override public byte[] getTile(int x, int y, int z) { try { PreparedStatement stmt = getTileStatement(); @@ -374,22 +375,9 @@ public final class Mbtiles implements TileArchive { } } - public List getAllTileCoords() { - List result = new ArrayList<>(); - try (Statement statement = connection.createStatement()) { - ResultSet rs = statement.executeQuery( - "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE) - ); - while (rs.next()) { - int z = rs.getInt(TILES_COL_Z); - int rawy = rs.getInt(TILES_COL_Y); - int x = rs.getInt(TILES_COL_X); - result.add(TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z)); - } - } catch (SQLException throwables) { - throw new IllegalStateException("Could not get all tile coordinates", throwables); - } - return result; + @Override + public CloseableIterator getAllTileCoords() { + return new TileCoordIterator(); } public Connection connection() { @@ -492,14 +480,70 @@ public final class Mbtiles implements TileArchive { if (this == obj) { return true; } - if (!(obj instanceof TileDataEntry)) { + if (!(obj instanceof TileDataEntry other)) { return false; } - TileDataEntry other = (TileDataEntry) obj; return Arrays.equals(tileData, other.tileData) && tileDataId == other.tileDataId; } } + /** Iterates through tile coordinates one at a time without materializing the entire list in memory. */ + private class TileCoordIterator implements CloseableIterator { + private final Statement statement; + private final ResultSet rs; + private boolean hasNext = false; + + private TileCoordIterator() { + try { + this.statement = connection.createStatement(); + this.rs = statement.executeQuery( + "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE) + ); + hasNext = rs.next(); + } catch (SQLException e) { + throw new FileFormatException("Could not read tile coordinates from mbtiles file", e); + } finally { + if (!hasNext) { + close(); + } + } + } + + @Override + public void close() { + try { + statement.close(); + } catch (SQLException e) { + throw new IllegalStateException("Could not close mbtiles file", e); + } + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public TileCoord next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + try { + int z = rs.getInt(TILES_COL_Z); + int rawy = rs.getInt(TILES_COL_Y); + int x = rs.getInt(TILES_COL_X); + var result = TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z); + hasNext = rs.next(); + if (!hasNext) { + close(); + } + return result; + } catch (SQLException e) { + throw new IllegalStateException("Could not read mbtiles file", e); + } + } + } + private abstract class BatchedTableWriterBase implements AutoCloseable { private static final int MAX_PARAMETERS_IN_PREPARED_STATEMENT = 999; @@ -667,12 +711,6 @@ public final class Mbtiles implements TileArchive { tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData())); } - // TODO: exists for compatibility purposes - @Override - public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { - tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData())); - } - @Override public void close() { tableWriter.close(); @@ -714,12 +752,6 @@ public final class Mbtiles implements TileArchive { batchedTileShallowTableWriter.write(new TileShallowEntry(encodingResult.coord(), tileDataId)); } - // TODO: exists for compatibility purposes - @Override - public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { - write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash())); - } - @Override public void close() { batchedTileShallowTableWriter.close(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java index 093ceea2..8fa73fd6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java @@ -59,16 +59,19 @@ public class Verify { public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map attrs, Envelope envelope, Class clazz) throws GeometryException { int num = 0; - for (var tileCoord : db.getAllTileCoords()) { - Envelope tileEnv = new Envelope(); - tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY())); - tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY())); - if (tileCoord.z() == zoom) { - byte[] data = db.getTile(tileCoord); - for (var feature : decode(data)) { - if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) { - Geometry geometry = feature.geometry().decode(); - num += getGeometryCounts(geometry, clazz); + try (var tileCoords = db.getAllTileCoords()) { + while (tileCoords.hasNext()) { + var tileCoord = tileCoords.next(); + Envelope tileEnv = new Envelope(); + tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY())); + tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY())); + if (tileCoord.z() == zoom) { + byte[] data = db.getTile(tileCoord); + for (var feature : decode(data)) { + if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) { + Geometry geometry = feature.geometry().decode(); + num += getGeometryCounts(geometry, clazz); + } } } } @@ -187,7 +190,7 @@ public class Verify { private void checkBasicStructure() { check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name")); - check("contains at least one tile", () -> !mbtiles.getAllTileCoords().isEmpty()); + check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent()); checkWithMessage("all tiles are valid", () -> { List invalidTiles = mbtiles.getAllTileCoords().stream() .flatMap(coord -> checkValidity(coord, decode(mbtiles.getTile(coord))).stream()) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java new file mode 100644 index 00000000..151ca9c1 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java @@ -0,0 +1,17 @@ +package com.onthegomap.planetiler.util; + +import java.io.Closeable; +import java.util.Iterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public interface CloseableIterator extends Closeable, Iterator { + + @Override + void close(); + + default Stream stream() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(this, 0), false).onClose(this::close); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java index 8a9bbde6..4d3ec988 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java @@ -1,9 +1,9 @@ package com.onthegomap.planetiler.util; import com.fasterxml.jackson.annotation.JsonProperty; +import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.render.RenderedFeature; -import com.onthegomap.planetiler.writer.TileArchive; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,8 +18,8 @@ import javax.annotation.concurrent.ThreadSafe; /** * Tracks the feature attributes and zoom range of each layer to populate the archive output metadata. *

- * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link TileArchive} formats. To - * minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a + * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link WriteableTileArchive} formats. + * To minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a * thread-local handler that can update stats without contention. *

* diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 32201a93..d8709a0d 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -3,6 +3,8 @@ package com.onthegomap.planetiler; import static com.onthegomap.planetiler.TestUtils.*; import static org.junit.jupiter.api.Assertions.*; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -23,8 +25,6 @@ import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.BuildInfo; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java index c4607b49..240e76a4 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.VectorTile; +import com.onthegomap.planetiler.archive.TileArchiveWriter; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.geo.TileOrder; @@ -17,7 +18,6 @@ import com.onthegomap.planetiler.render.RenderedFeature; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.CloseableConsumer; import com.onthegomap.planetiler.util.Gzip; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java index 42d5e8a2..696947b3 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java @@ -5,15 +5,14 @@ import static org.junit.jupiter.api.Assertions.*; import com.google.common.math.IntMath; import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.util.LayerStats; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.math.RoundingMode; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.OptionalLong; @@ -69,7 +68,7 @@ class MbtilesTest { assertEquals(howMany, all.size()); assertEquals(expected, all); assertEquals(expected.stream().map(Mbtiles.TileEntry::tile).collect(Collectors.toSet()), - new HashSet<>(db.getAllTileCoords())); + db.getAllTileCoords().stream().collect(Collectors.toSet())); for (var expectedEntry : expected) { var tile = expectedEntry.tile(); byte[] data = db.getTile(tile.x(), tile.y(), tile.z()); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java index cd0c607b..9324add2 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java @@ -7,8 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.planetiler.VectorTile; +import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.geo.TileCoord; -import com.onthegomap.planetiler.writer.TileEncodingResult; import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java index 60fb6cd6..1c5a8e61 100644 --- a/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java +++ b/planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java @@ -2,6 +2,8 @@ package com.onthegomap.planetiler.examples; import com.onthegomap.planetiler.Planetiler; import com.onthegomap.planetiler.Profile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveWriter; import com.onthegomap.planetiler.collection.FeatureGroup; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.LongLongMultimap; @@ -13,8 +15,6 @@ import com.onthegomap.planetiler.reader.osm.OsmInputFile; import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.FileUtils; -import com.onthegomap.planetiler.writer.TileArchiveMetadata; -import com.onthegomap.planetiler.writer.TileArchiveWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path;