Convert get all tile coords to iterator (#463)

ARCHITECTURE.md

@@ -96,7 +96,8 @@ of the intermediate features using a worker thread per core:
 
 ## 3) Emit Vector Tiles
 
-[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver.
+[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) is the main
+driver.
 First, a single-threaded reader reads features from disk:
 
 - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits
@@ -104,7 +105,8 @@ First, a single-threaded reader reads features from disk:
 - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects
   consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the
   grouping limit to limit point label density
-- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles
+- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java)
+  groups tiles
   into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy
   while the writer thread waits for finished tiles in order)
 

planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java

@@ -1,12 +1,12 @@
 package com.onthegomap.planetiler.benchmarks;
 
 import com.google.common.base.Stopwatch;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.archive.WriteableTileArchive.TileWriter;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
-import com.onthegomap.planetiler.writer.TileArchive.TileWriter;
-import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -1,5 +1,8 @@
 package com.onthegomap.planetiler;
 
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileArchiveWriter;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
@@ -26,9 +29,6 @@ import com.onthegomap.planetiler.util.ResourceUsage;
 import com.onthegomap.planetiler.util.Translations;
 import com.onthegomap.planetiler.util.Wikidata;
 import com.onthegomap.planetiler.worker.RunnableThatThrows;
-import com.onthegomap.planetiler.writer.TileArchive;
-import com.onthegomap.planetiler.writer.TileArchiveMetadata;
-import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
@@ -658,7 +658,7 @@ public class Planetiler {
       bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats));
     }
 
-    try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+    try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
       featureGroup =
         FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats);
       stats.monitorFile("nodes", nodeDbPath);

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/ReadableTileArchive.java

@@ -0,0 +1,44 @@
+package com.onthegomap.planetiler.archive;
+
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import java.io.Closeable;
+
+/**
+ * Read API for on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
+ * format.
+ * <p>
+ * See {@link WriteableTileArchive} for the write API.
+ */
+public interface ReadableTileArchive extends Closeable {
+
+  /** Returns the raw tile data associated with the tile at {@code coord}. */
+  default byte[] getTile(TileCoord coord) {
+    return getTile(coord.x(), coord.y(), coord.z());
+  }
+
+  /** Returns the raw tile data associated with the tile at coordinate {@code x, y, z}. */
+  byte[] getTile(int x, int y, int z);
+
+  /**
+   * Returns an iterator over the coordinates of tiles in this archive.
+   * <p>
+   * The order should respect {@link WriteableTileArchive#tileOrder()} of the corresponding writer.
+   * <p>
+   * Clients should be sure to close the iterator after iterating through it, for example:
+   *
+   * <pre>
+   * {@code
+   * try (var iter = archive.getAllTileCoords()) {
+   *   while (iter.hasNext()) {
+   *     var coord = iter.next();
+   *     ...
+   *   }
+   * }
+   * }
+   * </pre>
+   */
+  CloseableIterator<TileCoord> getAllTileCoords();
+
+  // TODO access archive metadata
+}

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.Profile;
 import com.onthegomap.planetiler.config.Arguments;

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import static com.onthegomap.planetiler.util.Gzip.gzip;
 import static com.onthegomap.planetiler.worker.Worker.joinFutures;
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a
- * {@link TileArchive}.
+ * {@link WriteableTileArchive}.
  */
 public class TileArchiveWriter {
 
@@ -49,7 +49,7 @@ public class TileArchiveWriter {
   private static final long MAX_TILES_PER_BATCH = 1_000;
   private final Counter.Readable featuresProcessed;
   private final Counter memoizedTiles;
-  private final TileArchive archive;
+  private final WriteableTileArchive archive;
   private final PlanetilerConfig config;
   private final Stats stats;
   private final LayerStats layerStats;
@@ -60,7 +60,7 @@ public class TileArchiveWriter {
   private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
   private final TileArchiveMetadata tileArchiveMetadata;
 
-  private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, TileArchive archive,
+  private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, WriteableTileArchive archive,
     PlanetilerConfig config,
     TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) {
     this.inputTiles = inputTiles;
@@ -88,7 +88,7 @@ public class TileArchiveWriter {
   }
 
   /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */
-  public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize,
+  public static void writeOutput(FeatureGroup features, WriteableTileArchive output, DiskBacked fileSize,
     TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) {
     var timer = stats.startStage("archive");
 
@@ -335,28 +335,30 @@ public class TileArchiveWriter {
   }
 
   private void printTileStats() {
-    Format format = Format.defaultInstance();
+    if (LOGGER.isDebugEnabled()) {
-    LOGGER.debug("Tile stats:");
+      Format format = Format.defaultInstance();
-    long sumSize = 0;
+      LOGGER.debug("Tile stats:");
-    long sumCount = 0;
+      long sumSize = 0;
-    long maxMax = 0;
+      long sumCount = 0;
-    for (int z = config.minzoom(); z <= config.maxzoom(); z++) {
+      long maxMax = 0;
-      long totalCount = tilesByZoom[z].get();
+      for (int z = config.minzoom(); z <= config.maxzoom(); z++) {
-      long totalSize = totalTileSizesByZoom[z].get();
+        long totalCount = tilesByZoom[z].get();
-      sumSize += totalSize;
+        long totalSize = totalTileSizesByZoom[z].get();
-      sumCount += totalCount;
+        sumSize += totalSize;
-      long maxSize = maxTileSizesByZoom[z].get();
+        sumCount += totalCount;
-      maxMax = Math.max(maxMax, maxSize);
+        long maxSize = maxTileSizesByZoom[z].get();
-      LOGGER.debug("z{} avg:{} max:{}",
+        maxMax = Math.max(maxMax, maxSize);
-        z,
+        LOGGER.debug("z{} avg:{} max:{}",
-        format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false),
+          z,
-        format.storage(maxSize, false));
+          format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false),
+          format.storage(maxSize, false));
+      }
+      LOGGER.debug("all avg:{} max:{}",
+        format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false),
+        format.storage(maxMax, false));
+      LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get()));
+      LOGGER.debug("    # tiles: {}", format.integer(this.tilesEmitted()));
     }
-    LOGGER.debug("all avg:{} max:{}",
-      format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false),
-      format.storage(maxMax, false));
-    LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get()));
-    LOGGER.debug("    # tiles: {}", format.integer(this.tilesEmitted()));
   }
 
   private long tilesEmitted() {

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.geo.TileCoord;
 import java.util.Arrays;

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java

@@ -1,4 +1,4 @@
-package com.onthegomap.planetiler.writer;
+package com.onthegomap.planetiler.archive;
 
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileOrder;
@@ -7,16 +7,22 @@ import java.io.Closeable;
 import javax.annotation.concurrent.NotThreadSafe;
 
 /**
- * A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
+ * Write API for an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
  * format.
+ * <p>
+ * See {@link ReadableTileArchive} for the read API.
  */
 @NotThreadSafe
-public interface TileArchive extends Closeable {
+public interface WriteableTileArchive extends Closeable {
+
   interface TileWriter extends Closeable {
+
     void write(TileEncodingResult encodingResult);
 
     // TODO: exists for compatibility reasons
-    void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult);
+    default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
+      write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
+    }
 
     @Override
     void close();
@@ -47,4 +53,6 @@ public interface TileArchive extends Closeable {
    * disk.
    */
   void finish(PlanetilerConfig config);
+
+  // TODO update archive metadata
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileOrder.java

@@ -1,13 +1,14 @@
 package com.onthegomap.planetiler.geo;
 
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import java.util.function.IntFunction;
 import java.util.function.ToDoubleBiFunction;
 import java.util.function.ToIntFunction;
 
 /**
  * Controls the sort order of {@link com.onthegomap.planetiler.collection.FeatureGroup}, which determines the ordering
- * of {@link com.onthegomap.planetiler.writer.TileEncodingResult}s when written to
+ * of {@link com.onthegomap.planetiler.archive.TileEncodingResult}s when written to
- * {@link com.onthegomap.planetiler.writer.TileArchive.TileWriter}.
+ * {@link WriteableTileArchive.TileWriter}.
  */
 public enum TileOrder {
   TMS(TileCoord::encoded, TileCoord::decode, TileCoord::progressOnLevel),

planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java

@@ -7,15 +7,18 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.onthegomap.planetiler.archive.ReadableTileArchive;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.geo.TileOrder;
+import com.onthegomap.planetiler.reader.FileFormatException;
+import com.onthegomap.planetiler.util.CloseableIterator;
 import com.onthegomap.planetiler.util.Format;
 import com.onthegomap.planetiler.util.LayerStats;
-import com.onthegomap.planetiler.writer.TileArchive;
-import com.onthegomap.planetiler.writer.TileArchiveMetadata;
-import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.sql.Connection;
@@ -31,6 +34,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.OptionalLong;
 import java.util.TreeMap;
@@ -48,7 +52,7 @@ import org.sqlite.SQLiteConfig;
  *
  * @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md">MBTiles Specification</a>
  */
-public final class Mbtiles implements TileArchive {
+public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive {
 
   // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
   private static final int MBTILES_APPLICATION_ID = 0x4d504258;
@@ -88,13 +92,8 @@ public final class Mbtiles implements TileArchive {
   }
 
   private final Connection connection;
-  private PreparedStatement getTileStatement = null;
   private final boolean compactDb;
-
+  private PreparedStatement getTileStatement = null;
-  @Override
-  public TileOrder tileOrder() {
-    return TileOrder.TMS;
-  }
 
   private Mbtiles(Connection connection, boolean compactDb) {
     this.connection = connection;
@@ -152,6 +151,11 @@ public final class Mbtiles implements TileArchive {
     }
   }
 
+  @Override
+  public TileOrder tileOrder() {
+    return TileOrder.TMS;
+  }
+
   @Override
   public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
     if (config.skipIndexCreation()) {
@@ -324,7 +328,7 @@ public final class Mbtiles implements TileArchive {
   }
 
   /** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */
-  public TileArchive.TileWriter newTileWriter() {
+  public WriteableTileArchive.TileWriter newTileWriter() {
     if (compactDb) {
       return new BatchedCompactTileWriter();
     } else {
@@ -333,7 +337,7 @@ public final class Mbtiles implements TileArchive {
   }
 
   // TODO: exists for compatibility purposes
-  public TileArchive.TileWriter newBatchedTileWriter() {
+  public WriteableTileArchive.TileWriter newBatchedTileWriter() {
     return newTileWriter();
   }
 
@@ -356,10 +360,7 @@ public final class Mbtiles implements TileArchive {
     return getTileStatement;
   }
 
-  public byte[] getTile(TileCoord coord) {
+  @Override
-    return getTile(coord.x(), coord.y(), coord.z());
-  }
-
   public byte[] getTile(int x, int y, int z) {
     try {
       PreparedStatement stmt = getTileStatement();
@@ -374,22 +375,9 @@ public final class Mbtiles implements TileArchive {
     }
   }
 
-  public List<TileCoord> getAllTileCoords() {
+  @Override
-    List<TileCoord> result = new ArrayList<>();
+  public CloseableIterator<TileCoord> getAllTileCoords() {
-    try (Statement statement = connection.createStatement()) {
+    return new TileCoordIterator();
-      ResultSet rs = statement.executeQuery(
-        "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE)
-      );
-      while (rs.next()) {
-        int z = rs.getInt(TILES_COL_Z);
-        int rawy = rs.getInt(TILES_COL_Y);
-        int x = rs.getInt(TILES_COL_X);
-        result.add(TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z));
-      }
-    } catch (SQLException throwables) {
-      throw new IllegalStateException("Could not get all tile coordinates", throwables);
-    }
-    return result;
   }
 
   public Connection connection() {
@@ -492,14 +480,70 @@ public final class Mbtiles implements TileArchive {
       if (this == obj) {
         return true;
       }
-      if (!(obj instanceof TileDataEntry)) {
+      if (!(obj instanceof TileDataEntry other)) {
         return false;
       }
-      TileDataEntry other = (TileDataEntry) obj;
       return Arrays.equals(tileData, other.tileData) && tileDataId == other.tileDataId;
     }
   }
 
+  /** Iterates through tile coordinates one at a time without materializing the entire list in memory. */
+  private class TileCoordIterator implements CloseableIterator<TileCoord> {
+    private final Statement statement;
+    private final ResultSet rs;
+    private boolean hasNext = false;
+
+    private TileCoordIterator() {
+      try {
+        this.statement = connection.createStatement();
+        this.rs = statement.executeQuery(
+          "select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE)
+        );
+        hasNext = rs.next();
+      } catch (SQLException e) {
+        throw new FileFormatException("Could not read tile coordinates from mbtiles file", e);
+      } finally {
+        if (!hasNext) {
+          close();
+        }
+      }
+    }
+
+    @Override
+    public void close() {
+      try {
+        statement.close();
+      } catch (SQLException e) {
+        throw new IllegalStateException("Could not close mbtiles file", e);
+      }
+    }
+
+    @Override
+    public boolean hasNext() {
+      return hasNext;
+    }
+
+    @Override
+    public TileCoord next() {
+      if (!hasNext()) {
+        throw new NoSuchElementException();
+      }
+      try {
+        int z = rs.getInt(TILES_COL_Z);
+        int rawy = rs.getInt(TILES_COL_Y);
+        int x = rs.getInt(TILES_COL_X);
+        var result = TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z);
+        hasNext = rs.next();
+        if (!hasNext) {
+          close();
+        }
+        return result;
+      } catch (SQLException e) {
+        throw new IllegalStateException("Could not read mbtiles file", e);
+      }
+    }
+  }
+
   private abstract class BatchedTableWriterBase<T> implements AutoCloseable {
 
     private static final int MAX_PARAMETERS_IN_PREPARED_STATEMENT = 999;
@@ -667,12 +711,6 @@ public final class Mbtiles implements TileArchive {
       tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData()));
     }
 
-    // TODO: exists for compatibility purposes
-    @Override
-    public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
-      tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData()));
-    }
-
     @Override
     public void close() {
       tableWriter.close();
@@ -714,12 +752,6 @@ public final class Mbtiles implements TileArchive {
       batchedTileShallowTableWriter.write(new TileShallowEntry(encodingResult.coord(), tileDataId));
     }
 
-    // TODO: exists for compatibility purposes
-    @Override
-    public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
-      write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
-    }
-
     @Override
     public void close() {
       batchedTileShallowTableWriter.close();

planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java

@@ -59,16 +59,19 @@ public class Verify {
   public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map<String, Object> attrs, Envelope envelope,
     Class<? extends Geometry> clazz) throws GeometryException {
     int num = 0;
-    for (var tileCoord : db.getAllTileCoords()) {
+    try (var tileCoords = db.getAllTileCoords()) {
-      Envelope tileEnv = new Envelope();
+      while (tileCoords.hasNext()) {
-      tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY()));
+        var tileCoord = tileCoords.next();
-      tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY()));
+        Envelope tileEnv = new Envelope();
-      if (tileCoord.z() == zoom) {
+        tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY()));
-        byte[] data = db.getTile(tileCoord);
+        tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY()));
-        for (var feature : decode(data)) {
+        if (tileCoord.z() == zoom) {
-          if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) {
+          byte[] data = db.getTile(tileCoord);
-            Geometry geometry = feature.geometry().decode();
+          for (var feature : decode(data)) {
-            num += getGeometryCounts(geometry, clazz);
+            if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) {
+              Geometry geometry = feature.geometry().decode();
+              num += getGeometryCounts(geometry, clazz);
+            }
           }
         }
       }
@@ -187,7 +190,7 @@ public class Verify {
 
   private void checkBasicStructure() {
     check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name"));
-    check("contains at least one tile", () -> !mbtiles.getAllTileCoords().isEmpty());
+    check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent());
     checkWithMessage("all tiles are valid", () -> {
       List<String> invalidTiles = mbtiles.getAllTileCoords().stream()
         .flatMap(coord -> checkValidity(coord, decode(mbtiles.getTile(coord))).stream())

planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java

@@ -0,0 +1,17 @@
+package com.onthegomap.planetiler.util;
+
+import java.io.Closeable;
+import java.util.Iterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+public interface CloseableIterator<T> extends Closeable, Iterator<T> {
+
+  @Override
+  void close();
+
+  default Stream<T> stream() {
+    return StreamSupport.stream(Spliterators.spliteratorUnknownSize(this, 0), false).onClose(this::close);
+  }
+}

planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java

@@ -1,9 +1,9 @@
 package com.onthegomap.planetiler.util;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.render.RenderedFeature;
-import com.onthegomap.planetiler.writer.TileArchive;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -18,8 +18,8 @@ import javax.annotation.concurrent.ThreadSafe;
 /**
  * Tracks the feature attributes and zoom range of each layer to populate the archive output metadata.
  * <p>
- * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link TileArchive} formats. To
+ * Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link WriteableTileArchive} formats.
- * minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
+ * To minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
  * thread-local handler that can update stats without contention.
  * </p>
  *

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -3,6 +3,8 @@ package com.onthegomap.planetiler;
 import static com.onthegomap.planetiler.TestUtils.*;
 import static org.junit.jupiter.api.Assertions.*;
 
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileArchiveWriter;
 import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
@@ -23,8 +25,6 @@ import com.onthegomap.planetiler.reader.osm.OsmReader;
 import com.onthegomap.planetiler.reader.osm.OsmRelationInfo;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.BuildInfo;
-import com.onthegomap.planetiler.writer.TileArchiveMetadata;
-import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;

planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java

@@ -10,6 +10,7 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest;
 
 import com.onthegomap.planetiler.Profile;
 import com.onthegomap.planetiler.VectorTile;
+import com.onthegomap.planetiler.archive.TileArchiveWriter;
 import com.onthegomap.planetiler.geo.GeometryType;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.geo.TileOrder;
@@ -17,7 +18,6 @@ import com.onthegomap.planetiler.render.RenderedFeature;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.CloseableConsumer;
 import com.onthegomap.planetiler.util.Gzip;
-import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;

planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java

@@ -5,15 +5,14 @@ import static org.junit.jupiter.api.Assertions.*;
 
 import com.google.common.math.IntMath;
 import com.onthegomap.planetiler.TestUtils;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.util.LayerStats;
-import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.IOException;
 import java.math.RoundingMode;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalLong;
@@ -69,7 +68,7 @@ class MbtilesTest {
       assertEquals(howMany, all.size());
       assertEquals(expected, all);
       assertEquals(expected.stream().map(Mbtiles.TileEntry::tile).collect(Collectors.toSet()),
-        new HashSet<>(db.getAllTileCoords()));
+        db.getAllTileCoords().stream().collect(Collectors.toSet()));
       for (var expectedEntry : expected) {
         var tile = expectedEntry.tile();
         byte[] data = db.getTile(tile.x(), tile.y(), tile.z());

planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java

@@ -7,8 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.onthegomap.planetiler.VectorTile;
+import com.onthegomap.planetiler.archive.TileEncodingResult;
 import com.onthegomap.planetiler.geo.TileCoord;
-import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;

planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java

@@ -2,6 +2,8 @@ package com.onthegomap.planetiler.examples;
 
 import com.onthegomap.planetiler.Planetiler;
 import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileArchiveWriter;
 import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
@@ -13,8 +15,6 @@ import com.onthegomap.planetiler.reader.osm.OsmInputFile;
 import com.onthegomap.planetiler.reader.osm.OsmReader;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.FileUtils;
-import com.onthegomap.planetiler.writer.TileArchiveMetadata;
-import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;