Tile archive refactor (#443)

ARCHITECTURE.md

@@ -8,7 +8,7 @@ Planetiler builds a map in 3 phases:
    intermediate files on disk
 2. [Sort Features](#2-sort-features) by tile ID
 3. [Emit Vector Tiles](#3-emit-vector-tiles) by iterating through sorted features to group by tile ID, encoding, and
-   writing to the output MBTiles file
+   writing to the output tile archive
 
 User-defined [profiles](#profiles) customize the behavior of each part of this pipeline.
 
@@ -96,7 +96,7 @@ of the intermediate features using a worker thread per core:
 
 ## 3) Emit Vector Tiles
 
-[MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) is the main driver.
+[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver.
 First, a single-threaded reader reads features from disk:
 
 - [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits
@@ -104,7 +104,7 @@ First, a single-threaded reader reads features from disk:
 - [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects
   consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the
   grouping limit to limit point label density
-- Then [MbtilesWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java) groups tiles
+- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles
   into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy
   while the writer thread waits for finished tiles in order)
 
@@ -116,9 +116,9 @@ Then, process tile batches in worker threads (default 1 per core):
 - gzip each encoded tile
 - Pass the batch of encoded vector tiles to the writer thread
 
-Finally, a single-threaded writer writes encoded vector tiles to the output MBTiles file:
+Finally, a single-threaded writer writes encoded vector tiles to the output archive format:
 
-- Create the largest prepared statement supported by SQLite (999 parameters)
+- For MBTiles, create the largest prepared statement supported by SQLite (999 parameters)
 - Iterate through finished vector tile batches until the prepared statement is full, flush to disk, then repeat
 - Then flush any remaining tiles at the end
 

planetiler-benchmarks/src/main/java/com/onthegomap/planetiler/benchmarks/BenchmarkMbtilesWriter.java

@@ -5,8 +5,8 @@ import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
-import com.onthegomap.planetiler.mbtiles.Mbtiles.BatchedTileWriter;
-import com.onthegomap.planetiler.mbtiles.TileEncodingResult;
+import com.onthegomap.planetiler.writer.TileArchive.TileWriter;
+import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
@@ -74,7 +74,7 @@ public class BenchmarkMbtilesWriter {
           mbtiles.createTablesWithIndexes();
         }
 
-        try (var writer = mbtiles.newBatchedTileWriter()) {
+        try (var writer = mbtiles.newTileWriter()) {
           Stopwatch sw = Stopwatch.createStarted();
           writeTiles(writer, tilesToWrite, distinctTilesInPercent, distinctTileData, dupeTileData, dupeSpreadInPercent);
           sw.stop();
@@ -92,7 +92,7 @@ public class BenchmarkMbtilesWriter {
   }
 
 
-  private static void writeTiles(BatchedTileWriter writer, int tilesToWrite, int distinctTilesInPercent,
+  private static void writeTiles(TileWriter writer, int tilesToWrite, int distinctTilesInPercent,
     byte[] distinctTileData, byte[] dupeTileData, int dupeSpreadInPercent) {
 
     int dupesToWrite = (int) Math.round(tilesToWrite * (100 - distinctTilesInPercent) / 100.0);

planetiler-core/src/main/java/com/onthegomap/planetiler/FeatureMerge.java

@@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory;
 /**
  * A collection of utilities for merging features with the same attributes in a rendered tile from
  * {@link Profile#postProcessLayerFeatures(String, int, List)} immediately before a tile is written to the output
- * mbtiles file.
+ * archive.
  * <p>
  * Unlike postgis-based solutions that have a full view of all features after they are loaded into the database, the
  * planetiler engine only sees a single input feature at a time while processing source features, then only has

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -4,9 +4,8 @@ import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
 import com.onthegomap.planetiler.config.Arguments;
-import com.onthegomap.planetiler.config.MbtilesMetadata;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
-import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
+import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.reader.GeoPackageReader;
 import com.onthegomap.planetiler.reader.NaturalEarthReader;
 import com.onthegomap.planetiler.reader.ShapefileReader;
@@ -27,6 +26,10 @@ import com.onthegomap.planetiler.util.ResourceUsage;
 import com.onthegomap.planetiler.util.Translations;
 import com.onthegomap.planetiler.util.Wikidata;
 import com.onthegomap.planetiler.worker.RunnableThatThrows;
+import com.onthegomap.planetiler.writer.TileArchive;
+import com.onthegomap.planetiler.writer.TileArchiveMetadata;
+import com.onthegomap.planetiler.writer.TileArchiveWriter;
+import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -94,7 +97,7 @@ public class Planetiler {
   private boolean useWikidata = false;
   private boolean onlyFetchWikidata = false;
   private boolean fetchWikidata = false;
-  private MbtilesMetadata mbtilesMetadata;
+  private TileArchiveMetadata tileArchiveMetadata;
 
   private Planetiler(Arguments arguments) {
     this.arguments = arguments;
@@ -176,9 +179,10 @@ public class Planetiler {
       ),
       ifSourceUsed(name, () -> {
         var header = osmInputFile.getHeader();
-        mbtilesMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant());
-        mbtilesMetadata.set("planetiler:" + name + ":osmosisreplicationseq", header.osmosisReplicationSequenceNumber());
-        mbtilesMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl());
+        tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationtime", header.instant());
+        tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationseq",
+          header.osmosisReplicationSequenceNumber());
+        tileArchiveMetadata.set("planetiler:" + name + ":osmosisreplicationurl", header.osmosisReplicationBaseUrl());
         try (
           var nodeLocations =
             LongLongMap.from(config.nodeMapType(), config.nodeMapStorage(), nodeDbPath, config.nodeMapMadvise());
@@ -489,30 +493,29 @@ public class Planetiler {
   }
 
   /**
-   * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Fails if the file already exists.
+   * Sets the location of the output archive to write rendered tiles to. Fails if the archive already exists.
    * <p>
-   * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments.
+   * To override the location of the file, set {@code argument=newpath} in the arguments.
    *
    * @param argument the argument key to check for an override to {@code fallback}
    * @param fallback the fallback value if {@code argument} is not set in arguments
    * @return this runner instance for chaining
-   * @see MbtilesWriter
+   * @see TileArchiveWriter
    */
   public Planetiler setOutput(String argument, Path fallback) {
-    this.output = arguments.file(argument, "mbtiles output file", fallback);
+    this.output = arguments.file(argument, "output tile archive", fallback);
     return this;
   }
 
   /**
-   * Sets the location of the output {@code .mbtiles} file to write rendered tiles to. Overwrites file if it already
-   * exists.
+   * Sets the location of the output archive to write rendered tiles to. Overwrites file if it already exists.
    * <p>
-   * To override the location of the file, set {@code argument=newpath.mbtiles} in the arguments.
+   * To override the location of the file, set {@code argument=newpath} in the arguments.
    *
    * @param argument the argument key to check for an override to {@code fallback}
    * @param fallback the fallback value if {@code argument} is not set in arguments
    * @return this runner instance for chaining
-   * @see MbtilesWriter
+   * @see TileArchiveWriter
    */
   public Planetiler overwriteOutput(String argument, Path fallback) {
     this.overwrite = true;
@@ -521,7 +524,7 @@ public class Planetiler {
 
   /**
    * Reads all elements from all sourced that have been added, generates map features according to the profile, and
-   * writes the rendered tiles to the output mbtiles file.
+   * writes the rendered tiles to the output archive.
    *
    * @throws IllegalArgumentException if expected inputs have not been provided
    * @throws Exception                if an error occurs while processing
@@ -550,7 +553,7 @@ public class Planetiler {
       throw new IllegalArgumentException("Can only run once");
     }
     ran = true;
-    mbtilesMetadata = new MbtilesMetadata(profile, config.arguments());
+    tileArchiveMetadata = new TileArchiveMetadata(profile, config.arguments());
 
     if (arguments.getBoolean("help", "show arguments then exit", false)) {
       System.exit(0);
@@ -579,7 +582,7 @@ public class Planetiler {
         }
       }
       LOGGER.info("  sort: Sort rendered features by tile ID");
-      LOGGER.info("  mbtiles: Encode each tile and write to {}", output);
+      LOGGER.info("  archive: Encode each tile and write to {}", output);
     }
 
     // in case any temp files are left from a previous run...
@@ -616,7 +619,7 @@ public class Planetiler {
     stats.monitorFile("nodes", nodeDbPath);
     stats.monitorFile("features", featureDbPath);
     stats.monitorFile("multipolygons", multipolygonPath);
-    stats.monitorFile("mbtiles", output);
+    stats.monitorFile("archive", output);
 
     for (Stage stage : stages) {
       stage.task.run();
@@ -633,7 +636,13 @@ public class Planetiler {
 
     featureGroup.prepare();
 
-    MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats);
+    try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+      TileArchiveWriter.writeOutput(featureGroup, archive, () -> FileUtils.fileSize(output), tileArchiveMetadata,
+        config,
+        stats);
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to write to " + output, e);
+    }
 
     overallTimer.stop();
     LOGGER.info("FINISHED!");
@@ -659,7 +668,7 @@ public class Planetiler {
     readPhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
     writePhase.addDisk(featureDbPath, featureSize, "temporary feature storage");
     // output only needed during write phase
-    writePhase.addDisk(output, outputSize, "mbtiles output");
+    writePhase.addDisk(output, outputSize, "archive output");
     // if the user opts to remove an input source after reading to free up additional space for the output...
     for (var input : inputPaths) {
       if (input.freeAfterReading()) {

planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java

@@ -31,7 +31,7 @@ import java.util.function.Consumer;
  * (i.e. one per layer) and forwarding each element/event to the handlers that care about it.
  */
 public interface Profile {
-  // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and MbtilesMetadata) can use too
+  // TODO might want to break this apart into sub-interfaces that ForwardingProfile (and TileArchiveMetadata) can use too
 
   /**
    * Allows profile to extract any information it needs from a {@link OsmElement.Node} during the first pass through OSM

planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java

@@ -529,7 +529,7 @@ public class VectorTile {
    * <p>
    * Deduplication code is aiming for a balance between filtering-out all duplicates and not spending too much CPU on
    * hash calculations: calculating hashes for all tiles costs too much CPU, not calculating hashes at all means
-   * generating mbtiles which are too big. This method is responsible for achieving that balance.
+   * generating archives which are too big. This method is responsible for achieving that balance.
    * <p>
    * Current understanding is, that for the whole planet, there are 267m total tiles and 38m unique tiles. The
    * {@link #containsOnlyFillsOrEdges()} heuristic catches >99.9% of repeated tiles and cuts down the number of tile
@@ -561,7 +561,7 @@ public class VectorTile {
    * To encode extra precision in intermediate feature geometries, the geometry contained in {@code commands} is scaled
    * to a tile extent of {@code EXTENT * 2^scale}, so when the {@code scale == 0} the extent is {@link #EXTENT} and when
    * {@code scale == 2} the extent is 4x{@link #EXTENT}. Geometries must be scaled back to 0 using {@link #unscale()}
-   * before outputting to mbtiles.
+   * before outputting to the archive.
    */
   public record VectorGeometry(int[] commands, GeometryType geomType, int scale) {
 
@@ -627,7 +627,7 @@ public class VectorTile {
       return decodeCommands(geomType, commands, scale);
     }
 
-    /** Returns this encoded geometry, scaled back to 0, so it is safe to emit to mbtiles output. */
+    /** Returns this encoded geometry, scaled back to 0, so it is safe to emit to archive output. */
     public VectorGeometry unscale() {
       return scale == 0 ? this : new VectorGeometry(VectorTile.unscale(commands, scale, geomType), geomType, 0);
     }

planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java

@@ -7,10 +7,14 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.util.Format;
-import java.io.Closeable;
+import com.onthegomap.planetiler.util.LayerStats;
+import com.onthegomap.planetiler.writer.TileArchive;
+import com.onthegomap.planetiler.writer.TileArchiveMetadata;
+import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.sql.Connection;
@@ -46,7 +50,7 @@ import org.sqlite.SQLiteConfig;
  *
  * @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md">MBTiles Specification</a>
  */
-public final class Mbtiles implements Closeable {
+public final class Mbtiles implements TileArchive {
 
   // https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
   private static final int MBTILES_APPLICATION_ID = 0x4d504258;
@@ -145,6 +149,42 @@ public final class Mbtiles implements Closeable {
     }
   }
 
+  @Override
+  public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
+    if (config.skipIndexCreation()) {
+      createTablesWithoutIndexes();
+      if (LOGGER.isInfoEnabled()) {
+        LOGGER.info("Skipping index creation. Add later by executing: {}",
+          String.join(" ; ", getManualIndexCreationStatements()));
+      }
+    } else {
+      createTablesWithIndexes();
+    }
+
+    var metadata = metadata()
+      .setName(tileArchiveMetadata.name())
+      .setFormat("pbf")
+      .setDescription(tileArchiveMetadata.description())
+      .setAttribution(tileArchiveMetadata.attribution())
+      .setVersion(tileArchiveMetadata.version())
+      .setType(tileArchiveMetadata.type())
+      .setBoundsAndCenter(config.bounds().latLon())
+      .setMinzoom(config.minzoom())
+      .setMaxzoom(config.maxzoom())
+      .setJson(layerStats.getTileStats());
+
+    for (var entry : tileArchiveMetadata.planetilerSpecific().entrySet()) {
+      metadata.setMetadata(entry.getKey(), entry.getValue());
+    }
+  }
+
+  @Override
+  public void finish(PlanetilerConfig config) {
+    if (config.optimizeDb()) {
+      vacuumAnalyze();
+    }
+  }
+
   @Override
   public void close() throws IOException {
     try {
@@ -281,7 +321,7 @@ public final class Mbtiles implements Closeable {
   }
 
   /** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */
-  public BatchedTileWriter newBatchedTileWriter() {
+  public TileArchive.TileWriter newTileWriter() {
     if (compactDb) {
       return new BatchedCompactTileWriter();
     } else {
@@ -289,6 +329,11 @@ public final class Mbtiles implements Closeable {
     }
   }
 
+  // TODO: exists for compatibility purposes
+  public TileArchive.TileWriter newBatchedTileWriter() {
+    return newTileWriter();
+  }
+
   /** Returns the contents of the metadata table. */
   public Metadata metadata() {
     return new Metadata();
@@ -659,20 +704,7 @@ public final class Mbtiles implements Closeable {
     }
   }
 
-
-  /**
-   * A high-throughput writer that accepts new tiles and queues up the writes to execute them in fewer large-batches.
-   */
-  public interface BatchedTileWriter extends AutoCloseable {
-    void write(TileEncodingResult encodingResult);
-
-    @Override
-    void close();
-
-    default void printStats() {}
-  }
-
-  private class BatchedNonCompactTileWriter implements BatchedTileWriter {
+  private class BatchedNonCompactTileWriter implements TileWriter {
 
     private final BatchedTileTableWriter tableWriter = new BatchedTileTableWriter();
 
@@ -681,6 +713,12 @@ public final class Mbtiles implements Closeable {
       tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData()));
     }
 
+    // TODO: exists for compatibility purposes
+    @Override
+    public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
+      tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData()));
+    }
+
     @Override
     public void close() {
       tableWriter.close();
@@ -688,7 +726,7 @@ public final class Mbtiles implements Closeable {
 
   }
 
-  private class BatchedCompactTileWriter implements BatchedTileWriter {
+  private class BatchedCompactTileWriter implements TileWriter {
 
     private final BatchedTileShallowTableWriter batchedTileShallowTableWriter = new BatchedTileShallowTableWriter();
     private final BatchedTileDataTableWriter batchedTileDataTableWriter = new BatchedTileDataTableWriter();
@@ -722,6 +760,12 @@ public final class Mbtiles implements Closeable {
       batchedTileShallowTableWriter.write(new TileShallowEntry(encodingResult.coord(), tileDataId));
     }
 
+    // TODO: exists for compatibility purposes
+    @Override
+    public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
+      write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
+    }
+
     @Override
     public void close() {
       batchedTileShallowTableWriter.close();

planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/TileEncodingResult.java

@@ -5,6 +5,7 @@ import java.util.Arrays;
 import java.util.Objects;
 import java.util.OptionalLong;
 
+// TODO: exists for compatibility reasons
 public record TileEncodingResult(
   TileCoord coord,
   byte[] tileData,

planetiler-core/src/main/java/com/onthegomap/planetiler/render/FeatureRenderer.java

@@ -239,7 +239,7 @@ public class FeatureRenderer implements Consumer<FeatureCollector.Feature>, Clos
           // Store lines with extra precision (2^scale) in intermediate feature storage so that
           // rounding does not introduce artificial endpoint intersections and confuse line merge
           // post-processing.  Features need to be "unscaled" in FeatureGroup after line merging,
-          // and before emitting to output mbtiles.
+          // and before emitting to the output archive.
           scale = Math.max(config.maxzoom(), 14) - zoom;
           // need 14 bits to represent tile coordinates (4096 * 2 for buffer * 2 for zigzag encoding)
           // so cap the scale factor to avoid overflowing 32-bit integer space

planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java

@@ -152,7 +152,7 @@ class PrometheusStats implements Stats {
   }
 
   private final Histogram tilesWrittenBytes = Histogram
-    .build(BASE + "mbtiles_tile_written_bytes", "Written tile sizes by zoom level")
+    .build(BASE + "archive_tile_written_bytes", "Written tile sizes by zoom level")
     .buckets(1_000, 10_000, 100_000, 500_000)
     .labelNames("zoom")
     .register(registry);

planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java

@@ -99,7 +99,7 @@ public interface Stats extends AutoCloseable {
   /** Records that an input element was processed and emitted some output features in {@code layer}. */
   void processedElement(String elemType, String layer);
 
-  /** Records that a tile has been written to the mbtiles output where compressed size is {@code bytes}. */
+  /** Records that a tile has been written to the archive output where compressed size is {@code bytes}. */
   void wroteTile(int zoom, int bytes);
 
   /** Returns the timers for all stages started with {@link #startStage(String)}. */

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchive.java

@@ -0,0 +1,44 @@
+package com.onthegomap.planetiler.writer;
+
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.util.LayerStats;
+import java.io.Closeable;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
+ * format.
+ */
+@NotThreadSafe
+public interface TileArchive extends Closeable {
+  interface TileWriter extends Closeable {
+    void write(TileEncodingResult encodingResult);
+
+    // TODO: exists for compatibility reasons
+    void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult);
+
+    @Override
+    void close();
+
+    default void printStats() {}
+  }
+
+
+  /**
+   * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
+   * required state here.
+   */
+  void initialize(PlanetilerConfig config, TileArchiveMetadata metadata, LayerStats layerStats);
+
+  /**
+   * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might
+   * depend on {@link PlanetilerConfig}.
+   */
+  TileWriter newTileWriter();
+
+  /**
+   * Called after all tiles are written into {@link TileWriter}. After this is called, the archive should be complete on
+   * disk.
+   */
+  void finish(PlanetilerConfig config);
+}

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveMetadata.java

@@ -1,13 +1,13 @@
-package com.onthegomap.planetiler.config;
+package com.onthegomap.planetiler.writer;
 
 import com.onthegomap.planetiler.Profile;
-import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
+import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.util.BuildInfo;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-/** Controls information that {@link MbtilesWriter} will write to the mbtiles metadata table. */
-public record MbtilesMetadata(
+/** Controls information that {@link TileArchiveWriter} will write to the archive metadata. */
+public record TileArchiveMetadata(
   String name,
   String description,
   String attribution,
@@ -16,7 +16,7 @@ public record MbtilesMetadata(
   Map<String, String> planetilerSpecific
 ) {
 
-  public MbtilesMetadata(Profile profile) {
+  public TileArchiveMetadata(Profile profile) {
     this(
       profile.name(),
       profile.description(),
@@ -27,13 +27,13 @@ public record MbtilesMetadata(
     );
   }
 
-  public MbtilesMetadata(Profile profile, Arguments args) {
+  public TileArchiveMetadata(Profile profile, Arguments args) {
     this(
-      args.getString("mbtiles_name", "'name' attribute for mbtiles metadata", profile.name()),
-      args.getString("mbtiles_description", "'description' attribute for mbtiles metadata", profile.description()),
-      args.getString("mbtiles_attribution", "'attribution' attribute for mbtiles metadata", profile.attribution()),
-      args.getString("mbtiles_version", "'version' attribute for mbtiles metadata", profile.version()),
-      args.getString("mbtiles_type", "'type' attribute for mbtiles metadata",
+      args.getString("mbtiles_name", "'name' attribute for tileset metadata", profile.name()),
+      args.getString("mbtiles_description", "'description' attribute for tileset metadata", profile.description()),
+      args.getString("mbtiles_attribution", "'attribution' attribute for tileset metadata", profile.attribution()),
+      args.getString("mbtiles_version", "'version' attribute for tileset metadata", profile.version()),
+      args.getString("mbtiles_type", "'type' attribute for tileset metadata",
         profile.isOverlay() ? "overlay" : "baselayer"),
       mapWithBuildInfo()
     );
@@ -56,7 +56,7 @@ public record MbtilesMetadata(
     return result;
   }
 
-  public MbtilesMetadata set(String key, Object value) {
+  public TileArchiveMetadata set(String key, Object value) {
     if (key != null && value != null) {
       planetilerSpecific.put(key, value.toString());
     }

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java

@@ -1,11 +1,10 @@
-package com.onthegomap.planetiler.mbtiles;
+package com.onthegomap.planetiler.writer;
 
 import static com.onthegomap.planetiler.util.Gzip.gzip;
 import static com.onthegomap.planetiler.worker.Worker.joinFutures;
 
 import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.collection.FeatureGroup;
-import com.onthegomap.planetiler.config.MbtilesMetadata;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.stats.Counter;
@@ -14,7 +13,6 @@ import com.onthegomap.planetiler.stats.ProgressLoggers;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.stats.Timer;
 import com.onthegomap.planetiler.util.DiskBacked;
-import com.onthegomap.planetiler.util.FileUtils;
 import com.onthegomap.planetiler.util.Format;
 import com.onthegomap.planetiler.util.Hashing;
 import com.onthegomap.planetiler.util.LayerStats;
@@ -22,7 +20,6 @@ import com.onthegomap.planetiler.worker.WorkQueue;
 import com.onthegomap.planetiler.worker.Worker;
 import com.onthegomap.planetiler.worker.WorkerPipeline;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
@@ -42,17 +39,17 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to an
- * {@link Mbtiles} file.
+ * Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a
+ * {@link TileArchive}.
  */
-public class MbtilesWriter {
+public class TileArchiveWriter {
 
-  private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(TileArchiveWriter.class);
   private static final long MAX_FEATURES_PER_BATCH = 10_000;
   private static final long MAX_TILES_PER_BATCH = 1_000;
   private final Counter.Readable featuresProcessed;
   private final Counter memoizedTiles;
-  private final Mbtiles db;
+  private final TileArchive archive;
   private final PlanetilerConfig config;
   private final Stats stats;
   private final LayerStats layerStats;
@@ -61,14 +58,15 @@ public class MbtilesWriter {
   private final LongAccumulator[] maxTileSizesByZoom;
   private final Iterable<FeatureGroup.TileFeatures> inputTiles;
   private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
-  private final MbtilesMetadata mbtilesMetadata;
+  private final TileArchiveMetadata tileArchiveMetadata;
 
-  private MbtilesWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, Mbtiles db, PlanetilerConfig config,
-    MbtilesMetadata mbtilesMetadata, Stats stats, LayerStats layerStats) {
+  private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, TileArchive archive,
+    PlanetilerConfig config,
+    TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) {
     this.inputTiles = inputTiles;
-    this.db = db;
+    this.archive = archive;
     this.config = config;
-    this.mbtilesMetadata = mbtilesMetadata;
+    this.tileArchiveMetadata = tileArchiveMetadata;
     this.stats = stats;
     this.layerStats = layerStats;
     tilesByZoom = IntStream.rangeClosed(0, config.maxzoom())
@@ -80,29 +78,19 @@ public class MbtilesWriter {
     maxTileSizesByZoom = IntStream.rangeClosed(0, config.maxzoom())
       .mapToObj(i -> new LongAccumulator(Long::max, 0))
       .toArray(LongAccumulator[]::new);
-    memoizedTiles = stats.longCounter("mbtiles_memoized_tiles");
-    featuresProcessed = stats.longCounter("mbtiles_features_processed");
+    memoizedTiles = stats.longCounter("archive_memoized_tiles");
+    featuresProcessed = stats.longCounter("archive_features_processed");
     Map<String, LongSupplier> countsByZoom = new LinkedHashMap<>();
     for (int zoom = config.minzoom(); zoom <= config.maxzoom(); zoom++) {
       countsByZoom.put(Integer.toString(zoom), tilesByZoom[zoom]);
     }
-    stats.counter("mbtiles_tiles_written", "zoom", () -> countsByZoom);
-  }
-
-  /** Reads all {@code features}, encodes them in parallel, and writes to {@code outputPath}. */
-  public static void writeOutput(FeatureGroup features, Path outputPath, MbtilesMetadata mbtilesMetadata,
-    PlanetilerConfig config, Stats stats) {
-    try (Mbtiles output = Mbtiles.newWriteToFileDatabase(outputPath, config.compactDb())) {
-      writeOutput(features, output, () -> FileUtils.fileSize(outputPath), mbtilesMetadata, config, stats);
-    } catch (IOException e) {
-      throw new IllegalStateException("Unable to write to " + outputPath, e);
-    }
+    stats.counter("archive_tiles_written", "zoom", () -> countsByZoom);
   }
 
   /** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */
-  public static void writeOutput(FeatureGroup features, Mbtiles output, DiskBacked fileSize,
-    MbtilesMetadata mbtilesMetadata, PlanetilerConfig config, Stats stats) {
-    var timer = stats.startStage("mbtiles");
+  public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize,
+    TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) {
+    var timer = stats.startStage("archive");
 
     int readThreads = config.featureReadThreads();
     int threads = config.threads();
@@ -123,10 +111,10 @@ public class MbtilesWriter {
       readWorker = reader.readWorker();
     }
 
-    MbtilesWriter writer = new MbtilesWriter(inputTiles, output, config, mbtilesMetadata, stats,
+    TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats,
       features.layerStats());
 
-    var pipeline = WorkerPipeline.start("mbtiles", stats);
+    var pipeline = WorkerPipeline.start("archive", stats);
 
     // a larger tile queue size helps keep cores busy, but needs a lot of RAM
     // 5k works fine with 100GB of RAM, so adjust the queue size down from there
@@ -143,7 +131,7 @@ public class MbtilesWriter {
        * waits on them to be encoded in the order they were received, and the encoder processes them in parallel.
        * One batch might take a long time to process, so make the queues very big to avoid idle encoding CPUs.
        */
-      WorkQueue<TileBatch> writerQueue = new WorkQueue<>("mbtiles_writer_queue", queueSize, 1, stats);
+      WorkQueue<TileBatch> writerQueue = new WorkQueue<>("archive_writer_queue", queueSize, 1, stats);
       encodeBranch = pipeline
         .<TileBatch>fromGenerator(secondStageName, next -> {
           var writerEnqueuer = writerQueue.threadLocalWriter();
@@ -317,36 +305,12 @@ public class MbtilesWriter {
 
   private void tileWriter(Iterable<TileBatch> tileBatches) throws ExecutionException, InterruptedException {
 
-    if (config.skipIndexCreation()) {
-      db.createTablesWithoutIndexes();
-      if (LOGGER.isInfoEnabled()) {
-        LOGGER.info("Skipping index creation. Add later by executing: {}",
-          String.join(" ; ", db.getManualIndexCreationStatements()));
-      }
-    } else {
-      db.createTablesWithIndexes();
-    }
-
-    var metadata = db.metadata()
-      .setName(mbtilesMetadata.name())
-      .setFormat("pbf")
-      .setDescription(mbtilesMetadata.description())
-      .setAttribution(mbtilesMetadata.attribution())
-      .setVersion(mbtilesMetadata.version())
-      .setType(mbtilesMetadata.type())
-      .setBoundsAndCenter(config.bounds().latLon())
-      .setMinzoom(config.minzoom())
-      .setMaxzoom(config.maxzoom())
-      .setJson(layerStats.getTileStats());
-
-    for (var entry : mbtilesMetadata.planetilerSpecific().entrySet()) {
-      metadata.setMetadata(entry.getKey(), entry.getValue());
-    }
+    archive.initialize(config, tileArchiveMetadata, layerStats);
 
     TileCoord lastTile = null;
     Timer time = null;
     int currentZ = Integer.MIN_VALUE;
-    try (var batchedTileWriter = db.newBatchedTileWriter()) {
+    try (var tileWriter = archive.newTileWriter()) {
       for (TileBatch batch : tileBatches) {
         Queue<TileEncodingResult> encodedTiles = batch.out.get();
         TileEncodingResult encodedTile;
@@ -365,23 +329,22 @@ public class MbtilesWriter {
             time = Timer.start();
             currentZ = z;
           }
-          batchedTileWriter.write(encodedTile);
+          tileWriter.write(encodedTile);
 
           stats.wroteTile(z, encodedTile.tileData() == null ? 0 : encodedTile.tileData().length);
           tilesByZoom[z].inc();
         }
         lastTileWritten.set(lastTile);
       }
-      batchedTileWriter.printStats();
+      tileWriter.printStats();
     }
 
     if (time != null) {
       LOGGER.info("Finished z{} in {}", currentZ, time.stop());
     }
 
-    if (config.optimizeDb()) {
-      db.vacuumAnalyze();
-    }
+
+    archive.finish(config);
   }
 
   private void printTileStats() {

planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileEncodingResult.java

@@ -0,0 +1,43 @@
+package com.onthegomap.planetiler.writer;
+
+import com.onthegomap.planetiler.geo.TileCoord;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.OptionalLong;
+
+public record TileEncodingResult(
+  TileCoord coord,
+  byte[] tileData,
+  /** will always be empty in non-compact mode and might also be empty in compact mode */
+  OptionalLong tileDataHash
+) {
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + Arrays.hashCode(tileData);
+    result = prime * result + Objects.hash(coord, tileDataHash);
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof TileEncodingResult)) {
+      return false;
+    }
+    TileEncodingResult other = (TileEncodingResult) obj;
+    return Objects.equals(coord, other.coord) && Arrays.equals(tileData, other.tileData) &&
+      Objects.equals(tileDataHash, other.tileDataHash);
+  }
+
+  @Override
+  public String toString() {
+    return "TileEncodingResult [coord=" + coord + ", tileData=" + Arrays.toString(tileData) + ", tileDataHash=" +
+      tileDataHash + "]";
+  }
+
+}

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -7,13 +7,11 @@ import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
 import com.onthegomap.planetiler.config.Arguments;
-import com.onthegomap.planetiler.config.MbtilesMetadata;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.GeometryException;
 import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
-import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
 import com.onthegomap.planetiler.reader.SimpleFeature;
 import com.onthegomap.planetiler.reader.SimpleReader;
 import com.onthegomap.planetiler.reader.SourceFeature;
@@ -24,6 +22,8 @@ import com.onthegomap.planetiler.reader.osm.OsmReader;
 import com.onthegomap.planetiler.reader.osm.OsmRelationInfo;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.BuildInfo;
+import com.onthegomap.planetiler.writer.TileArchiveMetadata;
+import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -141,7 +141,8 @@ class PlanetilerTests {
     runner.run(featureGroup, profile, config);
     featureGroup.prepare();
     try (Mbtiles db = Mbtiles.newInMemoryDatabase(config.compactDb())) {
-      MbtilesWriter.writeOutput(featureGroup, db, () -> 0L, new MbtilesMetadata(profile, config.arguments()), config,
+      TileArchiveWriter.writeOutput(featureGroup, db, () -> 0L, new TileArchiveMetadata(profile, config.arguments()),
+        config,
         stats);
       var tileMap = TestUtils.getTileMap(db);
       tileMap.values().forEach(fs -> fs.forEach(f -> f.geometry().validate()));
@@ -267,11 +268,11 @@ class PlanetilerTests {
   void testOverrideMetadata() throws Exception {
     var results = runWithReaderFeatures(
       Map.of(
-        "mbtiles_name", "mbtiles_name",
-        "mbtiles_description", "mbtiles_description",
-        "mbtiles_attribution", "mbtiles_attribution",
-        "mbtiles_version", "mbtiles_version",
-        "mbtiles_type", "mbtiles_type"
+        "mbtiles_name", "override_name",
+        "mbtiles_description", "override_description",
+        "mbtiles_attribution", "override_attribution",
+        "mbtiles_version", "override_version",
+        "mbtiles_type", "override_type"
       ),
       List.of(),
       (sourceFeature, features) -> {
@@ -279,11 +280,11 @@ class PlanetilerTests {
     );
     assertEquals(Map.of(), results.tiles);
     assertSubmap(Map.of(
-      "name", "mbtiles_name",
-      "description", "mbtiles_description",
-      "attribution", "mbtiles_attribution",
-      "version", "mbtiles_version",
-      "type", "mbtiles_type"
+      "name", "override_name",
+      "description", "override_description",
+      "attribution", "override_attribution",
+      "version", "override_version",
+      "type", "override_type"
     ), results.metadata);
   }
 

planetiler-core/src/test/java/com/onthegomap/planetiler/collection/FeatureGroupTest.java

@@ -12,11 +12,11 @@ import com.onthegomap.planetiler.Profile;
 import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.geo.GeometryType;
 import com.onthegomap.planetiler.geo.TileCoord;
-import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
 import com.onthegomap.planetiler.render.RenderedFeature;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.CloseableConusmer;
 import com.onthegomap.planetiler.util.Gzip;
+import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -365,10 +365,10 @@ class FeatureGroupTest {
     put(args1);
     sorter.sort();
     var iter = features.iterator();
-    var tileHash0 = MbtilesWriter.generateContentHash(
+    var tileHash0 = TileArchiveWriter.generateContentHash(
       Gzip.gzip(iter.next().getVectorTileEncoder().encode())
     );
-    var tileHash1 = MbtilesWriter.generateContentHash(
+    var tileHash1 = TileArchiveWriter.generateContentHash(
       Gzip.gzip(iter.next().getVectorTileEncoder().encode())
     );
     if (expectSame) {

planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java

@@ -7,6 +7,7 @@ import com.google.common.math.IntMath;
 import com.onthegomap.planetiler.TestUtils;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.IOException;
 import java.math.RoundingMode;
 import java.sql.SQLException;
@@ -45,7 +46,7 @@ class MbtilesTest {
 
       assertNull(db.getTile(0, 0, 0));
       Set<Mbtiles.TileEntry> expected = new TreeSet<>();
-      try (var writer = db.newBatchedTileWriter()) {
+      try (var writer = db.newTileWriter()) {
         for (int i = 0; i < howMany; i++) {
           var dataHash = i - (i % 2);
           var dataBase = howMany + dataHash;

planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/VerifyTest.java

@@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.onthegomap.planetiler.VectorTile;
 import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.writer.TileEncodingResult;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -45,7 +46,7 @@ class VerifyTest {
   void testValidWithNameAndOneTile() throws IOException {
     mbtiles.createTablesWithIndexes();
     mbtiles.metadata().setName("name");
-    try (var writer = mbtiles.newBatchedTileWriter()) {
+    try (var writer = mbtiles.newTileWriter()) {
       VectorTile tile = new VectorTile();
       tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
         "layer",
@@ -62,7 +63,7 @@ class VerifyTest {
   void testInvalidGeometry() throws IOException {
     mbtiles.createTablesWithIndexes();
     mbtiles.metadata().setName("name");
-    try (var writer = mbtiles.newBatchedTileWriter()) {
+    try (var writer = mbtiles.newTileWriter()) {
       VectorTile tile = new VectorTile();
       tile.addLayerFeatures("layer", List.of(new VectorTile.Feature(
         "layer",

planetiler-core/src/test/java/com/onthegomap/planetiler/stats/PrometheusStatsTest.java

@@ -91,8 +91,8 @@ class PrometheusStatsTest {
     PrometheusStats stats = new PrometheusStats("job");
     stats.wroteTile(0, 10);
     stats.wroteTile(0, 10_000);
-    assertContainsStat("^planetiler_mbtiles_tile_written_bytes_bucket\\{.*le=\"1000\\..* 1", stats);
-    assertContainsStat("^planetiler_mbtiles_tile_written_bytes_bucket\\{.*le=\"10000\\..* 2", stats);
+    assertContainsStat("^planetiler_archive_tile_written_bytes_bucket\\{.*le=\"1000\\..* 1", stats);
+    assertContainsStat("^planetiler_archive_tile_written_bytes_bucket\\{.*le=\"10000\\..* 2", stats);
   }
 
   @Test

planetiler-examples/src/main/java/com/onthegomap/planetiler/examples/ToiletsOverlayLowLevelApi.java

@@ -6,13 +6,14 @@ import com.onthegomap.planetiler.collection.FeatureGroup;
 import com.onthegomap.planetiler.collection.LongLongMap;
 import com.onthegomap.planetiler.collection.LongLongMultimap;
 import com.onthegomap.planetiler.config.Arguments;
-import com.onthegomap.planetiler.config.MbtilesMetadata;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
-import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
+import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.reader.osm.OsmInputFile;
 import com.onthegomap.planetiler.reader.osm.OsmReader;
 import com.onthegomap.planetiler.stats.Stats;
 import com.onthegomap.planetiler.util.FileUtils;
+import com.onthegomap.planetiler.writer.TileArchiveMetadata;
+import com.onthegomap.planetiler.writer.TileArchiveWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -55,7 +56,7 @@ public class ToiletsOverlayLowLevelApi {
     PlanetilerConfig config = PlanetilerConfig.from(Arguments.fromJvmProperties());
 
     // extract mbtiles metadata from profile
-    MbtilesMetadata mbtilesMetadata = new MbtilesMetadata(profile);
+    TileArchiveMetadata tileArchiveMetadata = new TileArchiveMetadata(profile);
 
     // overwrite output each time
     FileUtils.deleteFile(output);
@@ -109,7 +110,12 @@ public class ToiletsOverlayLowLevelApi {
 
     // then process rendered features, grouped by tile, encoding them into binary vector tile format
     // and writing to the output mbtiles file.
-    MbtilesWriter.writeOutput(featureGroup, output, mbtilesMetadata, config, stats);
+    try (Mbtiles db = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
+      TileArchiveWriter.writeOutput(featureGroup, db, () -> FileUtils.fileSize(output), tileArchiveMetadata, config,
+        stats);
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to write to " + output, e);
+    }
 
     // dump recorded timings at the end
     stats.printSummary();