scaffold mbtiles

src/main/java/com/onthegomap/flatmap/GeoUtils.java

@@ -83,4 +83,17 @@ public class GeoUtils {
     long y = (long) (worldY * QUANTIZED_WORLD_SIZE);
     return (x << 32) | y;
   }
+
+  public static int z(int key) {
+    int result = key >> 28;
+    return result < 0 ? 16 + result : result;
+  }
+
+  public static int x(int key) {
+    return (key >> 14) & ((1 << 14) - 1);
+  }
+
+  public static int y(int key) {
+    return (key) & ((1 << 14) - 1);
+  }
 }

src/main/java/com/onthegomap/flatmap/MbtilesWriter.java

@@ -1,11 +1,100 @@
 package com.onthegomap.flatmap;
 
+import static com.onthegomap.flatmap.GeoUtils.x;
+import static com.onthegomap.flatmap.GeoUtils.y;
+import static com.onthegomap.flatmap.GeoUtils.z;
+
 import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
+import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures;
+import com.onthegomap.flatmap.stats.Stats;
+import com.onthegomap.flatmap.worker.Topology;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.zip.GZIPOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class MbtilesWriter {
 
-  public static void writeOutput(MergeSortFeatureMap features, File output, FlatMapConfig threads) {
+  private final AtomicLong featuresProcessed = new AtomicLong(0);
+  private final AtomicLong memoizedTiles = new AtomicLong(0);
+  private final AtomicLong tiles = new AtomicLong(0);
+  private final Stats stats;
+
+  private MbtilesWriter(Stats stats) {
+    this.stats = stats;
+  }
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class);
+
+  private static record RenderedTile(int tile, byte[] contents) {
+
+  }
+
+  public static void writeOutput(long featureCount, MergeSortFeatureMap features, File output, FlatMapConfig config) {
+    Stats stats = config.stats();
+    output.delete();
+    MbtilesWriter writer = new MbtilesWriter(config.stats());
+
+    var topology = Topology.readFromIterator("mbtiles_reader", stats, features.getAll())
+      .addBuffer("mbtiles_reader_queue", 50_000, 1_000)
+      .addWorker("mbtiles_encoder", config.threads(), writer::tileEncoder)
+      .addBuffer("mbtiles_writer_queue", 50_000, 1_000)
+      .sinkTo("mbtiles_writer", 1, writer::tileWriter);
+
+    var loggers = new ProgressLoggers("mbtiles")
+      .addRatePercentCounter("features", featureCount, writer.featuresProcessed)
+      .addRateCounter("tiles", writer.tiles)
+      .addFileSize(output)
+      .add(" features ").addFileSize(features::getStorageSize)
+      .addProcessStats()
+      .addTopologyStats(topology);
+
+    topology.awaitAndLog(loggers, config.logIntervalSeconds());
+  }
+
+  public void tileEncoder(Supplier<TileFeatures> prev, Consumer<RenderedTile> next) throws Exception {
+    MergeSortFeatureMap.TileFeatures tileFeatures, last = null;
+    byte[] lastBytes = null, lastEncoded = null;
+    while ((tileFeatures = prev.get()) != null) {
+      featuresProcessed.addAndGet(tileFeatures.getNumFeatures());
+      byte[] bytes, encoded;
+      int zoom = z(tileFeatures.getTileId());
+      if (tileFeatures.hasSameContents(last)) {
+        bytes = lastBytes;
+        encoded = lastEncoded;
+        memoizedTiles.incrementAndGet();
+      } else {
+        VectorTile en = tileFeatures.getTile();
+        encoded = en.encode();
+        bytes = gzipCompress(encoded);
+        last = tileFeatures;
+        lastEncoded = encoded;
+        lastBytes = bytes;
+        if (encoded.length > 1_000_000) {
+          LOGGER.warn("Tile " + zoom + "/" + x(tileFeatures.getTileId()) + "/" + y(tileFeatures.getTileId()) + " "
+            + encoded.length / 1024 + "kb uncompressed");
+        }
+      }
+      stats.encodedTile(zoom, encoded.length);
+      next.accept(new RenderedTile(tileFeatures.getTileId(), bytes));
+    }
+  }
+
+  private void tileWriter(Supplier<RenderedTile> prev) throws Exception {
+
+  }
+
+  private static byte[] gzipCompress(byte[] uncompressedData) throws IOException {
+    var bos = new ByteArrayOutputStream(uncompressedData.length);
+    try (var gzipOS = new GZIPOutputStream(bos)) {
+      gzipOS.write(uncompressedData);
+    }
+    return bos.toByteArray();
   }
 
 }

src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java

@@ -89,7 +89,8 @@ public class OpenMapTilesMain {
 
     try (var osmReader = new OpenStreetMapReader(osmInputFile, nodeLocations, stats)) {
       stats.time("osm_pass1", () -> osmReader.pass1(config));
-      stats.time("osm_pass2", () -> osmReader.pass2(renderer, featureMap, config));
+      stats
+        .time("osm_pass2", () -> osmReader.pass2(renderer, featureMap, Math.max(threads / 4, 1), threads - 1, config));
     }
 
     LOGGER.info("Deleting node.db to make room for mbtiles");

src/main/java/com/onthegomap/flatmap/ProgressLoggers.java

@@ -61,4 +61,8 @@ public class ProgressLoggers {
       .addQueueStats(topology.inputQueue())
       .addThreadPoolStats(topology.name(), topology.worker());
   }
+
+  public ProgressLoggers add(String s) {
+    return this;
+  }
 }

src/main/java/com/onthegomap/flatmap/VectorTile.java

@@ -0,0 +1,7 @@
+package com.onthegomap.flatmap;
+
+public class VectorTile {
+
+  public byte[] encode() {
+  }
+}

src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java

@@ -1,8 +1,10 @@
 package com.onthegomap.flatmap.collections;
 
 import com.onthegomap.flatmap.RenderedFeature;
+import com.onthegomap.flatmap.VectorTile;
 import com.onthegomap.flatmap.stats.Stats;
 import java.nio.file.Path;
+import java.util.Iterator;
 import java.util.function.Consumer;
 
 public class MergeSortFeatureMap implements Consumer<RenderedFeature> {
@@ -22,4 +24,27 @@ public class MergeSortFeatureMap implements Consumer<RenderedFeature> {
   public long getStorageSize() {
     return 0;
   }
+
+  public Iterator<TileFeatures> getAll() {
+    return null;
+  }
+
+  public static class TileFeatures {
+
+    public long getNumFeatures() {
+      return 0;
+    }
+
+    public int getTileId() {
+      return 0;
+    }
+
+    public boolean hasSameContents(TileFeatures other) {
+      return false;
+    }
+
+    public VectorTile getTile() {
+      return null;
+    }
+  }
 }

src/main/java/com/onthegomap/flatmap/stats/Stats.java

@@ -14,6 +14,8 @@ public interface Stats {
 
   void stopTimer(String name);
 
+  void encodedTile(int zoom, int length);
+
   class InMemory implements Stats {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(InMemory.class);
@@ -40,5 +42,10 @@ public interface Stats {
     public void stopTimer(String name) {
 
     }
+
+    @Override
+    public void encodedTile(int zoom, int length) {
+
+    }
   }
 }

src/main/java/com/onthegomap/flatmap/worker/Topology.java

@@ -2,6 +2,7 @@ package com.onthegomap.flatmap.worker;
 
 import com.onthegomap.flatmap.ProgressLoggers;
 import com.onthegomap.flatmap.stats.Stats;
+import java.util.Iterator;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -24,6 +25,14 @@ public record Topology<T>(
     return fromGenerator(name, stats, producer, 1);
   }
 
+  public static <T> Bufferable<?, T> readFromIterator(String name, Stats stats, Iterator<T> iter) {
+    return fromGenerator(name, stats, next -> {
+      while (iter.hasNext()) {
+        next.accept(iter.next());
+      }
+    }, 1);
+  }
+
   public static <T> Builder<?, T> readFromQueue(Stats stats, WorkQueue<T> input) {
     return new Builder<>(input, stats);
   }