begin wireframing

.idea/codeStyles/Project.xml

@@ -140,6 +140,7 @@
       </indentOptions>
     </codeStyleSettings>
     <codeStyleSettings language="JAVA">
+      <option name="RIGHT_MARGIN" value="120" />
       <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
       <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
       <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />

src/main/java/com/onthegomap/flatmap/Arguments.java

@@ -1,7 +1,6 @@
-package com.onthegomap.flatmap.profiles;
+package com.onthegomap.flatmap;
 
-import com.onthegomap.flatmap.GeoUtils;
-import com.onthegomap.flatmap.OsmInputFile;
+import com.onthegomap.flatmap.stats.Stats;
 import java.io.File;
 import java.util.Arrays;
 import java.util.List;
@@ -75,4 +74,9 @@ public class Arguments {
     LOGGER.info("num threads: " + threads);
     return threads;
   }
+
+  public Stats getStats() {
+    // TODO
+    return null;
+  }
 }

src/main/java/com/onthegomap/flatmap/FeatureRenderer.java

@@ -0,0 +1,18 @@
+package com.onthegomap.flatmap;
+
+import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
+import com.onthegomap.flatmap.stats.Stats;
+import com.onthegomap.flatmap.worker.Worker.WorkerSink;
+
+public class FeatureRenderer {
+
+  public FeatureRenderer(MergeSortFeatureMap featureMap, Stats stats) {
+
+  }
+
+  public WorkerSink<RenderedFeature> newWriterQueue(String name) {
+  }
+
+  public void renderFeature(RenderableFeature renderable) {
+  }
+}

src/main/java/com/onthegomap/flatmap/GeoUtils.java

@@ -1,5 +1,10 @@
 package com.onthegomap.flatmap;
 
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.impl.PackedCoordinateSequence;
+import org.locationtech.jts.geom.util.GeometryTransformer;
+
 public class GeoUtils {
 
   private static final double DEGREES_TO_RADIANS = Math.PI / 180;
@@ -50,4 +55,22 @@ public class GeoUtils {
     double sin = Math.sin(lat * DEGREES_TO_RADIANS);
     return 0.5 - 0.25 * Math.log((1 + sin) / (1 - sin)) / Math.PI;
   }
+
+  public static final GeometryTransformer ProjectWorldCoords = new GeometryTransformer() {
+    @Override
+    protected CoordinateSequence transformCoordinates(CoordinateSequence coords, Geometry parent) {
+      if (coords.getDimension() != 2) {
+        throw new IllegalArgumentException("Dimension must be 2, was: " + coords.getDimension());
+      }
+      if (coords.getMeasures() != 0) {
+        throw new IllegalArgumentException("Measures must be 0, was: " + coords.getMeasures());
+      }
+      CoordinateSequence copy = new PackedCoordinateSequence.Double(coords.size(), 2, 0);
+      for (int i = 0; i < coords.size(); i++) {
+        copy.setOrdinate(i, 0, getWorldX(coords.getX(i)));
+        copy.setOrdinate(i, 1, getWorldY(coords.getY(i)));
+      }
+      return copy;
+    }
+  };
 }

src/main/java/com/onthegomap/flatmap/MbtilesWriter.java

@@ -0,0 +1,11 @@
+package com.onthegomap.flatmap;
+
+import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
+import java.io.File;
+
+public class MbtilesWriter {
+
+  public static void writeOutput(MergeSortFeatureMap features, File output, int threads) {
+  }
+
+}

src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java

@@ -0,0 +1,99 @@
+package com.onthegomap.flatmap;
+
+import com.onthegomap.flatmap.collections.MergeSortFeatureMap;
+import com.onthegomap.flatmap.profiles.OpenMapTilesProfile;
+import com.onthegomap.flatmap.reader.NaturalEarthReader;
+import com.onthegomap.flatmap.reader.OpenStreetMapReader;
+import com.onthegomap.flatmap.reader.ShapefileReader;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OpenMapTilesMain {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(OpenMapTilesMain.class);
+
+  public static void main(String[] args) throws IOException {
+    Arguments arguments = new Arguments(args);
+    var stats = arguments.getStats();
+    stats.startTimer("import");
+    LOGGER.info("Arguments:");
+    OsmInputFile osmInputFile = new OsmInputFile(
+      arguments.inputFile("input", "OSM input file", "./data/sources/massachusetts-latest.osm.pbf"));
+    File centerlines = arguments
+      .inputFile("centerline", "lake centerlines input", "./data/sources/lake_centerline.shp.zip");
+    File naturalEarth = arguments.inputFile("natural_earth", "natural earth input",
+      "./data/sources/natural_earth_vector.sqlite.zip");
+    File waterPolygons = arguments.inputFile("water_polygons", "water polygons input",
+      "./data/sources/water-polygons-split-3857.zip");
+    double[] bounds = arguments.bounds("bounds", "bounds", osmInputFile);
+    int threads = arguments.threads();
+    Path tmpDir = arguments.file("tmpdir", "temp directory", "./data/tmp").toPath();
+    boolean fetchWikidata = arguments.get("fetch_wikidata", "fetch wikidata translations", false);
+    boolean useWikidata = arguments.get("use_wikidata", "use wikidata translations", true);
+    File wikidataNamesFile = arguments.file("wikidata_cache", "wikidata cache file",
+      "./data/sources/wikidata_names.json");
+    File output = arguments.file("output", "mbtiles output file", "./massachusetts.mbtiles");
+    List<String> languages = arguments.get("name_languages", "languages to use",
+      "en,ru,ar,zh,ja,ko,fr,de,fi,pl,es,be,br,he".split(","));
+
+    LOGGER.info("Building OpenMapTiles profile into " + output + " in these phases:");
+    if (fetchWikidata) {
+      LOGGER.info("- [wikidata] Fetch OpenStreetMap element name translations from wikidata");
+    }
+    LOGGER.info("- [lake_centerlines] Extract lake centerlines");
+    LOGGER.info("- [water_polygons] Process ocean polygons");
+    LOGGER.info("- [natural_earth] Process natural earth features");
+    LOGGER.info("- [osm_pass1] Pre-process OpenStreetMap input (store node locations then relation members)");
+    LOGGER.info("- [osm_pass2] Process OpenStreetMap nodes, ways, then relations");
+    LOGGER.info("- [sort] Sort rendered features by tile ID");
+    LOGGER.info("- [mbtiles] Encode each tile and write to " + output);
+
+    var translations = Translations.defaultProvider(languages);
+    var profile = new OpenMapTilesProfile();
+
+    if (fetchWikidata) {
+      stats.time("wikidata",
+        () -> Wikidata.fetch(osmInputFile, wikidataNamesFile, threads, profile, stats));
+    }
+    if (useWikidata) {
+      translations.addTranslationProvider(Wikidata.load(wikidataNamesFile));
+    }
+
+    FileUtils.forceMkdir(tmpDir.toFile());
+    File nodeDb = tmpDir.resolve("node.db").toFile();
+    Path featureDb = tmpDir.resolve("feature.db");
+    MergeSortFeatureMap featureMap = new MergeSortFeatureMap(featureDb, stats);
+    FeatureRenderer renderer = new FeatureRenderer(featureMap, stats);
+
+    stats.time("lake_centerlines", () ->
+      new ShapefileReader("EPSG:3857", centerlines, stats).process(renderer, profile, threads));
+    stats.time("water_polygons", () ->
+      new ShapefileReader(waterPolygons, stats).process(renderer, profile, threads));
+    stats.time("natural_earth", () ->
+      new NaturalEarthReader(naturalEarth, tmpDir.resolve("natearth.sqlite").toFile(), stats)
+        .process(renderer, profile, threads));
+
+    try (var osmReader = new OpenStreetMapReader(osmInputFile, nodeDb, stats)) {
+      stats.time("osm_pass1", () -> osmReader.pass1(profile, threads));
+      stats.time("osm_pass2", () -> osmReader.pass2(renderer, profile, threads));
+    }
+
+    LOGGER.info("Deleting node.db to make room for mbtiles");
+    profile.release();
+    nodeDb.delete();
+
+    stats.time("sort", featureMap::sort);
+    stats.time("mbtiles", () -> MbtilesWriter.writeOutput(featureMap, output, threads));
+
+    stats.stopTimer("import");
+
+    LOGGER.info("FINISHED!");
+
+    stats.printSummary();
+  }
+}

src/main/java/com/onthegomap/flatmap/Profile.java

@@ -0,0 +1,14 @@
+package com.onthegomap.flatmap;
+
+import com.graphhopper.reader.ReaderRelation;
+import com.onthegomap.flatmap.reader.OpenStreetMapReader.RelationInfo;
+import java.util.List;
+
+public interface Profile {
+
+  List<RelationInfo> preprocessOsmRelation(ReaderRelation relation);
+
+  void processFeature(SourceFeature sourceFeature, RenderableFeatures features);
+
+  void release();
+}

src/main/java/com/onthegomap/flatmap/ProgressLoggers.java

@@ -0,0 +1,19 @@
+package com.onthegomap.flatmap;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongSupplier;
+
+public class ProgressLoggers {
+
+  public ProgressLoggers(String name) {
+
+  }
+
+  public ProgressLoggers addRatePercentCounter(String name, long total, AtomicLong value) {
+    return addRatePercentCounter(name, total, value::get);
+  }
+
+  public ProgressLoggers addRatePercentCounter(String name, long total, LongSupplier getValue) {
+    return this;
+  }
+}

src/main/java/com/onthegomap/flatmap/RenderableFeature.java

@@ -0,0 +1,12 @@
+package com.onthegomap.flatmap;
+
+import org.opengis.geometry.Geometry;
+
+public class RenderableFeature {
+
+  public Geometry getGeometry() {
+  }
+  // layer
+  // attrs
+  // geometry
+}

src/main/java/com/onthegomap/flatmap/RenderableFeatures.java

@@ -0,0 +1,10 @@
+package com.onthegomap.flatmap;
+
+public class RenderableFeatures {
+
+  public Iterable<RenderableFeature> all() {
+  }
+
+  public void reset(SourceFeature sourceFeature) {
+  }
+}

src/main/java/com/onthegomap/flatmap/RenderedFeature.java

@@ -0,0 +1,5 @@
+package com.onthegomap.flatmap;
+
+public class RenderedFeature {
+
+}

src/main/java/com/onthegomap/flatmap/SourceFeature.java

@@ -0,0 +1,13 @@
+package com.onthegomap.flatmap;
+
+import org.locationtech.jts.geom.Geometry;
+
+public class SourceFeature {
+
+  public Geometry getGeometry() {
+  }
+  // props
+  // lazy geometry
+  // lazy centroid
+  // lazy area
+}

src/main/java/com/onthegomap/flatmap/Translations.java

@@ -0,0 +1,24 @@
+package com.onthegomap.flatmap;
+
+import com.graphhopper.reader.ReaderElement;
+import com.onthegomap.flatmap.Wikidata.WikidataTranslations;
+import java.util.List;
+import java.util.Map;
+
+public class Translations {
+
+  public static Translations defaultProvider(List<String> languages) {
+    // TODO
+    return new Translations();
+  }
+
+  public void addTranslationProvider(WikidataTranslations load) {
+    // TODO
+  }
+
+
+  public interface TranslationProvider {
+
+    Map<String, String> getNameTranslations(ReaderElement elem);
+  }
+}

src/main/java/com/onthegomap/flatmap/Wikidata.java

@@ -0,0 +1,29 @@
+package com.onthegomap.flatmap;
+
+import com.graphhopper.reader.ReaderElement;
+import com.onthegomap.flatmap.profiles.Profile;
+import com.onthegomap.flatmap.stats.Stats;
+import java.io.File;
+import java.util.Map;
+
+public class Wikidata {
+
+  public static void fetch(OsmInputFile infile, File outfile, int threads, Profile profile,
+    Stats stats) {
+    // TODO
+  }
+
+  public static WikidataTranslations load(File namesFile) {
+    // TODO
+    return null;
+  }
+
+  public static class WikidataTranslations implements Translations.TranslationProvider {
+
+    @Override
+    public Map<String, String> getNameTranslations(ReaderElement elem) {
+      // TODO
+      return null;
+    }
+  }
+}

src/main/java/com/onthegomap/flatmap/collections/IntRange.java

@@ -0,0 +1,5 @@
+package com.onthegomap.flatmap.collections;
+
+public class IntRange {
+
+}

src/main/java/com/onthegomap/flatmap/collections/LongLongMap.java

@@ -0,0 +1,5 @@
+package com.onthegomap.flatmap.collections;
+
+public interface LongLongMap {
+
+}

src/main/java/com/onthegomap/flatmap/collections/LongLongMultimap.java

@@ -0,0 +1,5 @@
+package com.onthegomap.flatmap.collections;
+
+public interface LongLongMultimap {
+
+}

src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java

@@ -0,0 +1,14 @@
+package com.onthegomap.flatmap.collections;
+
+import com.onthegomap.flatmap.stats.Stats;
+import java.nio.file.Path;
+
+public class MergeSortFeatureMap {
+
+  public MergeSortFeatureMap(Path featureDb, Stats stats) {
+
+  }
+
+  public void sort() {
+  }
+}

src/main/java/com/onthegomap/flatmap/profiles/OpenMapTilesProfile.java

@@ -1,9 +1,11 @@
 package com.onthegomap.flatmap.profiles;
 
 
-import com.graphhopper.util.StopWatch;
-import com.onthegomap.flatmap.OsmInputFile;
-import java.io.File;
+import com.graphhopper.reader.ReaderRelation;
+import com.onthegomap.flatmap.Profile;
+import com.onthegomap.flatmap.RenderableFeature;
+import com.onthegomap.flatmap.SourceFeature;
+import com.onthegomap.flatmap.reader.OpenStreetMapReader.RelationInfo;
 import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -12,30 +14,17 @@ public class OpenMapTilesProfile implements Profile {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(OpenMapTilesProfile.class);
 
-  public static void main(String[] args) {
-    StopWatch watch = new StopWatch().start();
-    Arguments arguments = new Arguments(args);
-    OsmInputFile osmInputFile = new OsmInputFile(arguments.inputFile("input", "OSM input file",
-      "./data/sources/massachusetts-latest.osm.pbf"));
-    File centerlines = arguments.inputFile("centerline", "lake centerlines input",
-      "./data/sources/lake_centerline.shp.zip");
-    File naturalEarth = arguments.inputFile("natural_earth", "natural earth input",
-      "./data/sources/natural_earth_vector.sqlite.zip");
-    File waterPolygons = arguments.inputFile("water_polygons", "water polygons input",
-      "./data/sources/water-polygons-split-3857.zip");
-    double[] bounds = arguments.bounds("bounds", "bounds", osmInputFile);
-    int threads = arguments.threads();
-    File tmpDir = arguments.file("tmpdir", "temp directory", "./data/tmp");
-    boolean fetchWikidata = arguments.get("fetch_wikidata", "fetch wikidata translations", false);
-    boolean useWikidata = arguments.get("use_wikidata", "use wikidata translations", true);
-    File wikidataNamesFile = arguments.file("wikidata_cache", "wikidata cache file",
-      "./data/sources/wikidata_names.json");
-    String output = arguments.get("output", "mbtiles output file", "./massachusetts.mbtiles");
-    List<String> languages = arguments.get("name_languages", "languages to use",
-      "en,ru,ar,zh,ja,ko,fr,de,fi,pl,es,be,br,he".split(","));
+  @Override
+  public void release() {
+  }
 
-    var profile = new OpenMapTilesProfile();
+  @Override
+  public List<RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
+    return null;
+  }
+
+  @Override
+  public void processFeature(SourceFeature sourceFeature, RenderableFeature features) {
 
-    LOGGER.info("FINISHED! " + watch.stop());
   }
 }

src/main/java/com/onthegomap/flatmap/profiles/Profile.java

@@ -1,5 +0,0 @@
-package com.onthegomap.flatmap.profiles;
-
-public interface Profile {
-
-}

src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java

@@ -0,0 +1,16 @@
+package com.onthegomap.flatmap.reader;
+
+import com.onthegomap.flatmap.SourceFeature;
+import com.onthegomap.flatmap.stats.Stats;
+import java.io.File;
+
+public class NaturalEarthReader implements Reader {
+
+  public NaturalEarthReader(File input, File tmpFile, Stats stats) {
+  }
+
+  @Override
+  public SourceFeature getNext() {
+    return null;
+  }
+}

src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java

@@ -0,0 +1,30 @@
+package com.onthegomap.flatmap.reader;
+
+import com.onthegomap.flatmap.FeatureRenderer;
+import com.onthegomap.flatmap.OsmInputFile;
+import com.onthegomap.flatmap.profiles.OpenMapTilesProfile;
+import com.onthegomap.flatmap.stats.Stats;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+
+public class OpenStreetMapReader implements Closeable {
+
+  public OpenStreetMapReader(OsmInputFile osmInputFile, File nodeDb, Stats stats) {
+  }
+
+  public void pass1(OpenMapTilesProfile profile, int threads) {
+  }
+
+  public void pass2(FeatureRenderer renderer, OpenMapTilesProfile profile, int threads) {
+  }
+
+  @Override
+  public void close() throws IOException {
+
+  }
+
+  public static class RelationInfo {
+
+  }
+}

src/main/java/com/onthegomap/flatmap/reader/Reader.java

@@ -0,0 +1,85 @@
+package com.onthegomap.flatmap.reader;
+
+import com.onthegomap.flatmap.FeatureRenderer;
+import com.onthegomap.flatmap.Profile;
+import com.onthegomap.flatmap.ProgressLoggers;
+import com.onthegomap.flatmap.RenderableFeature;
+import com.onthegomap.flatmap.RenderableFeatures;
+import com.onthegomap.flatmap.SourceFeature;
+import com.onthegomap.flatmap.stats.Stats;
+import com.onthegomap.flatmap.worker.Worker;
+import com.onthegomap.flatmap.worker.Worker.WorkerSource;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import org.locationtech.jts.geom.Envelope;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class Reader {
+
+  private final Stats stats;
+  private final Envelope envelope;
+  private Logger LOGGER = LoggerFactory.getLogger(getClass());
+
+  public Reader(Stats stats, Envelope envelope) {
+    this.stats = stats;
+    this.envelope = envelope;
+  }
+
+  protected void log(String message) {
+    LOGGER.info("[" + getName() + "] " + message);
+  }
+
+  protected abstract String getName();
+
+  public final void process(FeatureRenderer renderer, Profile profile, int threads) {
+    threads = Math.max(threads, 1);
+    long featureCount = getCount();
+    AtomicLong featuresRead = new AtomicLong(0);
+    log("Reading with " + threads + " threads");
+    try (
+      var source = open(getName() + "-reader");
+      var sink = renderer.newWriterQueue(getName() + "-writer")
+    ) {
+      var worker = new Worker(getName() + "-processor", stats, threads, i -> {
+        SourceFeature sourceFeature;
+        RenderableFeatures features = new RenderableFeatures();
+        var sourceQueue = source.queue();
+        while ((sourceFeature = sourceQueue.getNext()) != null) {
+          featuresRead.incrementAndGet();
+          features.reset(sourceFeature);
+          if (sourceFeature.getGeometry().getEnvelopeInternal().intersects(envelope)) {
+            profile.processFeature(sourceFeature, features);
+            for (RenderableFeature renderable : features.all()) {
+              renderer.renderFeature(renderable);
+            }
+          }
+        }
+      });
+
+//      TODO:
+//      -where should this go ?
+//        -should the renderer hold a reusable feature writer / queue ?
+
+      var loggers = new ProgressLoggers(getName())
+        .addRatePercentCounter("read", featureCount, featuresRead)
+        .addRateCounter("write", featuresWritten)
+        .addFileSize(featureMap::getStorageSize)
+        .addProcessStats()
+        .addThreadPoolStats("read", getName() + "-reader")
+        .addQueueStats(readerQueue)
+        .addThreadPoolStats("process", worker)
+        .addQueueStats(toWrite)
+        .addThreadPoolStats("write", writer);
+
+      worker.awaitAndLong(loggers);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  protected abstract long getCount();
+
+  protected abstract WorkerSource<SourceFeature> open(String workerName);
+
+}

src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java

@@ -0,0 +1,19 @@
+package com.onthegomap.flatmap.reader;
+
+import com.onthegomap.flatmap.FeatureRenderer;
+import com.onthegomap.flatmap.Profile;
+import com.onthegomap.flatmap.stats.Stats;
+import java.io.File;
+
+public class ShapefileReader extends Reader {
+
+  public ShapefileReader(String sourceProjection, File input, Stats stats) {
+    super(stats);
+  }
+
+  public ShapefileReader(File input, Stats stats) {
+  }
+
+  public void process(FeatureRenderer renderer, Profile profile, int threads) {
+  }
+}

src/main/java/com/onthegomap/flatmap/stats/Stats.java

@@ -2,4 +2,11 @@ package com.onthegomap.flatmap.stats;
 
 public interface Stats {
 
+  void time(String name, Runnable task);
+
+  void printSummary();
+
+  void startTimer(String name);
+
+  void stopTimer(String name);
 }

src/main/java/com/onthegomap/flatmap/worker/Sink.java

@@ -0,0 +1,6 @@
+package com.onthegomap.flatmap.worker;
+
+public interface Sink<T> {
+
+  void process(T item);
+}

src/main/java/com/onthegomap/flatmap/worker/Source.java

@@ -0,0 +1,6 @@
+package com.onthegomap.flatmap.worker;
+
+public interface Source<T> {
+
+  T getNext();
+}

src/main/java/com/onthegomap/flatmap/worker/WorkQueue.java

@@ -0,0 +1,26 @@
+package com.onthegomap.flatmap.worker;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+public class WorkQueue<T> implements Closeable, Supplier<T>, Consumer<T> {
+
+
+  @Override
+  public void close() throws IOException {
+
+  }
+
+  @Override
+  public void accept(T t) {
+
+  }
+
+  @Override
+  public T get() {
+    return null;
+  }
+}
+

src/main/java/com/onthegomap/flatmap/worker/Worker.java

@@ -0,0 +1,11 @@
+package com.onthegomap.flatmap.worker;
+
+import com.onthegomap.flatmap.stats.Stats;
+import java.util.function.Consumer;
+
+public class Worker {
+
+  public Worker(String name, Stats stats, int threads, Consumer<Integer> task) {
+
+  }
+}