break out into subpoms

core/pom.xml

@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>flatmap-core</artifactId>
+
+  <parent>
+    <groupId>com.onthegomap</groupId>
+    <artifactId>flatmap-parent</artifactId>
+    <version>0.1-SNAPSHOT</version>
+  </parent>
+
+  <properties>
+    <graphhopper.version>2.3</graphhopper.version>
+    <geotools.version>25.0</geotools.version>
+    <junit.version>5.7.1</junit.version>
+    <jackson.version>2.12.3</jackson.version>
+    <log4j.version>2.14.1</log4j.version>
+    <prometheus.version>0.11.0</prometheus.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.graphhopper</groupId>
+      <artifactId>graphhopper-reader-osm</artifactId>
+      <version>${graphhopper.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.locationtech.jts</groupId>
+      <artifactId>jts-core</artifactId>
+      <version>1.18.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.geotools</groupId>
+      <artifactId>gt-shapefile</artifactId>
+      <version>${geotools.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.geotools</groupId>
+      <artifactId>gt-epsg-hsql</artifactId>
+      <version>${geotools.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.xerial</groupId>
+      <artifactId>sqlite-jdbc</artifactId>
+      <version>3.34.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.mapdb</groupId>
+      <artifactId>mapdb</artifactId>
+      <version>3.0.8</version>
+    </dependency>
+    <dependency>
+      <groupId>org.msgpack</groupId>
+      <artifactId>msgpack-core</artifactId>
+      <version>0.8.22</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>1.7.30</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_common</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_pushgateway</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_hotspot</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>3.9.0</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <testResources>
+      <testResource>
+        <directory>src/test/resources</directory>
+        <filtering>true</filtering>
+      </testResource>
+    </testResources>
+  </build>
+</project>

core/src/main/java/com/onthegomap/flatmap/FeatureMerge.java

@@ -121,7 +121,7 @@ public class FeatureMerge {
       current.addPoint(lastX, lastY);
     }
 
-    if (!current.isEmpty()) {
+    if (current.size() >= 2) {
       output.add(GeoUtils.JTS_FACTORY.createLineString(current));
     }
   }

core/src/main/java/com/onthegomap/flatmap/Profile.java

@@ -7,14 +7,19 @@ import java.util.List;
 
 public interface Profile {
 
-  List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation);
+  default List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
+    return null;
+  }
 
   void processFeature(SourceFeature sourceFeature, FeatureCollector features);
 
-  void release();
+  default void release() {
+  }
 
-  List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer, int zoom,
+  default List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer, int zoom,
-    List<VectorTileEncoder.Feature> items) throws GeometryException;
+    List<VectorTileEncoder.Feature> items) throws GeometryException {
+    return items;
+  }
 
   String name();
 
@@ -30,23 +35,17 @@ public interface Profile {
     return null;
   }
 
+  default boolean isOverlay() {
+    return false;
+  }
+
   class NullProfile implements Profile {
 
-    @Override
-    public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
-      return null;
-    }
-
     @Override
     public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
 
     }
 
-    @Override
-    public void release() {
-
-    }
-
     @Override
     public List<VectorTileEncoder.Feature> postProcessLayerFeatures(String layer, int zoom,
       List<VectorTileEncoder.Feature> items) {

core/src/main/java/com/onthegomap/flatmap/SourceFeature.java

@@ -15,13 +15,15 @@ public abstract class SourceFeature {
   private final String source;
   private final String sourceLayer;
   private final List<OpenStreetMapReader.RelationInfo> relationInfos;
+  private final long id;
 
   protected SourceFeature(Map<String, Object> properties, String source, String sourceLayer,
-    List<OpenStreetMapReader.RelationInfo> relationInfos) {
+    List<OpenStreetMapReader.RelationInfo> relationInfos, long id) {
     this.properties = properties;
     this.source = source;
     this.sourceLayer = sourceLayer;
     this.relationInfos = relationInfos;
+    this.id = id;
   }
 
   public abstract Geometry latLonGeometry() throws GeometryException;
@@ -168,4 +170,8 @@ public abstract class SourceFeature {
     }
     return result == null ? List.of() : result;
   }
+
+  public final long id() {
+    return id;
+  }
 }

core/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java

@@ -20,6 +20,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import org.msgpack.core.MessageBufferPacker;
@@ -329,14 +330,14 @@ public final class FeatureGroup implements Consumer<FeatureSort.Entry>, Iterable
     private final List<FeatureSort.Entry> entries = new ArrayList<>();
     private LongLongHashMap counts = null;
     private byte layer = Byte.MAX_VALUE;
-    private long numFeaturesProcessed = 0;
+    private final AtomicLong numFeaturesProcessed = new AtomicLong(0);
 
     public TileFeatures(int tile) {
       this.tile = TileCoord.decode(tile);
     }
 
     public long getNumFeatures() {
-      return numFeaturesProcessed;
+      return numFeaturesProcessed.get();
     }
 
     public TileCoord coord() {
@@ -395,7 +396,7 @@ public final class FeatureGroup implements Consumer<FeatureSort.Entry>, Iterable
 
     @Override
     public void accept(FeatureSort.Entry entry) {
-      numFeaturesProcessed++;
+      numFeaturesProcessed.incrementAndGet();
       long sortKey = entry.sortKey();
       if (extractHasGroupFromSortKey(sortKey)) {
         byte thisLayer = extractLayerIdFromSortKey(sortKey);

core/src/main/java/com/onthegomap/flatmap/read/NaturalEarthReader.java

@@ -104,6 +104,7 @@ public class NaturalEarthReader extends Reader {
   @Override
   public Topology.SourceStep<ReaderFeature> read() {
     return next -> {
+      long id = 0;
       var tables = tableNames();
       for (int i = 0; i < tables.size(); i++) {
         String table = tables.get(i);
@@ -127,7 +128,8 @@ public class NaturalEarthReader extends Reader {
                 continue;
               }
               Geometry latLonGeometry = GeoUtils.wkbReader.read(geometry);
-              ReaderFeature readerGeometry = new ReaderFeature(latLonGeometry, column.length - 1, sourceName, table);
+              ReaderFeature readerGeometry = new ReaderFeature(latLonGeometry, column.length - 1, sourceName, table,
+                id);
               for (int c = 0; c < column.length; c++) {
                 if (c != geometryColumn) {
                   Object value = rs.getObject(c + 1);

core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java

@@ -280,15 +280,13 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
     final boolean polygon;
     final boolean line;
     final boolean point;
-    final long osmId;
 
     public ProxyFeature(ReaderElement elem, boolean point, boolean line, boolean polygon,
       List<RelationInfo> relationInfo) {
-      super(ReaderElementUtils.getProperties(elem), name, null, relationInfo);
+      super(ReaderElementUtils.getProperties(elem), name, null, relationInfo, elem.getId());
       this.point = point;
       this.line = line;
       this.polygon = polygon;
-      this.osmId = elem.getId();
     }
 
     private Geometry latLonGeom;
@@ -349,7 +347,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
 
     @Override
     public String toString() {
-      return "OsmNode[" + osmId + ']';
+      return "OsmNode[" + id() + ']';
     }
   }
 
@@ -375,7 +373,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
         CoordinateSequence coords = nodeCache.getWayGeometry(nodeIds);
         return GeoUtils.JTS_FACTORY.createLineString(coords);
       } catch (IllegalArgumentException e) {
-        throw new GeometryException("osm_invalid_line", "Error building line for way " + osmId + ": " + e);
+        throw new GeometryException("osm_invalid_line", "Error building line for way " + id() + ": " + e);
       }
     }
 
@@ -385,7 +383,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
         CoordinateSequence coords = nodeCache.getWayGeometry(nodeIds);
         return GeoUtils.JTS_FACTORY.createPolygon(coords);
       } catch (IllegalArgumentException e) {
-        throw new GeometryException("osm_invalid_polygon", "Error building polygon for way " + osmId + ": " + e);
+        throw new GeometryException("osm_invalid_polygon", "Error building polygon for way " + id() + ": " + e);
       }
     }
 
@@ -396,7 +394,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
 
     @Override
     public String toString() {
-      return "OsmWay[" + osmId + ']';
+      return "OsmWay[" + id() + ']';
     }
   }
 
@@ -425,12 +423,12 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
           }
         }
       }
-      return OsmMultipolygon.build(rings, nodeCache, osmId);
+      return OsmMultipolygon.build(rings, nodeCache, id());
     }
 
     @Override
     public String toString() {
-      return "OsmRelation[" + osmId + ']';
+      return "OsmRelation[" + id() + ']';
     }
   }
 

core/src/main/java/com/onthegomap/flatmap/read/ReaderFeature.java

@@ -14,18 +14,19 @@ public class ReaderFeature extends SourceFeature {
   private final Geometry latLonGeometry;
   private final Map<String, Object> properties;
 
-  public ReaderFeature(Geometry latLonGeometry, Map<String, Object> properties) {
+  public ReaderFeature(Geometry latLonGeometry, Map<String, Object> properties, long id) {
-    this(latLonGeometry, properties, null, null);
+    this(latLonGeometry, properties, null, null, id);
   }
 
-  public ReaderFeature(Geometry latLonGeometry, Map<String, Object> properties, String source, String sourceLayer) {
+  public ReaderFeature(Geometry latLonGeometry, Map<String, Object> properties, String source, String sourceLayer,
-    super(properties, source, sourceLayer, null);
+    long id) {
+    super(properties, source, sourceLayer, null, id);
     this.latLonGeometry = latLonGeometry;
     this.properties = properties;
   }
 
-  public ReaderFeature(Geometry latLonGeometry, int numProperties, String source, String sourceLayer) {
+  public ReaderFeature(Geometry latLonGeometry, int numProperties, String source, String sourceLayer, long id) {
-    this(latLonGeometry, new HashMap<>(numProperties), source, sourceLayer);
+    this(latLonGeometry, new HashMap<>(numProperties), source, sourceLayer, id);
   }
 
   @Override

core/src/main/java/com/onthegomap/flatmap/read/ShapefileReader.java

@@ -106,7 +106,9 @@ public class ShapefileReader extends Reader implements Closeable {
   public Topology.SourceStep<ReaderFeature> read() {
     return next -> {
       try (var iter = inputSource.features()) {
+        long id = 0;
         while (iter.hasNext()) {
+          id++;
           SimpleFeature feature = iter.next();
           Geometry source = (Geometry) feature.getDefaultGeometry();
           Geometry latLonGeometry = source;
@@ -114,7 +116,7 @@ public class ShapefileReader extends Reader implements Closeable {
             latLonGeometry = JTS.transform(source, transformToLatLon);
           }
           if (latLonGeometry != null) {
-            ReaderFeature geom = new ReaderFeature(latLonGeometry, attributeNames.length, sourceName, null);
+            ReaderFeature geom = new ReaderFeature(latLonGeometry, attributeNames.length, sourceName, null, id);
             for (int i = 1; i < attributeNames.length; i++) {
               geom.setTag(attributeNames[i], feature.getAttribute(i));
             }

core/src/main/java/com/onthegomap/flatmap/write/MbtilesWriter.java

@@ -138,7 +138,7 @@ public class MbtilesWriter {
       .setDescription(profile.description())
       .setAttribution(profile.attribution())
       .setVersion(profile.version())
-      .setTypeIsBaselayer()
+      .setType(profile.isOverlay() ? "overlay" : "baselayer")
       .setBoundsAndCenter(config.latLonBounds())
       .setMinzoom(config.minzoom())
       .setMaxzoom(config.maxzoom())

core/src/test/java/com/onthegomap/flatmap/FeatureCollectorTest.java

@@ -16,6 +16,7 @@ import java.util.stream.StreamSupport;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.locationtech.jts.geom.Geometry;
 
 public class FeatureCollectorTest {
 
@@ -30,6 +31,12 @@ public class FeatureCollectorTest {
     }
   }
 
+  private long id = 0;
+
+  private ReaderFeature newReaderFeature(Geometry latLonGeometry, Map<String, Object> properties) {
+    return new ReaderFeature(latLonGeometry, properties, id++);
+  }
+
   @Test
   public void testEmpty() {
     var collector = factory.get(newReaderFeature(newPoint(0, 0), Map.of(

core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java

@@ -176,6 +176,12 @@ public class FlatMapTest {
     );
   }
 
+  private long id = 0;
+
+  private ReaderFeature newReaderFeature(Geometry geometry, Map<String, Object> attrs) {
+    return new ReaderFeature(geometry, attrs, id++);
+  }
+
   @Test
   public void testMetadataButNoPoints() throws Exception {
     var results = runWithReaderFeatures(

core/src/test/java/com/onthegomap/flatmap/read/ReaderFeatureTest.java

@@ -21,7 +21,7 @@ public class ReaderFeatureTest {
     MultiPolygon multiPolygon = (MultiPolygon) new ReaderFeature(newMultiPolygon(
       newPolygon(outerPoints2, List.of(innerPoints2)),
       newPolygon(outerPoints1, List.of(innerPoints1))
-    ), Map.of()).worldGeometry();
+    ), Map.of(), 1).worldGeometry();
 
     assertEquals(2, multiPolygon.getNumGeometries());
     assertSameNormalizedFeature(round(newPolygon(

core/src/test/java/com/onthegomap/flatmap/render/FeatureRendererTest.java

@@ -44,7 +44,7 @@ public class FeatureRendererTest {
 
   private FeatureCollector collector(Geometry worldGeom) {
     var latLonGeom = GeoUtils.worldToLatLonCoords(worldGeom);
-    return new FeatureCollector.Factory(config, stats).get(new ReaderFeature(latLonGeom, 0, null, null));
+    return new FeatureCollector.Factory(config, stats).get(new ReaderFeature(latLonGeom, 0, null, null, 1));
   }
 
   private Map<TileCoord, Collection<Geometry>> renderGeometry(FeatureCollector.Feature feature) {