natural earth reader

pom.xml

@@ -60,6 +60,11 @@
       <artifactId>gt-epsg-hsql</artifactId>
       <version>${geotools.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.xerial</groupId>
+      <artifactId>sqlite-jdbc</artifactId>
+      <version>3.34.0</version>
+    </dependency>
     <dependency>
       <groupId>org.mapdb</groupId>
       <artifactId>mapdb</artifactId>

src/main/java/com/graphhopper/reader/ReaderElementUtils.java

@@ -0,0 +1,13 @@
+package com.graphhopper.reader;
+
+import java.util.Map;
+
+/**
+ * Allows access to protected method ReaderElement.getTags
+ */
+public class ReaderElementUtils {
+
+  public static Map<String, Object> getProperties(ReaderElement elem) {
+    return elem.getTags();
+  }
+}

src/main/java/com/onthegomap/flatmap/GeoUtils.java

@@ -5,10 +5,12 @@ import org.locationtech.jts.geom.Geometry;
 import org.locationtech.jts.geom.GeometryFactory;
 import org.locationtech.jts.geom.impl.PackedCoordinateSequence;
 import org.locationtech.jts.geom.util.GeometryTransformer;
+import org.locationtech.jts.io.WKBReader;
 
 public class GeoUtils {
 
   public static final GeometryFactory gf = new GeometryFactory();
+  public static final WKBReader wkbReader = new WKBReader(gf);
 
   private static final double DEGREES_TO_RADIANS = Math.PI / 180;
   private static final double RADIANS_TO_DEGREES = 180 / Math.PI;

src/main/java/com/onthegomap/flatmap/SourceFeature.java

@@ -1,11 +1,22 @@
 package com.onthegomap.flatmap;
 
+import java.util.Map;
 import org.locationtech.jts.geom.Geometry;
 
 public interface SourceFeature {
 
-  Geometry getGeometry();
-  // props
+  Geometry geometry();
+
+  default void setTag(String key, Object value) {
+    properties().put(key, value);
+  }
+
+  Map<String, Object> properties();
+
+  default Object getTag(String name) {
+    return properties().get(name);
+  }
+
   // lazy geometry
   // lazy centroid
   // lazy area

src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java

@@ -1,28 +1,164 @@
 package com.onthegomap.flatmap.reader;
 
+import com.onthegomap.flatmap.GeoUtils;
 import com.onthegomap.flatmap.SourceFeature;
 import com.onthegomap.flatmap.monitoring.Stats;
 import com.onthegomap.flatmap.worker.Topology.SourceStep;
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import org.locationtech.jts.geom.Geometry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class NaturalEarthReader extends Reader {
 
-  public NaturalEarthReader(File input, File tmpFile, Stats stats) {
+  private static final Logger LOGGER = LoggerFactory.getLogger(NaturalEarthReader.class);
+
+  private final Connection conn;
+  private Path extracted;
+
+  public NaturalEarthReader(File input, Stats stats) {
+    this(input, null, stats);
+  }
+
+  public NaturalEarthReader(File input, File tmpDir, Stats stats) {
     super(stats);
+    try {
+      conn = open(input, tmpDir);
+    } catch (IOException | SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private Connection open(File file, File tmpLocation) throws IOException, SQLException {
+    String path = "jdbc:sqlite:" + file.getAbsolutePath();
+    File toOpen = file;
+    if (file.getName().endsWith(".zip")) {
+      toOpen = tmpLocation == null ? File.createTempFile("sqlite", "natearth") : tmpLocation;
+      extracted = toOpen.toPath();
+      toOpen.delete();
+      toOpen.deleteOnExit();
+      String sqliteFileInZip;
+      try (ZipFile zip = new ZipFile(file)) {
+        sqliteFileInZip = zip.stream()
+          .map(ZipEntry::getName)
+          .filter(z -> z.endsWith(".sqlite"))
+          .findFirst().orElse(null);
+      }
+      if (sqliteFileInZip == null) {
+        throw new IllegalArgumentException("No .sqlite file found inside " + file.getName());
+      } else {
+        LOGGER.info("unzipping " + file.getAbsolutePath() + " to " + extracted);
+        try (FileSystem fileSystem = FileSystems.newFileSystem(file.toPath())) {
+          Path fileToExtract = fileSystem.getPath(sqliteFileInZip);
+          Files.copy(
+            fileToExtract,
+            extracted
+          );
+        }
+      }
+      path = "jdbc:sqlite:" + toOpen.getAbsolutePath();
+    }
+    return DriverManager.getConnection(path);
+  }
+
+  private List<String> tableNames() {
+    List<String> result = new ArrayList<>();
+    try (ResultSet rs = conn.getMetaData().getTables(null, null, null, null)) {
+      while (rs.next()) {
+        String table = rs.getString("TABLE_NAME");
+        result.add(table);
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+    return result;
   }
 
   @Override
   public long getCount() {
-    return 0;
+    long count = 0;
+    for (String table : tableNames()) {
+      try (
+        var stmt = conn.createStatement();
+        var result = stmt.executeQuery("select count(*) from " + table + " where GEOMETRY is not null;")
+      ) {
+        count += result.getLong(1);
+      } catch (SQLException e) {
+        // maybe no GEOMETRY column?
+      }
+    }
+    return count;
   }
 
   @Override
   public SourceStep<SourceFeature> read() {
-    return null;
+    return next -> {
+      var tables = tableNames();
+      for (int i = 0; i < tables.size(); i++) {
+        String table = tables.get(i);
+        LOGGER.trace("Naturalearth loading " + i + "/" + tables.size() + ": " + table);
+
+        try (Statement statement = conn.createStatement()) {
+          ResultSet rs = statement.executeQuery("select * from " + table + ";");
+          String[] column = new String[rs.getMetaData().getColumnCount()];
+          int geometryColumn = -1;
+          for (int c = 0; c < column.length; c++) {
+            String name = rs.getMetaData().getColumnName(c + 1);
+            column[c] = name;
+            if ("GEOMETRY".equals(name)) {
+              geometryColumn = c;
+            }
+          }
+          if (geometryColumn >= 0) {
+            while (rs.next()) {
+              byte[] geometry = rs.getBytes(geometryColumn + 1);
+              if (geometry == null) {
+                continue;
+              }
+              Geometry geom = GeoUtils.wkbReader.read(geometry);
+              SourceFeature readerGeometry = new ReaderFeature(geom, column.length - 1);
+              for (int c = 0; c < column.length; c++) {
+                if (c != geometryColumn) {
+                  Object value = rs.getObject(c + 1);
+                  String key = column[c];
+                  readerGeometry.setTag(key, value);
+                }
+              }
+              next.accept(readerGeometry);
+            }
+          }
+        }
+      }
+    };
   }
 
   @Override
   public void close() {
-
+    try {
+      conn.close();
+    } catch (SQLException e) {
+      LOGGER.error("Error closing sqlite file", e);
+    }
+    if (extracted != null) {
+      try {
+        Files.deleteIfExists(extracted);
+      } catch (IOException e) {
+        LOGGER.error("Error deleting temp file", e);
+      }
+    }
   }
 }

src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java

@@ -4,6 +4,7 @@ import com.carrotsearch.hppc.LongHashSet;
 import com.graphhopper.coll.GHLongHashSet;
 import com.graphhopper.coll.GHLongObjectHashMap;
 import com.graphhopper.reader.ReaderElement;
+import com.graphhopper.reader.ReaderElementUtils;
 import com.graphhopper.reader.ReaderNode;
 import com.graphhopper.reader.ReaderRelation;
 import com.graphhopper.reader.ReaderWay;
@@ -25,6 +26,7 @@ import com.onthegomap.flatmap.worker.Topology;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicLong;
 import org.locationtech.jts.geom.Geometry;
@@ -194,38 +196,52 @@ public class OpenStreetMapReader implements Closeable {
     }
   }
 
-  private static class NodeSourceFeature implements SourceFeature {
+  private static abstract class ProxyFeature implements SourceFeature {
+
+    protected final Map<String, Object> tags;
+
+    public ProxyFeature(ReaderElement elem) {
+      tags = ReaderElementUtils.getProperties(elem);
+    }
+
+    @Override
+    public Map<String, Object> properties() {
+      return null;
+    }
+  }
+
+  private static class NodeSourceFeature extends ProxyFeature {
 
     public NodeSourceFeature(ReaderNode node) {
-      super();
+      super(node);
     }
 
     @Override
-    public Geometry getGeometry() {
+    public Geometry geometry() {
       return null;
     }
   }
 
-  private static class WaySourceFeature implements SourceFeature {
+  private static class WaySourceFeature extends ProxyFeature {
 
     public WaySourceFeature(ReaderWay way) {
-      super();
+      super(way);
     }
 
     @Override
-    public Geometry getGeometry() {
+    public Geometry geometry() {
       return null;
     }
   }
 
-  private static class MultipolygonSourceFeature implements SourceFeature {
+  private static class MultipolygonSourceFeature extends ProxyFeature {
 
     public MultipolygonSourceFeature(ReaderRelation relation) {
-      super();
+      super(relation);
     }
 
     @Override
-    public Geometry getGeometry() {
+    public Geometry geometry() {
       return null;
     }
   }

src/main/java/com/onthegomap/flatmap/reader/Reader.java

@@ -44,7 +44,7 @@ public abstract class Reader implements Closeable {
         while ((sourceFeature = prev.get()) != null) {
           featuresRead.incrementAndGet();
           features.reset(sourceFeature);
-          if (sourceFeature.getGeometry().getEnvelopeInternal().intersects(env)) {
+          if (sourceFeature.geometry().getEnvelopeInternal().intersects(env)) {
             profile.processFeature(sourceFeature, features);
             for (RenderableFeature renderable : features.all()) {
               renderer.renderFeature(renderable, next);

src/main/java/com/onthegomap/flatmap/reader/ReaderFeature.java

@@ -1,18 +1,14 @@
 package com.onthegomap.flatmap.reader;
 
 import com.onthegomap.flatmap.SourceFeature;
+import java.util.HashMap;
+import java.util.Map;
 import org.locationtech.jts.geom.Geometry;
 
-public class ReaderFeature implements SourceFeature {
+public record ReaderFeature(Geometry geometry, Map<String, Object> properties) implements SourceFeature {
 
-  private final Geometry geometry;
-
-  public ReaderFeature(Geometry geometry) {
-    this.geometry = geometry;
+  public ReaderFeature(Geometry geometry, int numProperties) {
+    this(geometry, new HashMap<>(numProperties));
   }
 
-  @Override
-  public Geometry getGeometry() {
-    return geometry;
-  }
 }

src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java

@@ -119,11 +119,10 @@ public class ShapefileReader extends Reader implements Closeable {
             transformed = JTS.transform(source, transform);
           }
           if (transformed != null) {
-            SourceFeature geom = new ReaderFeature(transformed);
-            // TODO
-            //          for (int i = 1; i < attributeNames.length; i++) {
-            //            geom.setTag(attributeNames[i], feature.getAttribute(i));
-            //          }
+            SourceFeature geom = new ReaderFeature(transformed, attributeNames.length);
+            for (int i = 1; i < attributeNames.length; i++) {
+              geom.setTag(attributeNames[i], feature.getAttribute(i));
+            }
             next.accept(geom);
           }
         }

src/test/java/com/onthegomap/flatmap/reader/NaturalEarthReaderTest.java

@@ -0,0 +1,51 @@
+package com.onthegomap.flatmap.reader;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.onthegomap.flatmap.GeoUtils;
+import com.onthegomap.flatmap.monitoring.Stats.InMemory;
+import com.onthegomap.flatmap.worker.Topology;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.locationtech.jts.geom.Geometry;
+
+public class NaturalEarthReaderTest {
+
+  @ParameterizedTest
+  @ValueSource(strings = {"natural_earth_vector.sqlite", "natural_earth_vector.sqlite.zip"})
+  @Timeout(30)
+  public void testReadNaturalEarth(String filename, @TempDir File tempDir) {
+    var file = new File("src/test/resources/" + filename);
+    try (var reader = new NaturalEarthReader(file, tempDir, new InMemory())) {
+      for (int i = 1; i <= 2; i++) {
+        assertEquals(19, reader.getCount(), "iter " + i);
+
+        List<Geometry> points = new ArrayList<>();
+        Topology.start("test", new InMemory())
+          .fromGenerator("naturalearth", reader.read())
+          .addBuffer("reader_queue", 100, 1)
+          .sinkToConsumer("counter", 1, elem -> {
+            Object elevation = elem.getTag("elevation");
+            assertTrue(elevation instanceof Double, Objects.toString(elevation));
+            points.add(elem.geometry());
+          }).await();
+        assertEquals(19, points.size());
+        var gc = GeoUtils.gf.createGeometryCollection(points.toArray(new Geometry[0]));
+        var centroid = gc.getCentroid();
+        assertArrayEquals(
+          new double[]{14.22422, 12.994629},
+          new double[]{centroid.getX(), centroid.getY()}, 5,
+          "iter " + i
+        );
+      }
+    }
+  }
+}

src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java

@@ -1,6 +1,7 @@
 package com.onthegomap.flatmap.reader;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.onthegomap.flatmap.GeoUtils;
 import com.onthegomap.flatmap.monitoring.Stats.InMemory;
@@ -8,8 +9,6 @@ import com.onthegomap.flatmap.worker.Topology;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
@@ -27,21 +26,20 @@ public class ShapefileReaderTest {
   @Test
   public void testCount() {
     assertEquals(86, reader.getCount());
+    assertEquals(86, reader.getCount());
   }
 
   @Test
   @Timeout(30)
-  public void testReadShapefileTwice() {
+  public void testReadShapefile() {
     for (int i = 1; i <= 2; i++) {
-      Map<String, Integer> counts = new TreeMap<>();
       List<Geometry> points = new ArrayList<>();
       Topology.start("test", new InMemory())
         .fromGenerator("shapefile", reader.read())
         .addBuffer("reader_queue", 100, 1)
         .sinkToConsumer("counter", 1, elem -> {
-          String type = elem.getGeometry().getGeometryType();
-          counts.put(type, counts.getOrDefault(type, 0) + 1);
-          points.add(elem.getGeometry());
+          assertTrue(elem.getTag("name") instanceof String);
+          points.add(elem.geometry());
         }).await();
       assertEquals(86, points.size());
       var gc = GeoUtils.gf.createGeometryCollection(points.toArray(new Geometry[0]));