Use rtree spatial index from PlanetilerConfig bounds by default for Geopackages. [#478]

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -371,6 +371,7 @@ public class Planetiler {
    * @see Downloader
    */
   public Planetiler addGeoPackageSource(String projection, String name, Path defaultPath, String defaultUrl) {
+
     Path path = getPath(name, "geopackage", defaultPath, defaultUrl);
     boolean keepUnzipped = getKeepUnzipped(name);
     return addStage(name, "Process features in " + path,

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java

@@ -14,14 +14,20 @@ import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.function.Consumer;
+import mil.nga.geopackage.BoundingBox;
 import mil.nga.geopackage.GeoPackage;
 import mil.nga.geopackage.GeoPackageManager;
+import mil.nga.geopackage.features.index.FeatureIndexManager;
+import mil.nga.geopackage.features.index.FeatureIndexResults;
+import mil.nga.geopackage.features.index.FeatureIndexType;
 import mil.nga.geopackage.features.user.FeatureColumns;
 import mil.nga.geopackage.features.user.FeatureDao;
+import mil.nga.geopackage.features.user.FeatureRow;
 import mil.nga.geopackage.geom.GeoPackageGeometryData;
 import org.geotools.geometry.jts.JTS;
 import org.geotools.geometry.jts.WKBReader;
 import org.geotools.referencing.CRS;
+import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Geometry;
 import org.opengis.referencing.FactoryException;
 import org.opengis.referencing.operation.MathTransform;
@@ -39,9 +45,14 @@ public class GeoPackageReader extends SimpleReader<SimpleFeature> {
   private final GeoPackage geoPackage;
   private final MathTransform coordinateTransform;
 
-  GeoPackageReader(String sourceProjection, String sourceName, Path input, Path tmpDir, boolean keepUnzipped) {
+  private final Envelope bounds;
+
+  GeoPackageReader(String sourceProjection, String sourceName, Path input, Path tmpDir, boolean keepUnzipped,
+    Envelope bounds) {
+
     super(sourceName);
     this.keepUnzipped = keepUnzipped;
+    this.bounds = bounds;
 
     if (sourceProjection != null) {
       try {
@@ -105,7 +116,7 @@ public class GeoPackageReader extends SimpleReader<SimpleFeature> {
     SourceFeatureProcessor.processFiles(
       sourceName,
       sourcePaths,
-      path -> new GeoPackageReader(sourceProjection, sourceName, path, tmpDir, keepUnzipped),
+      path -> new GeoPackageReader(sourceProjection, sourceName, path, tmpDir, keepUnzipped, config.bounds().latLon()),
       writer, config, profile, stats
     );
   }
@@ -138,7 +149,20 @@ public class GeoPackageReader extends SimpleReader<SimpleFeature> {
       MathTransform transform = (coordinateTransform != null) ? coordinateTransform :
         CRS.findMathTransform(CRS.decode("EPSG:" + srsId), latLonCRS);
 
-      for (var feature : features.queryForAll()) {
+      FeatureIndexManager indexer = new FeatureIndexManager(geoPackage,
+        features);
+      indexer.setIndexLocation(FeatureIndexType.RTREE);
+
+      BoundingBox boundingBox = BoundingBox.worldWGS84();
+
+      if (this.bounds != null) {
+        var l = this.bounds;
+
+        boundingBox = new BoundingBox(l.getMinX(), l.getMinY(), l.getMaxX(), l.getMaxY());
+      }
+      FeatureIndexResults indexResults = indexer.query(boundingBox);
+
+      for (FeatureRow feature : indexResults) {
         GeoPackageGeometryData geometryData = feature.getGeometry();
         byte[] wkb;
         if (geometryData == null || (wkb = geometryData.getWkb()).length == 0) {

planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java

@@ -18,6 +18,7 @@ import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Geometry;
 
 class GeoPackageReaderTest {
@@ -37,7 +38,7 @@ class GeoPackageReaderTest {
     for (var path : List.of(pathOutsideZip, pathInZip)) {
       for (var proj : projections) {
         try (
-          var reader = new GeoPackageReader(proj, "test", path, tmpDir, keepUnzipped)
+          var reader = new GeoPackageReader(proj, "test", path, tmpDir, keepUnzipped, null)
         ) {
           for (int iter = 0; iter < 2; iter++) {
             String id = "path=" + path + " proj=" + proj + " iter=" + iter;
@@ -66,13 +67,47 @@ class GeoPackageReaderTest {
     }
   }
 
+
+  @Test
+  @Timeout(30)
+  void testReadGeoPackageSpatialIndex() throws IOException {
+    Path pathOutsideZip = TestUtils.pathToResource("geopackage.gpkg");
+    Path zipPath = TestUtils.pathToResource("geopackage.gpkg.zip");
+    Path pathInZip = FileUtils.walkPathWithPattern(zipPath, "*.gpkg").get(0);
+
+    var projections = new String[]{null, "EPSG:4326"};
+
+    for (var path : List.of(pathOutsideZip, pathInZip)) {
+      for (var proj : projections) {
+        try (
+          var reader =
+            new GeoPackageReader(proj, "test", path, tmpDir, false, new Envelope(-77.0306, -77.0192, 38.8894, 38.9014))
+        ) {
+          for (int iter = 0; iter < 2; iter++) {
+            String id = "path=" + path + " proj=" + proj + " iter=" + iter;
+            assertEquals(86, reader.getFeatureCount(), id);
+            List<Geometry> points = new ArrayList<>();
+            List<String> names = new ArrayList<>();
+            WorkerPipeline.start("test", Stats.inMemory())
+              .fromGenerator("geopackage", reader::readFeatures, 1)
+              .addBuffer("reader_queue", 100, 1)
+              .sinkToConsumer("counter", 1, elem -> {
+                points.add(elem.latLonGeometry());
+              }).await();
+            assertEquals(4, points.size(), id);
+          }
+        }
+      }
+    }
+  }
+
   @Test
   @Timeout(30)
   void testReadEmptyGeoPackage() throws IOException {
     Path path = TestUtils.pathToResource("empty-geom.gpkg");
 
     try (
-      var reader = new GeoPackageReader(null, "test", path, tmpDir, false)
+      var reader = new GeoPackageReader(null, "test", path, tmpDir, false, null)
     ) {
       for (int iter = 0; iter < 2; iter++) {
         String id = "iter=" + iter;