Add basic support for reading GeoPackage files. (#413)

NOTICE.md

@@ -25,6 +25,7 @@ The `planetiler-core` module includes the following software:
   - com.github.jnr:jnr-ffi (Apache license)
   - org.roaringbitmap:RoaringBitmap (Apache license)
   - org.projectnessie.cel:cel-tools (Apache license)
+  - mil.nga.geopackage:geopackage (MIT license)
 - Adapted code:
   - `DouglasPeuckerSimplifier` from [JTS](https://github.com/locationtech/jts) (EDL)
   - `OsmMultipolygon` from [imposm3](https://github.com/omniscale/imposm3) (Apache license)

planetiler-core/pom.xml

@@ -20,6 +20,7 @@
     <log4j.version>2.19.0</log4j.version>
     <prometheus.version>0.16.0</prometheus.version>
     <protobuf.version>3.21.12</protobuf.version>
+    <geopackage.version>6.6.0</geopackage.version>
   </properties>
 
   <dependencies>
@@ -138,6 +139,11 @@
       <artifactId>guava</artifactId>
       <version>31.1-jre</version>
     </dependency>
+    <dependency>
+      <groupId>mil.nga.geopackage</groupId>
+      <artifactId>geopackage</artifactId>
+      <version>${geopackage.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -7,6 +7,7 @@ import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.MbtilesMetadata;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.mbtiles.MbtilesWriter;
+import com.onthegomap.planetiler.reader.GeoPackageReader;
 import com.onthegomap.planetiler.reader.NaturalEarthReader;
 import com.onthegomap.planetiler.reader.ShapefileReader;
 import com.onthegomap.planetiler.reader.osm.OsmInputFile;
@@ -337,6 +338,31 @@ public class Planetiler {
       }));
   }
 
+  /**
+   * Adds a new OGC GeoPackage source that will be processed when {@link #run()} is called.
+   * <p>
+   * If the file does not exist and {@code download=true} argument is set, then the file will first be downloaded from
+   * {@code defaultUrl}.
+   * <p>
+   * To override the location of the {@code geopackage} file, set {@code name_path=newpath.gpkg} in the arguments and to
+   * override the download URL set {@code name_url=http://url/of/file.gpkg}.
+   *
+   * @param name        string to use in stats and logs to identify this stage
+   * @param defaultPath path to the input file to use if {@code name_path} key is not set through arguments
+   * @param defaultUrl  remote URL that the file to download if {@code download=true} argument is set and {@code
+   *                    name_url} argument is not set
+   * @return this runner instance for chaining
+   * @see GeoPackageReader
+   * @see Downloader
+   */
+  public Planetiler addGeoPackageSource(String name, Path defaultPath, String defaultUrl) {
+    Path path = getPath(name, "geopackage", defaultPath, defaultUrl);
+    return addStage(name, "Process features in " + path,
+      ifSourceUsed(name,
+        () -> GeoPackageReader.process(name, List.of(path), featureGroup, config, profile, stats)));
+  }
+
+
   /**
    * Adds a new Natural Earth sqlite file source that will be processed when {@link #run()} is called.
    * <p>

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/GeoPackageReader.java

@@ -0,0 +1,109 @@
+package com.onthegomap.planetiler.reader;
+
+import com.onthegomap.planetiler.Profile;
+import com.onthegomap.planetiler.collection.FeatureGroup;
+import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.stats.Stats;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.function.Consumer;
+import mil.nga.geopackage.GeoPackage;
+import mil.nga.geopackage.GeoPackageManager;
+import mil.nga.geopackage.features.user.FeatureColumns;
+import mil.nga.geopackage.features.user.FeatureDao;
+import mil.nga.geopackage.geom.GeoPackageGeometryData;
+import org.geotools.geometry.jts.JTS;
+import org.geotools.geometry.jts.WKBReader;
+import org.geotools.referencing.CRS;
+import org.locationtech.jts.geom.Geometry;
+import org.opengis.referencing.crs.CoordinateReferenceSystem;
+import org.opengis.referencing.operation.MathTransform;
+
+/**
+ * Utility that reads {@link SourceFeature SourceFeatures} from the vector geometries contained in a GeoPackage file.
+ */
+public class GeoPackageReader extends SimpleReader<SimpleFeature> {
+
+  private final GeoPackage geoPackage;
+
+  GeoPackageReader(String sourceName, Path input) {
+    super(sourceName);
+
+    geoPackage = GeoPackageManager.open(false, input.toFile());
+  }
+
+  /**
+   * Renders map features for all elements from an OGC GeoPackage based on the mapping logic defined in {@code
+   * profile}.
+   *
+   * @param sourceName  string ID for this reader to use in logs and stats
+   * @param sourcePaths paths to the {@code .gpkg} files on disk
+   * @param writer      consumer for rendered features
+   * @param config      user-defined parameters controlling number of threads and log interval
+   * @param profile     logic that defines what map features to emit for each source feature
+   * @param stats       to keep track of counters and timings
+   * @throws IllegalArgumentException if a problem occurs reading the input file
+   */
+  public static void process(String sourceName, List<Path> sourcePaths, FeatureGroup writer, PlanetilerConfig config,
+    Profile profile, Stats stats) {
+    SourceFeatureProcessor.processFiles(
+      sourceName,
+      sourcePaths,
+      path -> new GeoPackageReader(sourceName, path),
+      writer, config, profile, stats
+    );
+  }
+
+  @Override
+  public long getFeatureCount() {
+    long numFeatures = 0;
+
+    for (String name : geoPackage.getFeatureTables()) {
+      FeatureDao features = geoPackage.getFeatureDao(name);
+      numFeatures += features.count();
+    }
+    return numFeatures;
+  }
+
+  @Override
+  public void readFeatures(Consumer<SimpleFeature> next) throws Exception {
+    CoordinateReferenceSystem latLonCRS = CRS.decode("EPSG:4326");
+    long id = 0;
+
+    for (var featureName : geoPackage.getFeatureTables()) {
+      FeatureDao features = geoPackage.getFeatureDao(featureName);
+
+      MathTransform transform = CRS.findMathTransform(
+        CRS.decode("EPSG:" + features.getSrsId()),
+        latLonCRS);
+
+      for (var feature : features.queryForAll()) {
+        GeoPackageGeometryData geometryData = feature.getGeometry();
+        if (geometryData == null) {
+          continue;
+        }
+
+        Geometry featureGeom = (new WKBReader()).read(geometryData.getWkb());
+        Geometry latLonGeom = (transform.isIdentity()) ? featureGeom : JTS.transform(featureGeom, transform);
+
+        FeatureColumns columns = feature.getColumns();
+        SimpleFeature geom = SimpleFeature.create(latLonGeom, new HashMap<>(columns.columnCount()),
+          sourceName, featureName, ++id);
+
+        for (int i = 0; i < columns.columnCount(); ++i) {
+          if (i != columns.getGeometryIndex()) {
+            geom.setTag(columns.getColumnName(i), feature.getValue(i));
+          }
+        }
+
+        next.accept(geom);
+      }
+    }
+  }
+
+  @Override
+  public void close() {
+    geoPackage.close();
+  }
+}

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -1667,6 +1667,7 @@ class PlanetilerTests {
       .addOsmSource("osm", tempOsm)
       .addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
       .addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
+      .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
       .setOutput("mbtiles", mbtiles)
       .run();
 
@@ -1743,12 +1744,52 @@ class PlanetilerTests {
     }
   }
 
+  @ParameterizedTest
+  @ValueSource(strings = {
+    "",
+    "--write-threads=2 --process-threads=2 --feature-read-threads=2 --threads=4",
+  })
+  void testPlanetilerRunnerGeoPackage(String args) throws Exception {
+    Path mbtiles = tempDir.resolve("output.mbtiles");
+
+    Planetiler.create(Arguments.fromArgs((args + " --tmpdir=" + tempDir.resolve("data")).split("\\s+")))
+      .setProfile(new Profile.NullProfile() {
+        @Override
+        public void processFeature(SourceFeature source, FeatureCollector features) {
+          features.point("stations")
+            .setZoomRange(0, 14)
+            .setAttr("name", source.getString("name"));
+        }
+      })
+      .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
+      .setOutput("mbtiles", mbtiles)
+      .run();
+
+    try (Mbtiles db = Mbtiles.newReadOnlyDatabase(mbtiles)) {
+      Set<String> uniqueNames = new HashSet<>();
+      long featureCount = 0;
+      var tileMap = TestUtils.getTileMap(db);
+      for (var tile : tileMap.values()) {
+        for (var feature : tile) {
+          feature.geometry().validate();
+          featureCount++;
+          uniqueNames.add((String) feature.attrs().get("name"));
+        }
+      }
+
+      assertTrue(featureCount > 0);
+      assertEquals(86, uniqueNames.size());
+      assertTrue(uniqueNames.contains("Van Dörn Street"));
+    }
+  }
+
   private void runWithProfile(Path tempDir, Profile profile, boolean force) throws Exception {
     Planetiler.create(Arguments.of("tmpdir", tempDir, "force", Boolean.toString(force)))
       .setProfile(profile)
       .addOsmSource("osm", TestUtils.pathToResource("monaco-latest.osm.pbf"))
       .addNaturalEarthSource("ne", TestUtils.pathToResource("natural_earth_vector.sqlite"))
       .addShapefileSource("shapefile", TestUtils.pathToResource("shapefile.zip"))
+      .addGeoPackageSource("geopackage", TestUtils.pathToResource("geopackage.gpkg"), null)
       .setOutput("mbtiles", tempDir.resolve("output.mbtiles"))
       .run();
   }

planetiler-core/src/test/java/com/onthegomap/planetiler/reader/GeoPackageReaderTest.java

@@ -0,0 +1,53 @@
+package com.onthegomap.planetiler.reader;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.onthegomap.planetiler.TestUtils;
+import com.onthegomap.planetiler.collection.IterableOnce;
+import com.onthegomap.planetiler.geo.GeoUtils;
+import com.onthegomap.planetiler.stats.Stats;
+import com.onthegomap.planetiler.worker.WorkerPipeline;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.locationtech.jts.geom.Geometry;
+
+class GeoPackageReaderTest {
+
+  @Test
+  @Timeout(30)
+  void testReadGeoPackage() {
+    Path path = TestUtils.pathToResource("geopackage.gpkg");
+
+    try (
+      var reader = new GeoPackageReader("test", path)
+    ) {
+      for (int i = 1; i <= 2; i++) {
+        assertEquals(86, reader.getFeatureCount());
+        List<Geometry> points = new ArrayList<>();
+        List<String> names = new ArrayList<>();
+        WorkerPipeline.start("test", Stats.inMemory())
+          .readFromTiny("files", List.of(Path.of("dummy-path")))
+          .addWorker("geopackage", 1, (IterableOnce<Path> p, Consumer<SimpleFeature> next) -> reader.readFeatures(next))
+          .addBuffer("reader_queue", 100, 1)
+          .sinkToConsumer("counter", 1, elem -> {
+            assertTrue(elem.getTag("name") instanceof String);
+            assertEquals("test", elem.getSource());
+            assertEquals("stations", elem.getSourceLayer());
+            points.add(elem.latLonGeometry());
+            names.add(elem.getTag("name").toString());
+          }).await();
+        assertEquals(86, points.size());
+        assertTrue(names.contains("Van Dörn Street"));
+        var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0]));
+        var centroid = gc.getCentroid();
+        assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i);
+        assertEquals(38.9119684, centroid.getY(), 5, "iter " + i);
+      }
+    }
+  }
+}

planetiler-custommap/README.md

@@ -57,7 +57,8 @@ examples: [...]
 A description that tells planetiler how to read geospatial objects with tags from an input file.
 
 - `type` - Enum representing the file format of the data source, one
-  of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format) or [`shapefile`](https://en.wikipedia.org/wiki/Shapefile)
+  of [`osm`](https://wiki.openstreetmap.org/wiki/PBF_Format), [`shapefile`](https://en.wikipedia.org/wiki/Shapefile),
+  or [`geopackage`](https://www.geopackage.org/).
 - `local_path` - Local path to the file to use, inferred from `url` if missing. Can be a string
   or [expression](#expression) that can reference [argument values](#arguments).
 - `url` - Location to download the file from if not present at `local_path`.

planetiler-custommap/planetiler.schema.json

@@ -40,7 +40,8 @@
             "description": "File format of the data source",
             "enum": [
               "osm",
-              "shapefile"
+              "shapefile",
+              "geopackage"
             ]
           },
           "url": {

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/ConfiguredMapMain.java

@@ -71,6 +71,7 @@ public class ConfiguredMapMain {
     switch (sourceType) {
       case OSM -> planetiler.addOsmSource(source.id(), localPath, source.url());
       case SHAPEFILE -> planetiler.addShapefileSource(source.id(), localPath, source.url());
+      case GEOPACKAGE -> planetiler.addGeoPackageSource(source.id(), localPath, source.url());
       default -> throw new IllegalArgumentException("Unhandled source type for " + source.id() + ": " + sourceType);
     }
   }

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/DataSourceType.java

@@ -6,5 +6,7 @@ public enum DataSourceType {
   @JsonProperty("osm")
   OSM,
   @JsonProperty("shapefile")
-  SHAPEFILE
+  SHAPEFILE,
+  @JsonProperty("geopackage")
+  GEOPACKAGE
 }

planetiler-custommap/src/test/resources/validSchema/road_motorway.yml

@@ -9,6 +9,9 @@ sources:
   osm:
     type: osm
     url: geofabrik:rhode-island
+  gpkg:
+    type: geopackage
+    url: https://example.com/geopackage.gpkg
 tag_mappings:
   bridge: boolean # input=bridge, output=bridge, type=boolean
   layer: long