diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java index f77560bf..2f322def 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Bounds.java @@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; public class Bounds { private static final Logger LOGGER = LoggerFactory.getLogger(Bounds.class); + public static final Bounds WORLD = new Bounds(null); private Envelope latLon; private Envelope world; @@ -24,7 +25,7 @@ public class Bounds { private Geometry shape; - Bounds(Envelope latLon) { + public Bounds(Envelope latLon) { set(latLon); } @@ -36,6 +37,10 @@ public class Bounds { return world == null ? GeoUtils.WORLD_BOUNDS : world; } + public boolean isWorld() { + return latLon == null || latLon.equals(GeoUtils.WORLD_LAT_LON_BOUNDS); + } + public TileExtents tileExtents() { if (tileExtents == null) { tileExtents = TileExtents.computeFromWorldBounds(PlanetilerConfig.MAX_MAXZOOM, world(), shape); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java index 45323561..299ce80d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/reader/ShapefileReader.java @@ -2,6 +2,7 @@ package com.onthegomap.planetiler.reader; import com.onthegomap.planetiler.Profile; import com.onthegomap.planetiler.collection.FeatureGroup; +import com.onthegomap.planetiler.config.Bounds; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.stats.Stats; import java.io.IOException; @@ -19,9 +20,13 @@ import org.geotools.api.referencing.operation.MathTransform; import org.geotools.api.referencing.operation.OperationNotFoundException; import org.geotools.api.referencing.operation.TransformException; import org.geotools.data.shapefile.ShapefileDataStore; +import org.geotools.factory.CommonFactoryFinder; import org.geotools.feature.FeatureCollection; import org.geotools.geometry.jts.JTS; +import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.referencing.CRS; +import org.geotools.util.factory.GeoTools; +import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +50,10 @@ public class ShapefileReader extends SimpleReader { private MathTransform transformToLatLon; public ShapefileReader(String sourceProjection, String sourceName, Path input) { + this(sourceProjection, sourceName, input, Bounds.WORLD); + } + + public ShapefileReader(String sourceProjection, String sourceName, Path input, Bounds bounds) { super(sourceName); this.layer = input.getFileName().toString().replaceAll("\\.shp$", ""); dataStore = open(input); @@ -52,8 +61,6 @@ public class ShapefileReader extends SimpleReader { String typeName = dataStore.getTypeNames()[0]; FeatureSource source = dataStore .getFeatureSource(typeName); - - inputSource = source.getFeatures(Filter.INCLUDE); CoordinateReferenceSystem src = sourceProjection == null ? source.getSchema().getCoordinateReferenceSystem() : CRS.decode(sourceProjection); CoordinateReferenceSystem dest = CRS.decode("EPSG:4326", true); @@ -61,6 +68,26 @@ public class ShapefileReader extends SimpleReader { if (transformToLatLon.isIdentity()) { transformToLatLon = null; } + + Filter filter = Filter.INCLUDE; + + Envelope env = bounds.latLon(); + if (!bounds.isWorld()) { + var ff = CommonFactoryFinder.getFilterFactory(GeoTools.getDefaultHints()); + var schema = source.getSchema(); + + String geometryPropertyName = schema.getGeometryDescriptor().getLocalName(); + + var bbox = new ReferencedEnvelope(env.getMinX(), env.getMaxX(), env.getMinY(), env.getMaxY(), dest); + try { + var bbox2 = bbox.transform(schema.getGeometryDescriptor().getCoordinateReferenceSystem(), true); + filter = ff.bbox(ff.property(geometryPropertyName), bbox2); + } catch (TransformException e) { + // just use include filter + } + } + + inputSource = source.getFeatures(filter); attributeNames = new String[inputSource.getSchema().getAttributeCount()]; for (int i = 0; i < attributeNames.length; i++) { attributeNames[i] = inputSource.getSchema().getDescriptor(i).getLocalName(); @@ -105,7 +132,7 @@ public class ShapefileReader extends SimpleReader { SourceFeatureProcessor.processFiles( sourceName, sourcePaths, - path -> new ShapefileReader(sourceProjection, sourceName, path), + path -> new ShapefileReader(sourceProjection, sourceName, path, config.bounds()), writer, config, profile, stats ); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java index b5340247..26b1c97e 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/reader/ShapefileReaderTest.java @@ -1,9 +1,11 @@ package com.onthegomap.planetiler.reader; +import static com.onthegomap.planetiler.TestUtils.newPoint; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.config.Bounds; import com.onthegomap.planetiler.geo.GeoUtils; import com.onthegomap.planetiler.stats.Stats; import com.onthegomap.planetiler.util.FileUtils; @@ -11,9 +13,9 @@ import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import org.geotools.api.data.SimpleFeatureStore; import org.geotools.api.referencing.FactoryException; import org.geotools.api.referencing.operation.TransformException; @@ -29,12 +31,18 @@ import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.io.TempDir; +import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; class ShapefileReaderTest { @TempDir private Path tempDir; + private static final Envelope env = newPoint(-77.12911152370515, 38.79930767201779).getEnvelopeInternal(); + private static final int numInEnv = 18; + static { + env.expandBy(0.1); + } @Test @Timeout(30) @@ -55,6 +63,35 @@ class ShapefileReaderTest { testReadShapefile(dest.resolve("shapefile").resolve("stations.shp")); } + @Test + @Timeout(30) + void testReadShapefileWithBoundingBox() { + var dest = tempDir.resolve("shapefile.zip"); + FileUtils.unzipResource("/shapefile.zip", dest); + try ( + var reader = new ShapefileReader(null, "test", dest.resolve("shapefile").resolve("stations.shp"), new Bounds(env)) + ) { + for (int i = 1; i <= 2; i++) { + assertEquals(numInEnv, reader.getFeatureCount()); + List points = new CopyOnWriteArrayList<>(); + WorkerPipeline.start("test", Stats.inMemory()) + .fromGenerator("source", reader::readFeatures) + .addBuffer("reader_queue", 100, 1) + .sinkToConsumer("counter", 1, elem -> { + assertTrue(elem.getTag("name") instanceof String); + assertEquals("test", elem.getSource()); + assertEquals("stations", elem.getSourceLayer()); + points.add(elem.latLonGeometry()); + }).await(); + assertEquals(numInEnv, points.size()); + var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0])); + var centroid = gc.getCentroid(); + assertEquals(-77.0934256, centroid.getX(), 1e-5, "iter " + i); + assertEquals(38.8509022, centroid.getY(), 1e-5, "iter " + i); + } + } + } + @Test void testReadShapefileLeniently(@TempDir Path dir) throws IOException, TransformException, FactoryException { var shpPath = dir.resolve("test.shp"); @@ -82,7 +119,7 @@ class ShapefileReaderTest { featureStore.setTransaction(transaction); var collection = new DefaultFeatureCollection(); var featureBuilder = new SimpleFeatureBuilder(type); - featureBuilder.add(TestUtils.newPoint(1, 2)); + featureBuilder.add(newPoint(1, 2)); featureBuilder.add(3); var feature = featureBuilder.buildFeature(null); collection.add(feature); @@ -92,7 +129,7 @@ class ShapefileReaderTest { try (var reader = new ShapefileReader(null, "test", shpPath)) { assertEquals(1, reader.getFeatureCount()); - List features = new ArrayList<>(); + List features = new CopyOnWriteArrayList<>(); reader.readFeatures(features::add); assertEquals(10.5113, features.getFirst().latLonGeometry().getCentroid().getX(), 1e-4); assertEquals(0, features.getFirst().latLonGeometry().getCentroid().getY(), 1e-4); @@ -105,8 +142,8 @@ class ShapefileReaderTest { for (int i = 1; i <= 2; i++) { assertEquals(86, reader.getFeatureCount()); - List points = new ArrayList<>(); - List names = new ArrayList<>(); + List points = new CopyOnWriteArrayList<>(); + List names = new CopyOnWriteArrayList<>(); WorkerPipeline.start("test", Stats.inMemory()) .fromGenerator("source", reader::readFeatures) .addBuffer("reader_queue", 100, 1) @@ -117,12 +154,13 @@ class ShapefileReaderTest { points.add(elem.latLonGeometry()); names.add(elem.getTag("name").toString()); }).await(); + assertEquals(numInEnv, points.stream().filter(point -> env.contains(point.getCoordinate())).count()); assertEquals(86, points.size()); assertTrue(names.contains("Van Dörn Street")); var gc = GeoUtils.JTS_FACTORY.createGeometryCollection(points.toArray(new Geometry[0])); var centroid = gc.getCentroid(); - assertEquals(-77.0297995, centroid.getX(), 5, "iter " + i); - assertEquals(38.9119684, centroid.getY(), 5, "iter " + i); + assertEquals(-77.0297995, centroid.getX(), 1e-5, "iter " + i); + assertEquals(38.9119684, centroid.getY(), 1e-5, "iter " + i); } } }