diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java index 3a2a517b..4d8ae8d4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java @@ -1,6 +1,7 @@ package com.onthegomap.planetiler; import com.onthegomap.planetiler.geo.GeometryException; +import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.reader.SourceFeature; import com.onthegomap.planetiler.reader.osm.OsmElement; @@ -103,13 +104,40 @@ public interface Profile { * @return the new list of output features or {@code null} to not change anything. Set any elements of the list to * {@code null} if they should be ignored. * @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit - * the original input features, and continue processing other tiles + * the original input features, and continue processing other layers */ default List postProcessLayerFeatures(String layer, int zoom, List items) throws GeometryException { return items; } + /** + * Apply any post-processing to layers in an output tile before writing it to the output. + *

+ * This is called before {@link #postProcessLayerFeatures(String, int, List)} gets called for each layer. Use this + * method if features in one layer should influence features in another layer, to create new layers from existing + * ones, or if you need to remove a layer entirely from the output. + *

+ * These transformations may add, remove, or change the tags, geometry, or ordering of output features based on other + * features present in this tile. See {@link FeatureMerge} class for a set of common transformations that merge + * linestrings/polygons. + *

+ * Many threads invoke this method concurrently so ensure thread-safe access to any shared data structures. + *

+ * The default implementation passes through input features unaltered + * + * @param tileCoord the tile being post-processed + * @param layers all the output features in each layer on this tile + * @return the new map from layer to features or {@code null} to not change anything. Set any elements of the lists to + * {@code null} if they should be ignored. + * @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit + * the original input features, and continue processing other tiles + */ + default Map> postProcessTileFeatures(TileCoord tileCoord, + Map> layers) throws GeometryException { + return layers; + } + /** * Returns the name of the generated tileset to put into {@link Mbtiles} metadata * @@ -158,7 +186,9 @@ public interface Profile { return false; } - default Map extraArchiveMetadata() { return Map.of(); } + default Map extraArchiveMetadata() { + return Map.of(); + } /** * Defines whether {@link Wikidata} should fetch wikidata translations for the input element. diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index 63dfa6fb..619c26f6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; @@ -449,28 +450,47 @@ public final class FeatureGroup implements Iterable, if (layerStats != null) { tile.trackLayerStats(layerStats.forZoom(tileCoord.z())); } - List items = new ArrayList<>(entries.size()); + List items = new ArrayList<>(); String currentLayer = null; + Map> layerFeatures = new TreeMap<>(); for (SortableFeature entry : entries) { var feature = decodeVectorTileFeature(entry); String layer = feature.layer(); if (currentLayer == null) { currentLayer = layer; + layerFeatures.put(currentLayer, items); } else if (!currentLayer.equals(layer)) { - postProcessAndAddLayerFeatures(tile, currentLayer, items); currentLayer = layer; - items.clear(); + items = new ArrayList<>(); + layerFeatures.put(layer, items); } items.add(feature); } - postProcessAndAddLayerFeatures(tile, currentLayer, items); + // first post-process entire tile by invoking postProcessTileFeatures to allow for post-processing that combines + // features across different layers, infers new layers, or removes layers + try { + var initialFeatures = layerFeatures; + layerFeatures = profile.postProcessTileFeatures(tileCoord, layerFeatures); + if (layerFeatures == null) { + layerFeatures = initialFeatures; + } + } catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors + handlePostProcessFailure(e, "entire tile"); + } + // then let profiles post-process each layer in isolation with postProcessLayerFeatures + for (var entry : layerFeatures.entrySet()) { + postProcessAndAddLayerFeatures(tile, entry.getKey(), entry.getValue()); + } return tile; } private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer, List features) { + if (features == null || features.isEmpty()) { + return; + } try { List postProcessed = profile .postProcessLayerFeatures(layer, tileCoord.z(), features); @@ -482,21 +502,25 @@ public final class FeatureGroup implements Iterable, // also remove points more than --max-point-buffer pixels outside the tile if the // user has requested a narrower buffer than the profile provides by default } catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors - // failures in tile post-processing happen very late so err on the side of caution and - // log failures, only throwing when it's a fatal error - if (e instanceof GeometryException geoe) { - geoe.log(stats, "postprocess_layer", - "Caught error postprocessing features for " + layer + " layer on " + tileCoord, config.logJtsExceptions()); - } else if (e instanceof Error err) { - LOGGER.error("Caught fatal error postprocessing features {} {}", layer, tileCoord, e); - throw err; - } else { - LOGGER.error("Caught error postprocessing features {} {}", layer, tileCoord, e); - } + handlePostProcessFailure(e, layer); } encoder.addLayerFeatures(layer, features); } + private void handlePostProcessFailure(Throwable e, String entity) { + // failures in tile post-processing happen very late so err on the side of caution and + // log failures, only throwing when it's a fatal error + if (e instanceof GeometryException geoe) { + geoe.log(stats, "postprocess_layer", + "Caught error postprocessing features for " + entity + " on " + tileCoord, config.logJtsExceptions()); + } else if (e instanceof Error err) { + LOGGER.error("Caught fatal error postprocessing features {} {}", entity, tileCoord, e); + throw err; + } else { + LOGGER.error("Caught error postprocessing features {} {}", entity, tileCoord, e); + } + } + void add(SortableFeature entry) { numFeaturesProcessed.incrementAndGet(); long key = entry.key(); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 3e4239a2..610af5d9 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -1316,6 +1316,55 @@ class PlanetilerTests { )), sortListValues(results.tiles)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void postProcessTileFeatures(boolean postProcessLayersToo) throws Exception { + double y = 0.5 + Z15_WIDTH / 2; + double lat = GeoUtils.getWorldLat(y); + + double x1 = 0.5 + Z15_WIDTH / 4; + double lng1 = GeoUtils.getWorldLon(x1); + double lng2 = GeoUtils.getWorldLon(x1 + Z15_WIDTH * 10d / 256); + + List features1 = List.of( + newReaderFeature(newPoint(lng1, lat), Map.of("from", "a")), + newReaderFeature(newPoint(lng2, lat), Map.of("from", "b")) + ); + var testProfile = new Profile.NullProfile() { + @Override + public void processFeature(SourceFeature sourceFeature, FeatureCollector features) { + features.point(sourceFeature.getString("from")) + .inheritAttrFromSource("from") + .setMinZoom(15); + } + + @Override + public Map> postProcessTileFeatures(TileCoord tileCoord, + Map> layers) { + List features = new ArrayList<>(); + features.addAll(layers.get("a")); + features.addAll(layers.get("b")); + return Map.of("c", features); + } + + @Override + public List postProcessLayerFeatures(String layer, int zoom, List items) { + return postProcessLayersToo ? items.reversed() : items; + } + }; + var results = run( + Map.of("threads", "1", "maxzoom", "15"), + (featureGroup, profile, config) -> processReaderFeatures(featureGroup, profile, config, features1), + testProfile); + + assertSubmap(sortListValues(Map.of( + TileCoord.ofXYZ(Z15_TILES / 2, Z15_TILES / 2, 15), List.of( + feature(newPoint(64, 128), "c", Map.of("from", "a")), + feature(newPoint(74, 128), "c", Map.of("from", "b")) + ) + )), sortListValues(results.tiles)); + } + @Test void testMergeLineStringsIgnoresRoundingIntersections() throws Exception { double y = 0.5 + Z14_WIDTH / 2; @@ -1681,6 +1730,12 @@ class PlanetilerTests { throws GeometryException; } + private interface TilePostprocessFunction { + + Map> process(TileCoord tileCoord, Map> layers) + throws GeometryException; + } + private record PlanetilerResults( Map> tiles, Map metadata, int tileDataCount ) {} @@ -1692,7 +1747,8 @@ class PlanetilerTests { @Override String version, BiConsumer processFeature, Function> preprocessOsmRelation, - LayerPostprocessFunction postprocessLayerFeatures + LayerPostprocessFunction postprocessLayerFeatures, + TilePostprocessFunction postprocessTileFeatures ) implements Profile { TestProfile( @@ -1701,8 +1757,16 @@ class PlanetilerTests { LayerPostprocessFunction postprocessLayerFeatures ) { this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature, - preprocessOsmRelation, - postprocessLayerFeatures); + preprocessOsmRelation, postprocessLayerFeatures, null); + } + + TestProfile( + BiConsumer processFeature, + Function> preprocessOsmRelation, + TilePostprocessFunction postprocessTileFeatures + ) { + this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature, + preprocessOsmRelation, null, postprocessTileFeatures); } static TestProfile processSourceFeatures(BiConsumer processFeature) { @@ -1712,7 +1776,7 @@ class PlanetilerTests { @Override public List preprocessOsmRelation( OsmElement.Relation relation) { - return preprocessOsmRelation.apply(relation); + return preprocessOsmRelation == null ? null : preprocessOsmRelation.apply(relation); } @Override @@ -1726,7 +1790,13 @@ class PlanetilerTests { @Override public List postProcessLayerFeatures(String layer, int zoom, List items) throws GeometryException { - return postprocessLayerFeatures.process(layer, zoom, items); + return postprocessLayerFeatures == null ? null : postprocessLayerFeatures.process(layer, zoom, items); + } + + @Override + public Map> postProcessTileFeatures(TileCoord tileCoord, + Map> layers) throws GeometryException { + return postprocessTileFeatures == null ? null : postprocessTileFeatures.process(tileCoord, layers); } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index a0b69852..603d28e5 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -279,7 +279,7 @@ public class TestUtils { case UNKNOWN -> throw new IllegalArgumentException("cannot decompress \"UNKNOWN\""); }; var decoded = VectorTile.decode(bytes).stream() - .map(feature -> feature(decodeSilently(feature.geometry()), feature.attrs())).toList(); + .map(feature -> feature(decodeSilently(feature.geometry()), feature.layer(), feature.attrs())).toList(); tiles.put(tile.coord(), decoded); } return tiles; @@ -466,11 +466,32 @@ public class TestUtils { public record ComparableFeature( GeometryComparision geometry, + String layer, Map attrs - ) {} + ) { + + @Override + public boolean equals(Object o) { + return o == this || (o instanceof ComparableFeature other && + geometry.equals(other.geometry) && + attrs.equals(other.attrs) && + (layer == null || other.layer == null || Objects.equals(layer, other.layer))); + } + + @Override + public int hashCode() { + int result = geometry.hashCode(); + result = 31 * result + attrs.hashCode(); + return result; + } + } + + public static ComparableFeature feature(Geometry geom, String layer, Map attrs) { + return new ComparableFeature(new NormGeometry(geom), layer, attrs); + } public static ComparableFeature feature(Geometry geom, Map attrs) { - return new ComparableFeature(new NormGeometry(geom), attrs); + return new ComparableFeature(new NormGeometry(geom), null, attrs); } public static Map toMap(FeatureCollector.Feature feature, int zoom) {