Add whole-tile postprocess hook (#802)

planetiler-core/src/main/java/com/onthegomap/planetiler/Profile.java

@@ -1,6 +1,7 @@
 package com.onthegomap.planetiler;
 
 import com.onthegomap.planetiler.geo.GeometryException;
+import com.onthegomap.planetiler.geo.TileCoord;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.reader.SourceFeature;
 import com.onthegomap.planetiler.reader.osm.OsmElement;
@@ -103,13 +104,40 @@ public interface Profile {
    * @return the new list of output features or {@code null} to not change anything. Set any elements of the list to
    *         {@code null} if they should be ignored.
    * @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
-   *                           the original input features, and continue processing other tiles
+   *                           the original input features, and continue processing other layers
    */
   default List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
     List<VectorTile.Feature> items) throws GeometryException {
     return items;
   }
 
+  /**
+   * Apply any post-processing to layers in an output tile before writing it to the output.
+   * <p>
+   * This is called before {@link #postProcessLayerFeatures(String, int, List)} gets called for each layer. Use this
+   * method if features in one layer should influence features in another layer, to create new layers from existing
+   * ones, or if you need to remove a layer entirely from the output.
+   * <p>
+   * These transformations may add, remove, or change the tags, geometry, or ordering of output features based on other
+   * features present in this tile. See {@link FeatureMerge} class for a set of common transformations that merge
+   * linestrings/polygons.
+   * <p>
+   * Many threads invoke this method concurrently so ensure thread-safe access to any shared data structures.
+   * <p>
+   * The default implementation passes through input features unaltered
+   *
+   * @param tileCoord the tile being post-processed
+   * @param layers    all the output features in each layer on this tile
+   * @return the new map from layer to features or {@code null} to not change anything. Set any elements of the lists to
+   *         {@code null} if they should be ignored.
+   * @throws GeometryException for any recoverable geometric operation failures - the framework will log the error, emit
+   *                           the original input features, and continue processing other tiles
+   */
+  default Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
+    Map<String, List<VectorTile.Feature>> layers) throws GeometryException {
+    return layers;
+  }
+
   /**
    * Returns the name of the generated tileset to put into {@link Mbtiles} metadata
    *
@@ -158,7 +186,9 @@ public interface Profile {
     return false;
   }
 
-  default Map<String,String> extraArchiveMetadata() { return Map.of(); }
+  default Map<String, String> extraArchiveMetadata() {
+    return Map.of();
+  }
 
   /**
    * Defines whether {@link Wikidata} should fetch wikidata translations for the input element.

planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java

@@ -26,6 +26,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import javax.annotation.concurrent.NotThreadSafe;
@@ -449,28 +450,47 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
       if (layerStats != null) {
         tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
       }
-      List<VectorTile.Feature> items = new ArrayList<>(entries.size());
+      List<VectorTile.Feature> items = new ArrayList<>();
       String currentLayer = null;
+      Map<String, List<VectorTile.Feature>> layerFeatures = new TreeMap<>();
       for (SortableFeature entry : entries) {
         var feature = decodeVectorTileFeature(entry);
         String layer = feature.layer();
 
         if (currentLayer == null) {
           currentLayer = layer;
+          layerFeatures.put(currentLayer, items);
         } else if (!currentLayer.equals(layer)) {
-          postProcessAndAddLayerFeatures(tile, currentLayer, items);
           currentLayer = layer;
-          items.clear();
+          items = new ArrayList<>();
+          layerFeatures.put(layer, items);
         }
 
         items.add(feature);
       }
-      postProcessAndAddLayerFeatures(tile, currentLayer, items);
+      // first post-process entire tile by invoking postProcessTileFeatures to allow for post-processing that combines
+      // features across different layers, infers new layers, or removes layers
+      try {
+        var initialFeatures = layerFeatures;
+        layerFeatures = profile.postProcessTileFeatures(tileCoord, layerFeatures);
+        if (layerFeatures == null) {
+          layerFeatures = initialFeatures;
+        }
+      } catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
+        handlePostProcessFailure(e, "entire tile");
+      }
+      // then let profiles post-process each layer in isolation with postProcessLayerFeatures
+      for (var entry : layerFeatures.entrySet()) {
+        postProcessAndAddLayerFeatures(tile, entry.getKey(), entry.getValue());
+      }
       return tile;
     }
 
     private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
       List<VectorTile.Feature> features) {
+      if (features == null || features.isEmpty()) {
+        return;
+      }
       try {
         List<VectorTile.Feature> postProcessed = profile
           .postProcessLayerFeatures(layer, tileCoord.z(), features);
@@ -482,21 +502,25 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
         // also remove points more than --max-point-buffer pixels outside the tile if the
         // user has requested a narrower buffer than the profile provides by default
       } catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
-        // failures in tile post-processing happen very late so err on the side of caution and
-        // log failures, only throwing when it's a fatal error
-        if (e instanceof GeometryException geoe) {
-          geoe.log(stats, "postprocess_layer",
-            "Caught error postprocessing features for " + layer + " layer on " + tileCoord, config.logJtsExceptions());
-        } else if (e instanceof Error err) {
-          LOGGER.error("Caught fatal error postprocessing features {} {}", layer, tileCoord, e);
-          throw err;
-        } else {
-          LOGGER.error("Caught error postprocessing features {} {}", layer, tileCoord, e);
-        }
+        handlePostProcessFailure(e, layer);
       }
       encoder.addLayerFeatures(layer, features);
     }
 
+    private void handlePostProcessFailure(Throwable e, String entity) {
+      // failures in tile post-processing happen very late so err on the side of caution and
+      // log failures, only throwing when it's a fatal error
+      if (e instanceof GeometryException geoe) {
+        geoe.log(stats, "postprocess_layer",
+          "Caught error postprocessing features for " + entity + " on " + tileCoord, config.logJtsExceptions());
+      } else if (e instanceof Error err) {
+        LOGGER.error("Caught fatal error postprocessing features {} {}", entity, tileCoord, e);
+        throw err;
+      } else {
+        LOGGER.error("Caught error postprocessing features {} {}", entity, tileCoord, e);
+      }
+    }
+
     void add(SortableFeature entry) {
       numFeaturesProcessed.incrementAndGet();
       long key = entry.key();

planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java

@@ -1316,6 +1316,55 @@ class PlanetilerTests {
     )), sortListValues(results.tiles));
   }
 
+  @ParameterizedTest
+  @ValueSource(booleans = {false, true})
+  void postProcessTileFeatures(boolean postProcessLayersToo) throws Exception {
+    double y = 0.5 + Z15_WIDTH / 2;
+    double lat = GeoUtils.getWorldLat(y);
+
+    double x1 = 0.5 + Z15_WIDTH / 4;
+    double lng1 = GeoUtils.getWorldLon(x1);
+    double lng2 = GeoUtils.getWorldLon(x1 + Z15_WIDTH * 10d / 256);
+
+    List<SimpleFeature> features1 = List.of(
+      newReaderFeature(newPoint(lng1, lat), Map.of("from", "a")),
+      newReaderFeature(newPoint(lng2, lat), Map.of("from", "b"))
+    );
+    var testProfile = new Profile.NullProfile() {
+      @Override
+      public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
+        features.point(sourceFeature.getString("from"))
+          .inheritAttrFromSource("from")
+          .setMinZoom(15);
+      }
+
+      @Override
+      public Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
+        Map<String, List<VectorTile.Feature>> layers) {
+        List<VectorTile.Feature> features = new ArrayList<>();
+        features.addAll(layers.get("a"));
+        features.addAll(layers.get("b"));
+        return Map.of("c", features);
+      }
+
+      @Override
+      public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom, List<VectorTile.Feature> items) {
+        return postProcessLayersToo ? items.reversed() : items;
+      }
+    };
+    var results = run(
+      Map.of("threads", "1", "maxzoom", "15"),
+      (featureGroup, profile, config) -> processReaderFeatures(featureGroup, profile, config, features1),
+      testProfile);
+
+    assertSubmap(sortListValues(Map.of(
+      TileCoord.ofXYZ(Z15_TILES / 2, Z15_TILES / 2, 15), List.of(
+        feature(newPoint(64, 128), "c", Map.of("from", "a")),
+        feature(newPoint(74, 128), "c", Map.of("from", "b"))
+      )
+    )), sortListValues(results.tiles));
+  }
+
   @Test
   void testMergeLineStringsIgnoresRoundingIntersections() throws Exception {
     double y = 0.5 + Z14_WIDTH / 2;
@@ -1681,6 +1730,12 @@ class PlanetilerTests {
       throws GeometryException;
   }
 
+  private interface TilePostprocessFunction {
+
+    Map<String, List<VectorTile.Feature>> process(TileCoord tileCoord, Map<String, List<VectorTile.Feature>> layers)
+      throws GeometryException;
+  }
+
   private record PlanetilerResults(
     Map<TileCoord, List<TestUtils.ComparableFeature>> tiles, Map<String, String> metadata, int tileDataCount
   ) {}
@@ -1692,7 +1747,8 @@ class PlanetilerTests {
     @Override String version,
     BiConsumer<SourceFeature, FeatureCollector> processFeature,
     Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
-    LayerPostprocessFunction postprocessLayerFeatures
+    LayerPostprocessFunction postprocessLayerFeatures,
+    TilePostprocessFunction postprocessTileFeatures
   ) implements Profile {
 
     TestProfile(
@@ -1701,8 +1757,16 @@ class PlanetilerTests {
       LayerPostprocessFunction postprocessLayerFeatures
     ) {
       this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
-        preprocessOsmRelation,
-        postprocessLayerFeatures);
+        preprocessOsmRelation, postprocessLayerFeatures, null);
+    }
+
+    TestProfile(
+      BiConsumer<SourceFeature, FeatureCollector> processFeature,
+      Function<OsmElement.Relation, List<OsmRelationInfo>> preprocessOsmRelation,
+      TilePostprocessFunction postprocessTileFeatures
+    ) {
+      this(TEST_PROFILE_NAME, TEST_PROFILE_DESCRIPTION, TEST_PROFILE_ATTRIBUTION, TEST_PROFILE_VERSION, processFeature,
+        preprocessOsmRelation, null, postprocessTileFeatures);
     }
 
     static TestProfile processSourceFeatures(BiConsumer<SourceFeature, FeatureCollector> processFeature) {
@@ -1712,7 +1776,7 @@ class PlanetilerTests {
     @Override
     public List<OsmRelationInfo> preprocessOsmRelation(
       OsmElement.Relation relation) {
-      return preprocessOsmRelation.apply(relation);
+      return preprocessOsmRelation == null ? null : preprocessOsmRelation.apply(relation);
     }
 
     @Override
@@ -1726,7 +1790,13 @@ class PlanetilerTests {
     @Override
     public List<VectorTile.Feature> postProcessLayerFeatures(String layer, int zoom,
       List<VectorTile.Feature> items) throws GeometryException {
-      return postprocessLayerFeatures.process(layer, zoom, items);
+      return postprocessLayerFeatures == null ? null : postprocessLayerFeatures.process(layer, zoom, items);
+    }
+
+    @Override
+    public Map<String, List<VectorTile.Feature>> postProcessTileFeatures(TileCoord tileCoord,
+      Map<String, List<VectorTile.Feature>> layers) throws GeometryException {
+      return postprocessTileFeatures == null ? null : postprocessTileFeatures.process(tileCoord, layers);
     }
   }
 

planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java

@@ -279,7 +279,7 @@ public class TestUtils {
         case UNKNOWN -> throw new IllegalArgumentException("cannot decompress \"UNKNOWN\"");
       };
       var decoded = VectorTile.decode(bytes).stream()
-        .map(feature -> feature(decodeSilently(feature.geometry()), feature.attrs())).toList();
+        .map(feature -> feature(decodeSilently(feature.geometry()), feature.layer(), feature.attrs())).toList();
       tiles.put(tile.coord(), decoded);
     }
     return tiles;
@@ -466,11 +466,32 @@ public class TestUtils {
 
   public record ComparableFeature(
     GeometryComparision geometry,
+    String layer,
     Map<String, Object> attrs
-  ) {}
+  ) {
+
+    @Override
+    public boolean equals(Object o) {
+      return o == this || (o instanceof ComparableFeature other &&
+        geometry.equals(other.geometry) &&
+        attrs.equals(other.attrs) &&
+        (layer == null || other.layer == null || Objects.equals(layer, other.layer)));
+    }
+
+    @Override
+    public int hashCode() {
+      int result = geometry.hashCode();
+      result = 31 * result + attrs.hashCode();
+      return result;
+    }
+  }
+
+  public static ComparableFeature feature(Geometry geom, String layer, Map<String, Object> attrs) {
+    return new ComparableFeature(new NormGeometry(geom), layer, attrs);
+  }
 
   public static ComparableFeature feature(Geometry geom, Map<String, Object> attrs) {
-    return new ComparableFeature(new NormGeometry(geom), attrs);
+    return new ComparableFeature(new NormGeometry(geom), null, attrs);
   }
 
   public static Map<String, Object> toMap(FeatureCollector.Feature feature, int zoom) {