diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index c4016d5e..9459eece 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -495,6 +495,34 @@ public class VectorTile { return tile.build().toByteArray(); } + /** + * Returns true if this tile contains only polygon fills. + */ + public boolean containsOnlyFills() { + return containsOnlyFillsOrEdges(false); + } + + /** + * Returns true if this tile contains only polygon fills or horizontal/vertical edges that are likely to be repeated + * across tiles. + */ + public boolean containsOnlyFillsOrEdges() { + return containsOnlyFillsOrEdges(true); + } + + private boolean containsOnlyFillsOrEdges(boolean allowEdges) { + boolean empty = true; + for (var layer : layers.values()) { + for (var feature : layer.encodedFeatures) { + empty = false; + if (!feature.geometry.isFillOrEdge(allowEdges)) { + return false; + } + } + } + return !empty; + } + private enum Command { MOVE_TO(1), LINE_TO(2), @@ -519,12 +547,63 @@ public class VectorTile { */ public record VectorGeometry(int[] commands, GeometryType geomType, int scale) { + private static final int LEFT = 1; + private static final int RIGHT = 1 << 1; + private static final int TOP = 1 << 2; + private static final int BOTTOM = 1 << 3; + private static final int INSIDE = 0; + private static final int ALL = TOP | LEFT | RIGHT | BOTTOM; + public VectorGeometry { if (scale < 0) { throw new IllegalArgumentException("scale can not be less than 0, got: " + scale); } } + private static int getSide(int x, int y, int extent) { + int result = INSIDE; + if (x < 0) { + result |= LEFT; + } else if (x > extent) { + result |= RIGHT; + } + if (y < 0) { + result |= TOP; + } else if (y > extent) { + result |= BOTTOM; + } + return result; + } + + private static boolean slanted(int x1, int y1, int x2, int y2) { + return x1 != x2 && y1 != y2; + } + + private static boolean segmentCrossesTile(int x1, int y1, int x2, int y2, int extent) { + return (y1 >= 0 || y2 >= 0) && + (y1 <= extent || y2 <= extent) && + (x1 >= 0 || x2 >= 0) && + (x1 <= extent || x2 <= extent); + } + + private static boolean isSegmentInvalid(boolean allowEdges, int x1, int y1, int x2, int y2, int extent) { + boolean crossesTile = segmentCrossesTile(x1, y1, x2, y2, extent); + if (allowEdges) { + return crossesTile && slanted(x1, y1, x2, y2); + } else { + return crossesTile; + } + } + + + private static boolean visitedEnoughSides(boolean allowEdges, int sides) { + if (allowEdges) { + return ((sides & LEFT) > 0 && (sides & RIGHT) > 0) || ((sides & TOP) > 0 && (sides & BOTTOM) > 0); + } else { + return sides == ALL; + } + } + /** Converts an encoded geometry back to a JTS geometry. */ public Geometry decode() throws GeometryException { return decodeCommands(geomType, commands, scale); @@ -566,6 +645,95 @@ public class VectorTile { "], geomType=" + geomType + " (" + geomType.asByte() + ")]"; } + + /** Returns true if the encoded geometry is a polygon fill. */ + public boolean isFill() { + return isFillOrEdge(false); + } + + /** + * Returns true if the encoded geometry is a polygon fill, rectangle edge, or part of a horizontal/vertical line + * that is likely to be repeated across tiles. + */ + public boolean isFillOrEdge() { + return isFillOrEdge(true); + } + + /** + * Returns true if the encoded geometry is a polygon fill, or if {@code allowEdges == true} then also a rectangle + * edge, or part of a horizontal/vertical line that is likely to be repeated across tiles. + */ + public boolean isFillOrEdge(boolean allowEdges) { + if (geomType != GeometryType.POLYGON && (!allowEdges || geomType != GeometryType.LINE)) { + return false; + } + + boolean isLine = geomType == GeometryType.LINE; + + int extent = EXTENT << scale; + int visited = INSIDE; + int firstX = 0; + int firstY = 0; + int x = 0; + int y = 0; + + int geometryCount = commands.length; + int length = 0; + int command = 0; + int i = 0; + while (i < geometryCount) { + + if (length <= 0) { + length = commands[i++]; + command = length & ((1 << 3) - 1); + length = length >> 3; + if (isLine && length > 2) { + return false; + } + } + + if (length > 0) { + if (command == Command.CLOSE_PATH.value) { + if (isSegmentInvalid(allowEdges, x, y, firstX, firstY, extent) || + !visitedEnoughSides(allowEdges, visited)) { + return false; + } + length--; + continue; + } + + int dx = commands[i++]; + int dy = commands[i++]; + + length--; + + dx = zigZagDecode(dx); + dy = zigZagDecode(dy); + + int nextX = x + dx; + int nextY = y + dy; + + if (command == Command.MOVE_TO.value) { + firstX = nextX; + firstY = nextY; + if ((visited = getSide(firstX, firstY, extent)) == INSIDE) { + return false; + } + } else { + if (isSegmentInvalid(allowEdges, x, y, nextX, nextY, extent)) { + return false; + } + visited |= getSide(nextX, nextY, extent); + } + y = nextY; + x = nextX; + } + + } + + return visitedEnoughSides(allowEdges, visited); + } + } /** diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 9b18857c..b924ddc3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -41,7 +41,8 @@ public record PlanetilerConfig( double simplifyToleranceAtMaxZoom, double simplifyToleranceBelowMaxZoom, boolean osmLazyReads, - boolean compactDb + boolean compactDb, + boolean skipFilledTiles ) { public static final int MIN_MINZOOM = 0; @@ -142,7 +143,10 @@ public record PlanetilerConfig( false), arguments.getBoolean("compact_db", "Reduce the DB size by separating and deduping the tile data", - true) + true), + arguments.getBoolean("skip_filled_tiles", + "Skip writing tiles containing only polygon fills to the output", + false) ); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java index c6d3760a..fca49b1c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/MbtilesWriter.java @@ -49,7 +49,6 @@ public class MbtilesWriter { private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class); private static final long MAX_FEATURES_PER_BATCH = 10_000; private static final long MAX_TILES_PER_BATCH = 1_000; - private static final int MAX_FEATURES_HASHING_THRESHOLD = 5; private final Counter.Readable featuresProcessed; private final Counter memoizedTiles; private final Mbtiles db; @@ -258,7 +257,9 @@ public class MbtilesWriter { */ byte[] lastBytes = null, lastEncoded = null; Long lastTileDataHash = null; + boolean lastIsFill = false; boolean compactDb = config.compactDb(); + boolean skipFilled = config.skipFilledTiles(); for (TileBatch batch : prev) { Queue result = new ArrayDeque<>(batch.size()); @@ -270,23 +271,30 @@ public class MbtilesWriter { byte[] bytes, encoded; Long tileDataHash; if (tileFeatures.hasSameContents(last)) { + if (skipFilled && lastIsFill) { + continue; + } bytes = lastBytes; encoded = lastEncoded; tileDataHash = lastTileDataHash; memoizedTiles.inc(); } else { VectorTile en = tileFeatures.getVectorTileEncoder(); - encoded = en.encode(); - bytes = gzip(encoded); + if (skipFilled) { + lastIsFill = en.containsOnlyFills(); + if (lastIsFill) { + continue; + } + } + lastEncoded = encoded = en.encode(); + lastBytes = bytes = gzip(encoded); last = tileFeatures; - lastEncoded = encoded; - lastBytes = bytes; if (encoded.length > 1_000_000) { LOGGER.warn("{} {}kb uncompressed", tileFeatures.tileCoord(), encoded.length / 1024); } - if (compactDb && tileFeatures.getNumFeaturesToEmit() < MAX_FEATURES_HASHING_THRESHOLD) { + if (compactDb && en.containsOnlyFillsOrEdges()) { tileDataHash = tileFeatures.generateContentHash(); } else { tileDataHash = null; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 9aff431d..3978a185 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -675,6 +675,25 @@ class PlanetilerTests { )).stream().map(d -> d.geometry().geom().norm()).toList()); } + @Test + void testSkipFill() throws Exception { + var results = runWithReaderFeatures( + Map.of("threads", "1", "skip-filled-tiles", "true"), + List.of( + newReaderFeature(WORLD_POLYGON, Map.of()) + ), + (in, features) -> features.polygon("layer") + .setZoomRange(0, 6) + .setBufferPixels(4) + ); + + assertEquals(481, results.tiles.size()); + // spot-check one filled tile does not exist + assertNull(results.tiles.get(TileCoord.ofXYZ( + Z4_TILES / 2, Z4_TILES / 2, 4 + ))); + } + @ParameterizedTest @CsvSource({ "chesapeake.wkb, 4076", diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java index bde78c49..e8fe51ca 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/VectorTileTest.java @@ -33,6 +33,8 @@ import java.util.stream.Stream; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateXY; import org.locationtech.jts.geom.Geometry; @@ -333,6 +335,99 @@ class VectorTileTest { assertEquals("layer2", decoded.get(2).layer()); } + @ParameterizedTest + @CsvSource({ + "true,true,-1,-1,257,257", + "true,true,-5,-5,260,260", + "false,false,-1,-1,254,254", + "true,false,0,-1,257,257", + "true,false,-1,0,257,257", + "true,false,-1,-1,256,257", + "true,false,-1,-1,257,256", + + "false,false,0,0,1,1", + "false,false,1,1,2,2", + "false,false,1,1,2,2", + + "false,false,-10,-10,-5,-5", + "false,false,260,-10,270,5", + "false,false,-10,260,-5,270", + "false,false,260,260,270,270", + + "true,false,1,-1,257,257", + "true,false,1,-1,255,257", + "false,false,1,-1,255,255", + }) + void testRectangleIsFillOrEdge(boolean isFillOrEdge, boolean isFill, double x1, double y1, double x2, double y2) { + assertIsFillOrEdge(isFillOrEdge, isFill, rectangle(x1, y1, x2, y2)); + } + + @Test + void testRectangleWithSlantedEdgeIsNotFill() { + assertIsFillOrEdge(false, false, newPolygon( + 1, -1, + 257, -1, + 257, 257, + 2, 257, + 1, -1 + )); + } + + @ParameterizedTest + @CsvSource({ + "true,1,-1,1,257", + "false,1,1,1,257", + "false,1,-1,1,255", + "false,1,-1,2,257" + }) + void testLineIsEdge(boolean isFillOrEdge, double x1, double y1, double x2, double y2) { + assertIsFillOrEdge(isFillOrEdge, false, newLineString( + x1, y1, + x2, y2 + )); + } + + @Test + void testCrossBoundaryNotFillOrEdge() { + assertIsFillOrEdge(false, false, newPolygon( + -1, -1, + 257, -1, + -1, 257, + -1, -1 + )); + assertIsFillOrEdge(false, false, newPolygon( + 257, -1, + -1, 257, + -1, -1, + 257, -1 + )); + assertIsFillOrEdge(false, false, newPolygon( + -1, 257, + -1, -1, + 257, -1, + -1, 257 + )); + assertIsFillOrEdge(false, false, newPolygon( + -1, -1, + 513, -1, + -1, 513, + -1, -1 + )); + } + + private static void assertIsFillOrEdge(boolean isFillOrEdge, boolean isFill, Geometry geom) { + for (int rotation : List.of(0, 90, 180, 270)) { + var rectangle = + AffineTransformation.rotationInstance(Math.PI * rotation / 180, 128, 128).transform(geom); + for (int scale = 0; scale < 4; scale++) { + assertEquals(isFillOrEdge, VectorTile.encodeGeometry(rectangle, scale).isFillOrEdge(), + "scale=" + scale + " rotation=" + rotation); + assertEquals(isFill, VectorTile.encodeGeometry(rectangle, scale).isFill(), + "scale=" + scale + " rotation=" + rotation); + } + } + } + private void testRoundTripAttrs(Map attrs) { testRoundTrip(JTS_FACTORY.createPoint(new CoordinateXY(0, 0)), "layer", attrs, 1); }