From 3391abfef4a8eaf77d721206d823f6cf63e9170b Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 14 Apr 2023 05:18:02 -0400 Subject: [PATCH] tilestats --- .../planetiler/archive/TileArchiveConfig.java | 3 +- .../planetiler/archive/TileArchiveWriter.java | 17 ++++- .../planetiler/archive/TileArchives.java | 4 + .../archive/TileEncodingResult.java | 14 +++- .../archive/WriteableTileArchive.java | 3 +- .../planetiler/collection/FeatureGroup.java | 6 ++ .../onthegomap/planetiler/geo/TileCoord.java | 59 +++++++++++++++ .../planetiler/util/TileStatsArchive.java | 75 +++++++++++++++++++ 8 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileStatsArchive.java diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java index a4e736d2..b6bfdefc 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -173,7 +173,8 @@ public record TileArchiveConfig( public enum Format { MBTILES("mbtiles"), PMTILES("pmtiles"), - POSTGRES("postgres"); + POSTGRES("postgres"), + STATS("stats"); private final String id; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index 27a34144..236125e0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -19,6 +19,7 @@ import com.onthegomap.planetiler.worker.WorkQueue; import com.onthegomap.planetiler.worker.Worker; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; +import java.time.Duration; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -229,6 +230,7 @@ public class TileArchiveWriter { Long lastTileDataHash = null; boolean lastIsFill = false; boolean skipFilled = config.skipFilledTiles(); + long time = -1; for (TileBatch batch : prev) { Queue result = new ArrayDeque<>(batch.size()); @@ -239,12 +241,15 @@ public class TileArchiveWriter { featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed()); byte[] bytes, encoded; Long tileDataHash; + boolean cached = false; if (tileFeatures.hasSameContents(last)) { bytes = lastBytes; encoded = lastEncoded; tileDataHash = lastTileDataHash; memoizedTiles.inc(); + cached = true; } else { + long start = System.nanoTime(); VectorTile en = tileFeatures.getVectorTileEncoder(); if (skipFilled && (lastIsFill = en.containsOnlyFills())) { encoded = null; @@ -267,6 +272,7 @@ public class TileArchiveWriter { tileDataHash = null; } lastTileDataHash = tileDataHash; + time = System.nanoTime() - start; } if (skipFilled && lastIsFill) { continue; @@ -276,8 +282,15 @@ public class TileArchiveWriter { totalTileSizesByZoom[zoom].incBy(encodedLength); maxTileSizesByZoom[zoom].accumulate(encodedLength); result.add( - new TileEncodingResult(tileFeatures.tileCoord(), bytes, - tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash)) + new TileEncodingResult( + tileFeatures.tileCoord(), + bytes, + tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash), + tileFeatures.getInputFeatureBytes(), + tileFeatures.getNumFeaturesProcessed(), + cached, + Duration.ofNanos(time).toMillis() + ) ); } // hand result off to writer diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java index d100d055..b083c70f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java @@ -5,7 +5,9 @@ import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; import com.onthegomap.planetiler.util.Pgtiles; +import com.onthegomap.planetiler.util.TileStatsArchive; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.nio.file.Path; /** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */ @@ -47,6 +49,7 @@ public class TileArchives { .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION))); case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath()); case POSTGRES -> Pgtiles.writer(archive.uri(), options); + case STATS -> new TileStatsArchive(archive.getLocalPath()); }; } @@ -62,6 +65,7 @@ public class TileArchives { case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options); case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath()); case POSTGRES -> Pgtiles.reader(archive.uri(), options); + case STATS -> throw new UnsupportedEncodingException(); }; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java index 5df7c0a4..8c9a92f3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java @@ -9,8 +9,20 @@ public record TileEncodingResult( TileCoord coord, byte[] tileData, /** will always be empty in non-compact mode and might also be empty in compact mode */ - OptionalLong tileDataHash + OptionalLong tileDataHash, + long inputBytes, + long inputFeatures, + boolean cached, + long time ) { + public TileEncodingResult( + TileCoord coord, + byte[] tileData, + /** will always be empty in non-compact mode and might also be empty in compact mode */ + OptionalLong tileDataHash + ) { + this(coord, tileData, tileDataHash, 0, 0, false, 0); + } @Override public int hashCode() { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index 3c3b6df7..66160522 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -50,7 +50,8 @@ public interface WriteableTileArchive extends Closeable { // TODO: exists for compatibility reasons default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) { - write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash())); + write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash(), 0, + 0, false, 0)); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index 6e415f4e..32279f7e 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -355,6 +355,7 @@ public final class FeatureGroup implements Iterable, private final TileCoord tileCoord; private final List entries = new ArrayList<>(); private final AtomicLong numFeaturesProcessed = new AtomicLong(0); + private final AtomicLong inputFeatureBytes = new AtomicLong(0); private LongLongHashMap counts = null; private byte lastLayer = Byte.MAX_VALUE; @@ -507,6 +508,7 @@ public final class FeatureGroup implements Iterable, void add(SortableFeature entry) { numFeaturesProcessed.incrementAndGet(); + inputFeatureBytes.addAndGet(entry.value().length); long key = entry.key(); if (extractHasGroupFromKey(key)) { byte thisLayer = extractLayerIdFromKey(key); @@ -535,5 +537,9 @@ public final class FeatureGroup implements Iterable, ", num entries=" + entries.size() + '}'; } + + public long getInputFeatureBytes() { + return inputFeatureBytes.get(); + } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java index 90212880..65ac3d85 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java @@ -4,6 +4,7 @@ import static com.onthegomap.planetiler.config.PlanetilerConfig.MAX_MAXZOOM; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.Hilbert; +import java.util.regex.Pattern; import javax.annotation.concurrent.Immutable; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.CoordinateXY; @@ -26,6 +27,64 @@ import org.locationtech.jts.geom.Envelope; @Immutable public record TileCoord(int encoded, int x, int y, int z) implements Comparable { + public static void main(String[] args) { + + + insertLinks(""" + | tile | inbytes | infeatures | outbytes | time | + |------------|---------|------------|----------|-------| + | 6/52/24 | 316,843 | 7,592 | 118,047 | 0.739 | + | 6/50/26 | 263,488 | 5,587 | 101,577 | 0.668 | + | 6/52/26 | 332,374 | 6,722 | 138,996 | 0.667 | + | 10/308/381 | 604,776 | 6,533 | 362,984 | 0.662 | + | 6/45/25 | 147,850 | 2,085 | 77,298 | 0.634 | + | 6/51/26 | 221,077 | 4,312 | 89,517 | 0.63 | + | 6/51/27 | 264,509 | 5,301 | 102,592 | 0.617 | + | 6/15/22 | 124,147 | 1,480 | 70,605 | 0.597 | + | 6/52/27 | 310,144 | 6,443 | 122,242 | 0.593 | + | 1/0/0 | 149,781 | 3,906 | 63,429 | 0.586 | + """); + insertLinks(""" + | tile | inbytes | infeatures | outbytes | time | + |--------------|---------|------------|----------|--------| + | 7/68/33 | 574,106 | 3,310 | 253,432 | 48.02 | + | 13/6527/4239 | 2,792,615 | 106,023 | 135,957 | 35.365 | + | 13/6661/4261 | 3,111,509 | 114,647 | 130,210 | 32.91 | + | 13/6661/4262 | 3,046,934 | 111,292 | 126,727 | 32.741 | + | 13/6527/4240 | 2,344,566 | 88,393 | 116,492 | 29.108 | + | 13/6527/4238 | 2,900,531 | 105,839 | 143,392 | 28.857 | + | 13/6526/4236 | 3,025,297 | 106,137 | 137,592 | 28.829 | + | 7/68/61 | 731,969 | 5,077 | 268,917 | 28.583 | + | 13/6527/4237 | 2,843,197 | 99,475 | 144,424 | 28.142 | + | 13/6607/4274 | 2,228,848 | 90,111 | 92,170 | 27.981 | + """); + insertLinks(""" + | tile | inbytes | infeatures | outbytes | time | + |--------------|---------|------------|----------|--------| + | 10/236/413 | 6,676,345 | 179,762 | 17,9050 | 16.77 | + | 13/3037/4648 | 2,991,716 | 116,502 | 13,0376 | 13.434 | + | 7/68/33 | 574,320 | 3,311 | 25,3391 | 11.826 | + | 13/1436/3313 | 2,554,569 | 104,277 | 12,5866 | 11.517 | + | 13/3038/4646 | 2,664,982 | 103,904 | 13,4165 | 11.47 | + | 13/3037/4647 | 2,579,326 | 99,017 | 13,9578 | 11.2 | + | 13/6527/4239 | 2,792,855 | 106,027 | 13,6069 | 11.08 | + | 13/3036/4648 | 2,553,943 | 96,097 | 13,6788 | 11.064 | + | 13/3039/4646 | 2,629,057 | 104,042 | 13,3678 | 11.02 | + | 13/6661/4261 | 3,111,945 | 114,652 | 13,0269 | 10.987 | + """); + } + + private static void insertLinks(String input) { + var matcher = Pattern.compile("([0-9]+)/([0-9]+)/([0-9]+)").matcher(input); + System.out.println(matcher.replaceAll(matchResult -> { + int z = Integer.parseInt(matchResult.group(1)); + int x = Integer.parseInt(matchResult.group(2)); + int y = Integer.parseInt(matchResult.group(3)); + String url = TileCoord.ofXYZ(x, y, z).getDebugUrl(); + return "[" + z + "/" + x + "/" + y + "](" + url + ")"; + })); + } + private static final int[] ZOOM_START_INDEX = new int[MAX_MAXZOOM + 1]; static { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileStatsArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileStatsArchive.java new file mode 100644 index 00000000..3769dec0 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/TileStatsArchive.java @@ -0,0 +1,75 @@ +package com.onthegomap.planetiler.util; + +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.archive.WriteableTileArchive; +import com.onthegomap.planetiler.geo.TileOrder; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TileStatsArchive implements WriteableTileArchive { + private static final Logger LOGGER = LoggerFactory.getLogger(TileStatsArchive.class); + private final BufferedWriter writer; + + public TileStatsArchive(Path output) throws IOException { + this.writer = Files.newBufferedWriter(output, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + } + + @Override + public boolean deduplicates() { + return false; + } + + @Override + public TileOrder tileOrder() { + return TileOrder.TMS; + } + + @Override + public TileWriter newTileWriter() { + return new BulkLoader(); + } + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public void initialize(TileArchiveMetadata metadata) { + try { + writer.write("z\tx\ty\tinbytes\tinfeatures\tcached\toutbytes\ttime\n"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private class BulkLoader implements TileWriter { + + @Override + public void write(TileEncodingResult encodingResult) { + try { + writer.write("%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d%n".formatted( + encodingResult.coord().z(), + encodingResult.coord().x(), + encodingResult.coord().y(), + encodingResult.inputBytes(), + encodingResult.inputFeatures(), + encodingResult.cached() ? 1 : 0, + encodingResult.tileData().length, + encodingResult.time() + )); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() {} + } +}