From c5fb5e30bfdc5735b7b59f610d2615f001cc0c52 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Thu, 14 Dec 2023 06:53:21 -0500 Subject: [PATCH] Compute layer attr stats from actual output features --- .../com/onthegomap/planetiler/VectorTile.java | 15 ++- .../planetiler/archive/TileArchiveWriter.java | 13 +- .../archive/WriteableTileArchive.java | 2 +- .../planetiler/collection/FeatureGroup.java | 22 ++-- .../planetiler/mbtiles/Mbtiles.java | 5 +- .../stream/WriteableJsonStreamArchive.java | 6 +- .../stream/WriteableProtoStreamArchive.java | 10 +- .../planetiler/util/LayerAttrStats.java | 74 ++++++----- .../src/main/proto/stream_archive_proto.proto | 2 - .../planetiler/pmtiles/PmtilesTest.java | 10 +- .../stream/WriteableCsvArchiveTest.java | 6 +- .../WriteableJsonStreamArchiveTest.java | 30 ++--- .../WriteableProtoStreamArchiveTest.java | 14 +-- .../planetiler/util/LayerAttrStatsTest.java | 116 ++++-------------- 14 files changed, 132 insertions(+), 193 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java index cc7ca9b8..06d5c3d0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java @@ -27,6 +27,7 @@ import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.MutableCoordinateSequence; import com.onthegomap.planetiler.util.Hilbert; +import com.onthegomap.planetiler.util.LayerAttrStats; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -80,6 +81,7 @@ public class VectorTile { private static final int EXTENT = 4096; private static final double SIZE = 256d; private final Map layers = new LinkedHashMap<>(); + private LayerAttrStats.Updater.ForZoom layerStatsTracker = LayerAttrStats.Updater.ForZoom.NO_OP; private static int[] getCommands(Geometry input, int scale) { var encoder = new CommandEncoder(scale); @@ -467,12 +469,12 @@ public class VectorTile { if (features.isEmpty()) { return this; } - Layer layer = layers.get(layerName); if (layer == null) { layer = new Layer(); layers.put(layerName, layer); } + var statsTracker = layerStatsTracker.forLayer(layerName); for (Feature inFeature : features) { if (inFeature != null && inFeature.geometry().commands().length > 0) { @@ -481,8 +483,11 @@ public class VectorTile { for (Map.Entry e : inFeature.attrs().entrySet()) { // skip attribute without value if (e.getValue() != null) { - outFeature.tags.add(layer.key(e.getKey())); - outFeature.tags.add(layer.value(e.getValue())); + String key = e.getKey(); + Object value = e.getValue(); + outFeature.tags.add(layer.key(key)); + outFeature.tags.add(layer.value(value)); + statsTracker.accept(key, value); } } @@ -594,6 +599,10 @@ public class VectorTile { return layers.values().stream().allMatch(v -> v.encodedFeatures.isEmpty()) || containsOnlyFillsOrEdges(); } + public void trackLayerStats(LayerAttrStats.Updater.ForZoom layerStats) { + this.layerStatsTracker = layerStats; + } + enum Command { MOVE_TO(1), LINE_TO(2), diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index afa6048b..ad67d041 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -15,6 +15,7 @@ import com.onthegomap.planetiler.stats.Timer; import com.onthegomap.planetiler.util.DiskBacked; import com.onthegomap.planetiler.util.Format; import com.onthegomap.planetiler.util.Hashing; +import com.onthegomap.planetiler.util.LayerAttrStats; import com.onthegomap.planetiler.util.TileSizeStats; import com.onthegomap.planetiler.util.TileWeights; import com.onthegomap.planetiler.util.TilesetSummaryStatistics; @@ -59,6 +60,7 @@ public class TileArchiveWriter { private final AtomicReference lastTileWritten = new AtomicReference<>(); private final TileArchiveMetadata tileArchiveMetadata; private final TilesetSummaryStatistics tileStats; + private final LayerAttrStats layerAttrStats = new LayerAttrStats(); private TileArchiveWriter(Iterable inputTiles, WriteableTileArchive archive, PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) { @@ -105,9 +107,7 @@ public class TileArchiveWriter { readWorker = reader.readWorker(); } - TileArchiveWriter writer = - new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats() - .getTileStats()), stats); + TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats); var pipeline = WorkerPipeline.start("archive", stats); @@ -260,6 +260,7 @@ public class TileArchiveWriter { boolean skipFilled = config.skipFilledTiles(); var tileStatsUpdater = tileStats.threadLocalUpdater(); + var layerAttrStatsUpdater = layerAttrStats.handlerForThread(); for (TileBatch batch : prev) { List result = new ArrayList<>(batch.size()); FeatureGroup.TileFeatures last = null; @@ -277,7 +278,7 @@ public class TileArchiveWriter { layerStats = lastLayerStats; memoizedTiles.inc(); } else { - VectorTile tile = tileFeatures.getVectorTile(); + VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater); if (skipFilled && (lastIsFill = tile.containsOnlyFills())) { encoded = null; layerStats = null; @@ -333,7 +334,7 @@ public class TileArchiveWriter { var f = NumberFormat.getNumberInstance(Locale.getDefault()); f.setMaximumFractionDigits(5); - archive.initialize(tileArchiveMetadata); + archive.initialize(); var order = archive.tileOrder(); TileCoord lastTile = null; @@ -371,7 +372,7 @@ public class TileArchiveWriter { LOGGER.info("Finished z{} in {}", currentZ, time.stop()); } - archive.finish(tileArchiveMetadata); + archive.finish(tileArchiveMetadata.withLayerStats(layerAttrStats.getTileStats())); } @SuppressWarnings("java:S2629") diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index 26c2a3bc..d3852b08 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -30,7 +30,7 @@ public interface WriteableTileArchive extends Closeable { * Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any * required state here. */ - default void initialize(TileArchiveMetadata metadata) {} + default void initialize() {} /** * Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java index dfb3fa11..ba074ef7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; import org.msgpack.core.MessageBufferPacker; @@ -59,7 +58,6 @@ public final class FeatureGroup implements Iterable, private final CommonStringEncoder.AsByte commonLayerStrings = new CommonStringEncoder.AsByte(); private final CommonStringEncoder commonValueStrings = new CommonStringEncoder(100_000); private final Stats stats; - private final LayerAttrStats layerStats = new LayerAttrStats(); private final PlanetilerConfig config; private volatile boolean prepared = false; private final TileOrder tileOrder; @@ -141,14 +139,6 @@ public final class FeatureGroup implements Iterable, return (byte) ((geometry.geomType().asByte() & 0xff) | (geometry.scale() << 3)); } - /** - * Returns statistics about each layer written through {@link #newRenderedFeatureEncoder()} including min/max zoom, - * features on elements in that layer, and their types. - */ - public LayerAttrStats layerStats() { - return layerStats; - } - public long numFeaturesWritten() { return sorter.numFeaturesWritten(); } @@ -159,16 +149,13 @@ public final class FeatureGroup implements Iterable, // This method gets called billions of times when generating the planet, so these optimizations make a big difference: // 1) Re-use the same buffer packer to avoid allocating and resizing new byte arrays for every feature. private final MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); - // 2) Avoid a ThreadLocal lookup on every layer stats call by getting the handler for this thread once - private final Consumer threadLocalLayerStats = layerStats.handlerForThread(); - // 3) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values + // 2) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values // FeatureRenderer ensures that a separate VectorTileEncoder.Feature is used for each zoom level private VectorTile.Feature lastFeature = null; private byte[] lastEncodedValue = null; @Override public SortableFeature apply(RenderedFeature feature) { - threadLocalLayerStats.accept(feature); var group = feature.group().orElse(null); var thisFeature = feature.vectorTileFeature(); byte[] encodedValue; @@ -450,7 +437,14 @@ public final class FeatureGroup implements Iterable, } public VectorTile getVectorTile() { + return getVectorTile(null); + } + + public VectorTile getVectorTile(LayerAttrStats.Updater layerStats) { VectorTile tile = new VectorTile(); + if (layerStats != null) { + tile.trackLayerStats(layerStats.forZoom(tileCoord.z())); + } List items = new ArrayList<>(entries.size()); String currentLayer = null; for (SortableFeature entry : entries) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index b33181e2..1479c5eb 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -220,7 +220,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive } @Override - public void initialize(TileArchiveMetadata tileArchiveMetadata) { + public void initialize() { if (skipIndexCreation) { createTablesWithoutIndexes(); if (LOGGER.isInfoEnabled()) { @@ -230,12 +230,11 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive } else { createTablesWithIndexes(); } - - metadataTable().set(tileArchiveMetadata); } @Override public void finish(TileArchiveMetadata tileArchiveMetadata) { + metadataTable().set(tileArchiveMetadata); if (vacuumAnalyze) { vacuumAnalyze(); } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java index 6ec617a0..30d683ee 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java @@ -76,11 +76,11 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { } @Override - public void initialize(TileArchiveMetadata metadata) { + public void initialize() { if (writeTilesOnly) { return; } - writeEntryFlush(new InitializationEntry(metadata)); + writeEntryFlush(new InitializationEntry()); } @Override @@ -204,7 +204,7 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { } } - record InitializationEntry(TileArchiveMetadata metadata) implements Entry {} + record InitializationEntry() implements Entry {} record FinishEntry(TileArchiveMetadata metadata) implements Entry {} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java index ebe9f965..2c193a8c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java @@ -50,14 +50,8 @@ public final class WriteableProtoStreamArchive extends WriteableStreamArchive { } @Override - public void initialize(TileArchiveMetadata metadata) { - writeEntry( - StreamArchiveProto.Entry.newBuilder() - .setInitialization( - StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(toExportData(metadata)).build() - ) - .build() - ); + public void initialize() { + writeEntry(StreamArchiveProto.Entry.newBuilder().build()); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java index 9685f0fe..6ba65bf4 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java @@ -3,7 +3,6 @@ package com.onthegomap.planetiler.util; import com.fasterxml.jackson.annotation.JsonProperty; import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; -import com.onthegomap.planetiler.render.RenderedFeature; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -11,7 +10,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.TreeMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Consumer; import javax.annotation.concurrent.NotThreadSafe; import javax.annotation.concurrent.ThreadSafe; @@ -27,7 +25,7 @@ import javax.annotation.concurrent.ThreadSafe; * @see MBtiles spec */ @ThreadSafe -public class LayerAttrStats implements Consumer { +public class LayerAttrStats { /* * This utility is called for billions of features by multiple threads when processing the planet which can make * access to shared data structures a bottleneck. So give each thread an individual ThreadLocalLayerStatsHandler to @@ -63,6 +61,10 @@ public class LayerAttrStats implements Consumer { .toList(); } + void accept(String layer, int zoom, String key, Object value) { + handlerForThread().forZoom(zoom).forLayer(layer).accept(key, value); + } + public enum FieldType { @JsonProperty("Number") NUMBER, @@ -114,7 +116,7 @@ public class LayerAttrStats implements Consumer { /** Accepts features from a single thread that will be combined across all threads in {@link #getTileStats()}. */ @NotThreadSafe - private class ThreadLocalHandler implements Consumer { + private class ThreadLocalHandler implements Updater { private final Map layers = new TreeMap<>(); @@ -123,45 +125,53 @@ public class LayerAttrStats implements Consumer { } @Override - public void accept(RenderedFeature feature) { - var vectorTileFeature = feature.vectorTileFeature(); - var stats = layers.computeIfAbsent(vectorTileFeature.layer(), StatsForLayer::new); - stats.expandZoomRangeToInclude(feature.tile().z()); - for (var entry : vectorTileFeature.attrs().entrySet()) { - String key = entry.getKey(); - Object value = entry.getValue(); - - FieldType fieldType = null; - if (value instanceof Number) { - fieldType = FieldType.NUMBER; - } else if (value instanceof Boolean) { - fieldType = FieldType.BOOLEAN; - } else if (value != null) { - fieldType = FieldType.STRING; - } - if (fieldType != null) { - // widen different types to string - stats.fields.merge(key, fieldType, FieldType::merge); - } - } + public Updater.ForZoom forZoom(int zoom) { + return layer -> { + var stats = layers.computeIfAbsent(layer, StatsForLayer::new); + stats.expandZoomRangeToInclude(zoom); + return (key, value) -> { + FieldType fieldType = null; + if (value instanceof Number) { + fieldType = FieldType.NUMBER; + } else if (value instanceof Boolean) { + fieldType = FieldType.BOOLEAN; + } else if (value != null) { + fieldType = FieldType.STRING; + } + if (fieldType != null) { + // widen different types to string + stats.fields.merge(key, fieldType, FieldType::merge); + } + }; + }; } } /** * Returns a handler optimized for accepting features from a single thread. - *

- * Use this instead of {@link #accept(RenderedFeature)} */ - public Consumer handlerForThread() { + public Updater handlerForThread() { return layerStats.get(); } - @Override - public void accept(RenderedFeature feature) { - handlerForThread().accept(feature); + public interface Updater { + + ForZoom forZoom(int zoom); + + interface ForZoom { + + ForZoom NO_OP = layer -> (key, value) -> { + }; + + ForLayer forLayer(String layer); + + interface ForLayer { + void accept(String key, Object value); + } + } } - private static class StatsForLayer { + public static class StatsForLayer { private final String layer; private final Map fields = new HashMap<>(); diff --git a/planetiler-core/src/main/proto/stream_archive_proto.proto b/planetiler-core/src/main/proto/stream_archive_proto.proto index b33d94ae..17fd6961 100644 --- a/planetiler-core/src/main/proto/stream_archive_proto.proto +++ b/planetiler-core/src/main/proto/stream_archive_proto.proto @@ -1,4 +1,3 @@ - syntax = "proto3"; package com.onthegomap.planetiler.proto; @@ -19,7 +18,6 @@ message TileEntry { } message InitializationEntry { - Metadata metadata = 1; } message FinishEntry { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java index 08384a94..48c09cea 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java @@ -188,7 +188,7 @@ class PmtilesTest { var config = PlanetilerConfig.defaults(); var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); - in.initialize(metadata); + in.initialize(); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.empty())); @@ -259,7 +259,7 @@ class PmtilesTest { var channel = new SeekableInMemoryByteChannel(0); var in = WriteablePmtiles.newWriteToMemory(channel) ) { - in.initialize(input); + in.initialize(); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.empty())); @@ -299,7 +299,7 @@ class PmtilesTest { var config = PlanetilerConfig.defaults(); var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); - in.initialize(metadata); + in.initialize(); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42))); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42))); @@ -337,7 +337,7 @@ class PmtilesTest { var config = PlanetilerConfig.defaults(); var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); - in.initialize(metadata); + in.initialize(); var writer = in.newTileWriter(); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42))); writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42))); @@ -372,7 +372,7 @@ class PmtilesTest { var config = PlanetilerConfig.defaults(); var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config); - in.initialize(metadata); + in.initialize(); var writer = in.newTileWriter(); int ENTRIES = 20000; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java index aef43850..39fd244a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java @@ -32,7 +32,7 @@ class WriteableCsvArchiveTest { final Path csvFile = tempDir.resolve("out.csv"); try (var archive = WriteableCsvArchive.newWriteToFile(format, csvFile, defaultConfig)) { - archive.initialize(defaultMetadata); // ignored + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())); tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1))); @@ -67,7 +67,7 @@ class WriteableCsvArchiveTest { try ( var archive = WriteableCsvArchive.newWriteToFile(TileArchiveConfig.Format.CSV, csvFilePrimary, defaultConfig) ) { - archive.initialize(defaultMetadata); // ignored + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(11, 12, 1), new byte[]{0}, OptionalLong.empty())); tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(21, 22, 2), new byte[]{1}, OptionalLong.empty())); @@ -159,7 +159,7 @@ class WriteableCsvArchiveTest { final Path csvFile = tempDir.resolve("out.csv"); try (var archive = WriteableCsvArchive.newWriteToFile(TileArchiveConfig.Format.CSV, csvFile, config)) { - archive.initialize(defaultMetadata); + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0, 1}, OptionalLong.empty())); tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 1, 1), new byte[]{2, 3}, OptionalLong.empty())); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java index 21005e4f..0a9dd22f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java @@ -32,7 +32,7 @@ import org.locationtech.jts.geom.Envelope; class WriteableJsonStreamArchiveTest { private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, Arguments.of()); - private static final TileArchiveMetadata maxMetadataIn = + private static final TileArchiveMetadata MAX_METADATA_IN = new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3), new CoordinateXY(1.3, 3.7), 1.0, 2, 3, List.of( @@ -46,7 +46,7 @@ class WriteableJsonStreamArchiveTest { ), ImmutableMap.of("a", "b", "c", "d"), TileCompression.GZIP); - private static final String maxMetadataOut = """ + private static final String MAX_METADATA_OUT = """ { "name":"name", "description":"description", @@ -88,7 +88,7 @@ class WriteableJsonStreamArchiveTest { "c":"d" }""".lines().map(String::trim).collect(Collectors.joining("")); - private static final TileArchiveMetadata minMetadataIn = + private static final TileArchiveMetadata MIN_METADATA_IN = new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, null, null); private static final String MIN_METADATA_OUT = "{}"; @@ -98,21 +98,21 @@ class WriteableJsonStreamArchiveTest { final Path csvFile = tempDir.resolve("out.json"); try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFile, defaultConfig)) { - archive.initialize(maxMetadataIn); + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())); tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1))); } - archive.finish(minMetadataIn); + archive.finish(MIN_METADATA_IN); } assertEqualsDelimitedJson( """ - {"type":"initialization","metadata":%s} + {"type":"initialization"} {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="} {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="} {"type":"finish","metadata":%s} - """.formatted(maxMetadataOut, MIN_METADATA_OUT), + """.formatted(MIN_METADATA_OUT), Files.readString(csvFile) ); @@ -132,7 +132,7 @@ class WriteableJsonStreamArchiveTest { final var tile3 = new TileEncodingResult(TileCoord.ofXYZ(41, 42, 4), new byte[]{3}, OptionalLong.empty()); final var tile4 = new TileEncodingResult(TileCoord.ofXYZ(51, 52, 5), new byte[]{4}, OptionalLong.empty()); try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFilePrimary, defaultConfig)) { - archive.initialize(minMetadataIn); + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(tile0); tileWriter.write(tile1); @@ -144,16 +144,16 @@ class WriteableJsonStreamArchiveTest { try (var tileWriter = archive.newTileWriter()) { tileWriter.write(tile4); } - archive.finish(maxMetadataIn); + archive.finish(MAX_METADATA_IN); } assertEqualsDelimitedJson( """ - {"type":"initialization","metadata":%s} + {"type":"initialization"} {"type":"tile","x":11,"y":12,"z":1,"encodedData":"AA=="} {"type":"tile","x":21,"y":22,"z":2,"encodedData":"AQ=="} {"type":"finish","metadata":%s} - """.formatted(MIN_METADATA_OUT, maxMetadataOut), + """.formatted(MAX_METADATA_OUT), Files.readString(csvFilePrimary) ); @@ -199,11 +199,11 @@ class WriteableJsonStreamArchiveTest { final String expectedJson = """ - {"type":"initialization","metadata":%s} + {"type":"initialization"} {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="} {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="} {"type":"finish","metadata":%s} - """.formatted(MIN_METADATA_OUT, maxMetadataOut) + """.formatted(MAX_METADATA_OUT) .replace('\n', ' '); testTileOptions(tempDir, config, expectedJson); @@ -216,12 +216,12 @@ class WriteableJsonStreamArchiveTest { final Path csvFile = tempDir.resolve("out.json"); try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFile, config)) { - archive.initialize(minMetadataIn); + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty())); tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.empty())); } - archive.finish(maxMetadataIn); + archive.finish(MAX_METADATA_IN); } assertEqualsDelimitedJson(expectedJson, Files.readString(csvFile)); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java index b69f3a50..dd7299ef 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java @@ -76,7 +76,7 @@ class WriteableProtoStreamArchiveTest { final var tile0 = new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()); final var tile1 = new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1)); try (var archive = WriteableProtoStreamArchive.newWriteToFile(csvFile, defaultConfig)) { - archive.initialize(maxMetadataIn); + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(tile0); tileWriter.write(tile1); @@ -86,7 +86,7 @@ class WriteableProtoStreamArchiveTest { try (InputStream in = Files.newInputStream(csvFile)) { assertEquals( - List.of(wrapInit(maxMetadataOut), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)), + List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)), readAllEntries(in) ); } @@ -105,7 +105,7 @@ class WriteableProtoStreamArchiveTest { final var tile3 = new TileEncodingResult(TileCoord.ofXYZ(41, 42, 4), new byte[]{3}, OptionalLong.empty()); final var tile4 = new TileEncodingResult(TileCoord.ofXYZ(51, 52, 5), new byte[]{4}, OptionalLong.empty()); try (var archive = WriteableProtoStreamArchive.newWriteToFile(csvFilePrimary, defaultConfig)) { - archive.initialize(minMetadataIn); + archive.initialize(); try (var tileWriter = archive.newTileWriter()) { tileWriter.write(tile0); tileWriter.write(tile1); @@ -122,7 +122,7 @@ class WriteableProtoStreamArchiveTest { try (InputStream in = Files.newInputStream(csvFilePrimary)) { assertEquals( - List.of(wrapInit(minMetadataOut), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)), + List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)), readAllEntries(in) ); } @@ -167,10 +167,8 @@ class WriteableProtoStreamArchiveTest { .build(); } - private static StreamArchiveProto.Entry wrapInit(StreamArchiveProto.Metadata metadata) { - return StreamArchiveProto.Entry.newBuilder() - .setInitialization(StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(metadata).build()) - .build(); + private static StreamArchiveProto.Entry wrapInit() { + return StreamArchiveProto.Entry.newBuilder().build(); } private static StreamArchiveProto.Entry wrapFinish(StreamArchiveProto.Metadata metadata) { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java index 8835d68c..d3f567fa 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java @@ -2,13 +2,8 @@ package com.onthegomap.planetiler.util; import static org.junit.jupiter.api.Assertions.assertEquals; -import com.onthegomap.planetiler.VectorTile; -import com.onthegomap.planetiler.geo.GeoUtils; -import com.onthegomap.planetiler.geo.TileCoord; -import com.onthegomap.planetiler.render.RenderedFeature; -import java.util.Arrays; +import java.util.List; import java.util.Map; -import java.util.Optional; import org.junit.jupiter.api.Test; class LayerAttrStatsTest { @@ -17,109 +12,50 @@ class LayerAttrStatsTest { @Test void testEmptyLayerStats() { - assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{}), layerStats.getTileStats()); + assertEquals(List.of(), layerStats.getTileStats()); } @Test void testEmptyLayerStatsOneLayer() { - layerStats.accept(new RenderedFeature( - TileCoord.ofXYZ(1, 2, 3), - new VectorTile.Feature( - "layer1", - 1, - VectorTile.encodeGeometry(GeoUtils.point(1, 2)), - Map.of("a", 1, "b", "string", "c", true) - ), - 1, - Optional.empty() - )); - assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{ - new LayerAttrStats.VectorLayer("layer1", Map.of( - "a", LayerAttrStats.FieldType.NUMBER, - "b", LayerAttrStats.FieldType.STRING, - "c", LayerAttrStats.FieldType.BOOLEAN - ), 3, 3) - }), layerStats.getTileStats()); + layerStats.accept("layer1", 3, "a", 1); + layerStats.accept("layer1", 3, "b", "string"); + layerStats.accept("layer1", 3, "c", true); + assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of( + "a", LayerAttrStats.FieldType.NUMBER, + "b", LayerAttrStats.FieldType.STRING, + "c", LayerAttrStats.FieldType.BOOLEAN + ), 3, 3)), layerStats.getTileStats()); } @Test void testEmptyLayerStatsTwoLayers() { - layerStats.accept(new RenderedFeature( - TileCoord.ofXYZ(1, 2, 3), - new VectorTile.Feature( - "layer1", - 1, - VectorTile.encodeGeometry(GeoUtils.point(1, 2)), - Map.of() - ), - 1, - Optional.empty() - )); - layerStats.accept(new RenderedFeature( - TileCoord.ofXYZ(1, 2, 4), - new VectorTile.Feature( - "layer2", - 1, - VectorTile.encodeGeometry(GeoUtils.point(1, 2)), - Map.of("a", 1, "b", true, "c", true) - ), - 1, - Optional.empty() - )); - layerStats.accept(new RenderedFeature( - TileCoord.ofXYZ(1, 2, 1), - new VectorTile.Feature( - "layer2", - 1, - VectorTile.encodeGeometry(GeoUtils.point(1, 2)), - Map.of("a", 1, "b", true, "c", 1) - ), - 1, - Optional.empty() - )); - assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{ - new LayerAttrStats.VectorLayer("layer1", Map.of( - ), 3, 3), + layerStats.handlerForThread().forZoom(3).forLayer("layer1"); + layerStats.accept("layer2", 4, "a", 1); + layerStats.accept("layer2", 4, "b", true); + layerStats.accept("layer2", 4, "c", true); + layerStats.accept("layer2", 1, "a", 1); + layerStats.accept("layer2", 1, "b", true); + layerStats.accept("layer2", 1, "c", 1); + assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of( + ), 3, 3), new LayerAttrStats.VectorLayer("layer2", Map.of( "a", LayerAttrStats.FieldType.NUMBER, "b", LayerAttrStats.FieldType.BOOLEAN, "c", LayerAttrStats.FieldType.STRING - ), 1, 4) - }), layerStats.getTileStats()); + ), 1, 4)), layerStats.getTileStats()); } @Test void testMergeFromMultipleThreads() throws InterruptedException { - Thread t1 = new Thread(() -> layerStats.accept(new RenderedFeature( - TileCoord.ofXYZ(1, 2, 3), - new VectorTile.Feature( - "layer1", - 1, - VectorTile.encodeGeometry(GeoUtils.point(1, 2)), - Map.of("a", 1) - ), - 1, - Optional.empty() - ))); + layerStats.accept("layer1", 3, "a", true); + Thread t1 = new Thread(() -> layerStats.accept("layer1", 3, "a", 1)); t1.start(); - Thread t2 = new Thread(() -> layerStats.accept(new RenderedFeature( - TileCoord.ofXYZ(1, 2, 4), - new VectorTile.Feature( - "layer1", - 1, - VectorTile.encodeGeometry(GeoUtils.point(1, 2)), - Map.of("a", true) - ), - 1, - Optional.empty() - ))); + Thread t2 = new Thread(() -> layerStats.accept("layer1", 4, "a", true)); t2.start(); t1.join(); t2.join(); - assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{ - new LayerAttrStats.VectorLayer("layer1", Map.of( - "a", LayerAttrStats.FieldType.STRING - ), 3, 4) - }), layerStats.getTileStats()); + assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of( + "a", LayerAttrStats.FieldType.STRING + ), 3, 4)), layerStats.getTileStats()); } }