mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Compute layer attr stats from actual output features

fix-749
Mike Barry 2023-12-14 06:53:21 -05:00
rodzic d98d5d6ae1
commit c5fb5e30bf
14 zmienionych plików z 132 dodań i 193 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

15
planetiler-core/src/main/java/com/onthegomap/planetiler/VectorTile.java
Wyświetl plik

@ -27,6 +27,7 @@ import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.geo.GeometryType;
import com.onthegomap.planetiler.geo.MutableCoordinateSequence;
import com.onthegomap.planetiler.util.Hilbert;
import com.onthegomap.planetiler.util.LayerAttrStats;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@ -80,6 +81,7 @@ public class VectorTile {
private static final int EXTENT = 4096;
private static final double SIZE = 256d;
private final Map<String, Layer> layers = new LinkedHashMap<>();
private LayerAttrStats.Updater.ForZoom layerStatsTracker = LayerAttrStats.Updater.ForZoom.NO_OP;
private static int[] getCommands(Geometry input, int scale) {
var encoder = new CommandEncoder(scale);
@ -467,12 +469,12 @@ public class VectorTile {
if (features.isEmpty()) {
return this;
}
Layer layer = layers.get(layerName);
if (layer == null) {
layer = new Layer();
layers.put(layerName, layer);
}
var statsTracker = layerStatsTracker.forLayer(layerName);
for (Feature inFeature : features) {
if (inFeature != null && inFeature.geometry().commands().length > 0) {
@ -481,8 +483,11 @@ public class VectorTile {
for (Map.Entry<String, ?> e : inFeature.attrs().entrySet()) {
// skip attribute without value
if (e.getValue() != null) {
outFeature.tags.add(layer.key(e.getKey()));
outFeature.tags.add(layer.value(e.getValue()));
String key = e.getKey();
Object value = e.getValue();
outFeature.tags.add(layer.key(key));
outFeature.tags.add(layer.value(value));
statsTracker.accept(key, value);
}
}
@ -594,6 +599,10 @@ public class VectorTile {
return layers.values().stream().allMatch(v -> v.encodedFeatures.isEmpty()) || containsOnlyFillsOrEdges();
}
public void trackLayerStats(LayerAttrStats.Updater.ForZoom layerStats) {
this.layerStatsTracker = layerStats;
}
enum Command {
MOVE_TO(1),
LINE_TO(2),

13
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java
Wyświetl plik

@ -15,6 +15,7 @@ import com.onthegomap.planetiler.stats.Timer;
import com.onthegomap.planetiler.util.DiskBacked;
import com.onthegomap.planetiler.util.Format;
import com.onthegomap.planetiler.util.Hashing;
import com.onthegomap.planetiler.util.LayerAttrStats;
import com.onthegomap.planetiler.util.TileSizeStats;
import com.onthegomap.planetiler.util.TileWeights;
import com.onthegomap.planetiler.util.TilesetSummaryStatistics;
@ -59,6 +60,7 @@ public class TileArchiveWriter {
private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
private final TileArchiveMetadata tileArchiveMetadata;
private final TilesetSummaryStatistics tileStats;
private final LayerAttrStats layerAttrStats = new LayerAttrStats();
private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, WriteableTileArchive archive,
PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, Stats stats) {
@ -105,9 +107,7 @@ public class TileArchiveWriter {
readWorker = reader.readWorker();
}
TileArchiveWriter writer =
new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata.withLayerStats(features.layerStats()
.getTileStats()), stats);
TileArchiveWriter writer = new TileArchiveWriter(inputTiles, output, config, tileArchiveMetadata, stats);
var pipeline = WorkerPipeline.start("archive", stats);
@ -260,6 +260,7 @@ public class TileArchiveWriter {
boolean skipFilled = config.skipFilledTiles();
var tileStatsUpdater = tileStats.threadLocalUpdater();
var layerAttrStatsUpdater = layerAttrStats.handlerForThread();
for (TileBatch batch : prev) {
List<TileEncodingResult> result = new ArrayList<>(batch.size());
FeatureGroup.TileFeatures last = null;
@ -277,7 +278,7 @@ public class TileArchiveWriter {
layerStats = lastLayerStats;
memoizedTiles.inc();
} else {
VectorTile tile = tileFeatures.getVectorTile();
VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater);
if (skipFilled && (lastIsFill = tile.containsOnlyFills())) {
encoded = null;
layerStats = null;
@ -333,7 +334,7 @@ public class TileArchiveWriter {
var f = NumberFormat.getNumberInstance(Locale.getDefault());
f.setMaximumFractionDigits(5);
archive.initialize(tileArchiveMetadata);
archive.initialize();
var order = archive.tileOrder();
TileCoord lastTile = null;
@ -371,7 +372,7 @@ public class TileArchiveWriter {
LOGGER.info("Finished z{} in {}", currentZ, time.stop());
}
archive.finish(tileArchiveMetadata);
archive.finish(tileArchiveMetadata.withLayerStats(layerAttrStats.getTileStats()));
}
@SuppressWarnings("java:S2629")

2
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java
Wyświetl plik

@ -30,7 +30,7 @@ public interface WriteableTileArchive extends Closeable {
* Called before any tiles are written into {@link TileWriter}. Implementations of TileArchive should set up any
* required state here.
*/
default void initialize(TileArchiveMetadata metadata) {}
default void initialize() {}
/**
* Implementations should return a object that implements {@link TileWriter} The specific TileWriter returned might

22
planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java
Wyświetl plik

@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Function;
import javax.annotation.concurrent.NotThreadSafe;
import org.msgpack.core.MessageBufferPacker;
@ -59,7 +58,6 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
private final CommonStringEncoder.AsByte commonLayerStrings = new CommonStringEncoder.AsByte();
private final CommonStringEncoder commonValueStrings = new CommonStringEncoder(100_000);
private final Stats stats;
private final LayerAttrStats layerStats = new LayerAttrStats();
private final PlanetilerConfig config;
private volatile boolean prepared = false;
private final TileOrder tileOrder;
@ -141,14 +139,6 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
return (byte) ((geometry.geomType().asByte() & 0xff) | (geometry.scale() << 3));
}
/**
* Returns statistics about each layer written through {@link #newRenderedFeatureEncoder()} including min/max zoom,
* features on elements in that layer, and their types.
*/
public LayerAttrStats layerStats() {
return layerStats;
}
public long numFeaturesWritten() {
return sorter.numFeaturesWritten();
}
@ -159,16 +149,13 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
// This method gets called billions of times when generating the planet, so these optimizations make a big difference:
// 1) Re-use the same buffer packer to avoid allocating and resizing new byte arrays for every feature.
private final MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
// 2) Avoid a ThreadLocal lookup on every layer stats call by getting the handler for this thread once
private final Consumer<RenderedFeature> threadLocalLayerStats = layerStats.handlerForThread();
// 3) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values
// 2) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values
// FeatureRenderer ensures that a separate VectorTileEncoder.Feature is used for each zoom level
private VectorTile.Feature lastFeature = null;
private byte[] lastEncodedValue = null;
@Override
public SortableFeature apply(RenderedFeature feature) {
threadLocalLayerStats.accept(feature);
var group = feature.group().orElse(null);
var thisFeature = feature.vectorTileFeature();
byte[] encodedValue;
@ -450,7 +437,14 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
}
public VectorTile getVectorTile() {
return getVectorTile(null);
}
public VectorTile getVectorTile(LayerAttrStats.Updater layerStats) {
VectorTile tile = new VectorTile();
if (layerStats != null) {
tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
}
List<VectorTile.Feature> items = new ArrayList<>(entries.size());
String currentLayer = null;
for (SortableFeature entry : entries) {

5
planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
Wyświetl plik

@ -220,7 +220,7 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
}
@Override
public void initialize(TileArchiveMetadata tileArchiveMetadata) {
public void initialize() {
if (skipIndexCreation) {
createTablesWithoutIndexes();
if (LOGGER.isInfoEnabled()) {
@ -230,12 +230,11 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
} else {
createTablesWithIndexes();
}
metadataTable().set(tileArchiveMetadata);
}
@Override
public void finish(TileArchiveMetadata tileArchiveMetadata) {
metadataTable().set(tileArchiveMetadata);
if (vacuumAnalyze) {
vacuumAnalyze();
}

6
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java
Wyświetl plik

@ -76,11 +76,11 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
}
@Override
public void initialize(TileArchiveMetadata metadata) {
public void initialize() {
if (writeTilesOnly) {
return;
}
writeEntryFlush(new InitializationEntry(metadata));
writeEntryFlush(new InitializationEntry());
}
@Override
@ -204,7 +204,7 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
}
}
record InitializationEntry(TileArchiveMetadata metadata) implements Entry {}
record InitializationEntry() implements Entry {}
record FinishEntry(TileArchiveMetadata metadata) implements Entry {}

10
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java
Wyświetl plik

@ -50,14 +50,8 @@ public final class WriteableProtoStreamArchive extends WriteableStreamArchive {
}
@Override
public void initialize(TileArchiveMetadata metadata) {
writeEntry(
StreamArchiveProto.Entry.newBuilder()
.setInitialization(
StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(toExportData(metadata)).build()
)
.build()
);
public void initialize() {
writeEntry(StreamArchiveProto.Entry.newBuilder().build());
}
@Override

74
planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerAttrStats.java
Wyświetl plik

@ -3,7 +3,6 @@ package com.onthegomap.planetiler.util;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.onthegomap.planetiler.archive.WriteableTileArchive;
import com.onthegomap.planetiler.mbtiles.Mbtiles;
import com.onthegomap.planetiler.render.RenderedFeature;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -11,7 +10,6 @@ import java.util.Optional;
import java.util.OptionalInt;
import java.util.TreeMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import javax.annotation.concurrent.NotThreadSafe;
import javax.annotation.concurrent.ThreadSafe;
@ -27,7 +25,7 @@ import javax.annotation.concurrent.ThreadSafe;
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#content">MBtiles spec</a>
*/
@ThreadSafe
public class LayerAttrStats implements Consumer<RenderedFeature> {
public class LayerAttrStats {
/*
* This utility is called for billions of features by multiple threads when processing the planet which can make
* access to shared data structures a bottleneck. So give each thread an individual ThreadLocalLayerStatsHandler to
@ -63,6 +61,10 @@ public class LayerAttrStats implements Consumer<RenderedFeature> {
.toList();
}
void accept(String layer, int zoom, String key, Object value) {
handlerForThread().forZoom(zoom).forLayer(layer).accept(key, value);
}
public enum FieldType {
@JsonProperty("Number")
NUMBER,
@ -114,7 +116,7 @@ public class LayerAttrStats implements Consumer<RenderedFeature> {
/** Accepts features from a single thread that will be combined across all threads in {@link #getTileStats()}. */
@NotThreadSafe
private class ThreadLocalHandler implements Consumer<RenderedFeature> {
private class ThreadLocalHandler implements Updater {
private final Map<String, StatsForLayer> layers = new TreeMap<>();
@ -123,45 +125,53 @@ public class LayerAttrStats implements Consumer<RenderedFeature> {
}
@Override
public void accept(RenderedFeature feature) {
var vectorTileFeature = feature.vectorTileFeature();
var stats = layers.computeIfAbsent(vectorTileFeature.layer(), StatsForLayer::new);
stats.expandZoomRangeToInclude(feature.tile().z());
for (var entry : vectorTileFeature.attrs().entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
FieldType fieldType = null;
if (value instanceof Number) {
fieldType = FieldType.NUMBER;
} else if (value instanceof Boolean) {
fieldType = FieldType.BOOLEAN;
} else if (value != null) {
fieldType = FieldType.STRING;
}
if (fieldType != null) {
// widen different types to string
stats.fields.merge(key, fieldType, FieldType::merge);
}
}
public Updater.ForZoom forZoom(int zoom) {
return layer -> {
var stats = layers.computeIfAbsent(layer, StatsForLayer::new);
stats.expandZoomRangeToInclude(zoom);
return (key, value) -> {
FieldType fieldType = null;
if (value instanceof Number) {
fieldType = FieldType.NUMBER;
} else if (value instanceof Boolean) {
fieldType = FieldType.BOOLEAN;
} else if (value != null) {
fieldType = FieldType.STRING;
}
if (fieldType != null) {
// widen different types to string
stats.fields.merge(key, fieldType, FieldType::merge);
}
};
};
}
}
/**
* Returns a handler optimized for accepting features from a single thread.
* <p>
* Use this instead of {@link #accept(RenderedFeature)}
*/
public Consumer<RenderedFeature> handlerForThread() {
public Updater handlerForThread() {
return layerStats.get();
}
@Override
public void accept(RenderedFeature feature) {
handlerForThread().accept(feature);
public interface Updater {
ForZoom forZoom(int zoom);
interface ForZoom {
ForZoom NO_OP = layer -> (key, value) -> {
};
ForLayer forLayer(String layer);
interface ForLayer {
void accept(String key, Object value);
}
}
}
private static class StatsForLayer {
public static class StatsForLayer {
private final String layer;
private final Map<String, FieldType> fields = new HashMap<>();

2
planetiler-core/src/main/proto/stream_archive_proto.proto
Wyświetl plik

@ -1,4 +1,3 @@
syntax = "proto3";
package com.onthegomap.planetiler.proto;
@ -19,7 +18,6 @@ message TileEntry {
}
message InitializationEntry {
Metadata metadata = 1;
}
message FinishEntry {

10
planetiler-core/src/test/java/com/onthegomap/planetiler/pmtiles/PmtilesTest.java
Wyświetl plik

@ -188,7 +188,7 @@ class PmtilesTest {
var config = PlanetilerConfig.defaults();
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
in.initialize(metadata);
in.initialize();
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.empty()));
@ -259,7 +259,7 @@ class PmtilesTest {
var channel = new SeekableInMemoryByteChannel(0);
var in = WriteablePmtiles.newWriteToMemory(channel)
) {
in.initialize(input);
in.initialize();
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.empty()));
@ -299,7 +299,7 @@ class PmtilesTest {
var config = PlanetilerConfig.defaults();
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
in.initialize(metadata);
in.initialize();
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
@ -337,7 +337,7 @@ class PmtilesTest {
var config = PlanetilerConfig.defaults();
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
in.initialize(metadata);
in.initialize();
var writer = in.newTileWriter();
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 1), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
writer.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0xa, 0x2}, OptionalLong.of(42)));
@ -372,7 +372,7 @@ class PmtilesTest {
var config = PlanetilerConfig.defaults();
var metadata = new TileArchiveMetadata(new Profile.NullProfile(), config);
in.initialize(metadata);
in.initialize();
var writer = in.newTileWriter();
int ENTRIES = 20000;

6
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
Wyświetl plik

@ -32,7 +32,7 @@ class WriteableCsvArchiveTest {
final Path csvFile = tempDir.resolve("out.csv");
try (var archive = WriteableCsvArchive.newWriteToFile(format, csvFile, defaultConfig)) {
archive.initialize(defaultMetadata); // ignored
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()));
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1)));
@ -67,7 +67,7 @@ class WriteableCsvArchiveTest {
try (
var archive = WriteableCsvArchive.newWriteToFile(TileArchiveConfig.Format.CSV, csvFilePrimary, defaultConfig)
) {
archive.initialize(defaultMetadata); // ignored
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(11, 12, 1), new byte[]{0}, OptionalLong.empty()));
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(21, 22, 2), new byte[]{1}, OptionalLong.empty()));
@ -159,7 +159,7 @@ class WriteableCsvArchiveTest {
final Path csvFile = tempDir.resolve("out.csv");
try (var archive = WriteableCsvArchive.newWriteToFile(TileArchiveConfig.Format.CSV, csvFile, config)) {
archive.initialize(defaultMetadata);
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0, 1}, OptionalLong.empty()));
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 1, 1), new byte[]{2, 3}, OptionalLong.empty()));

30
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchiveTest.java
Wyświetl plik

@ -32,7 +32,7 @@ import org.locationtech.jts.geom.Envelope;
class WriteableJsonStreamArchiveTest {
private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, Arguments.of());
private static final TileArchiveMetadata maxMetadataIn =
private static final TileArchiveMetadata MAX_METADATA_IN =
new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3),
new CoordinateXY(1.3, 3.7), 1.0, 2, 3,
List.of(
@ -46,7 +46,7 @@ class WriteableJsonStreamArchiveTest {
),
ImmutableMap.of("a", "b", "c", "d"),
TileCompression.GZIP);
private static final String maxMetadataOut = """
private static final String MAX_METADATA_OUT = """
{
"name":"name",
"description":"description",
@ -88,7 +88,7 @@ class WriteableJsonStreamArchiveTest {
"c":"d"
}""".lines().map(String::trim).collect(Collectors.joining(""));
private static final TileArchiveMetadata minMetadataIn =
private static final TileArchiveMetadata MIN_METADATA_IN =
new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, null, null);
private static final String MIN_METADATA_OUT = "{}";
@ -98,21 +98,21 @@ class WriteableJsonStreamArchiveTest {
final Path csvFile = tempDir.resolve("out.json");
try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFile, defaultConfig)) {
archive.initialize(maxMetadataIn);
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()));
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1)));
}
archive.finish(minMetadataIn);
archive.finish(MIN_METADATA_IN);
}
assertEqualsDelimitedJson(
"""
{"type":"initialization","metadata":%s}
{"type":"initialization"}
{"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
{"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
{"type":"finish","metadata":%s}
""".formatted(maxMetadataOut, MIN_METADATA_OUT),
""".formatted(MIN_METADATA_OUT),
Files.readString(csvFile)
);
@ -132,7 +132,7 @@ class WriteableJsonStreamArchiveTest {
final var tile3 = new TileEncodingResult(TileCoord.ofXYZ(41, 42, 4), new byte[]{3}, OptionalLong.empty());
final var tile4 = new TileEncodingResult(TileCoord.ofXYZ(51, 52, 5), new byte[]{4}, OptionalLong.empty());
try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFilePrimary, defaultConfig)) {
archive.initialize(minMetadataIn);
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(tile0);
tileWriter.write(tile1);
@ -144,16 +144,16 @@ class WriteableJsonStreamArchiveTest {
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(tile4);
}
archive.finish(maxMetadataIn);
archive.finish(MAX_METADATA_IN);
}
assertEqualsDelimitedJson(
"""
{"type":"initialization","metadata":%s}
{"type":"initialization"}
{"type":"tile","x":11,"y":12,"z":1,"encodedData":"AA=="}
{"type":"tile","x":21,"y":22,"z":2,"encodedData":"AQ=="}
{"type":"finish","metadata":%s}
""".formatted(MIN_METADATA_OUT, maxMetadataOut),
""".formatted(MAX_METADATA_OUT),
Files.readString(csvFilePrimary)
);
@ -199,11 +199,11 @@ class WriteableJsonStreamArchiveTest {
final String expectedJson =
"""
{"type":"initialization","metadata":%s}
{"type":"initialization"}
{"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
{"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
{"type":"finish","metadata":%s}
""".formatted(MIN_METADATA_OUT, maxMetadataOut)
""".formatted(MAX_METADATA_OUT)
.replace('\n', ' ');
testTileOptions(tempDir, config, expectedJson);
@ -216,12 +216,12 @@ class WriteableJsonStreamArchiveTest {
final Path csvFile = tempDir.resolve("out.json");
try (var archive = WriteableJsonStreamArchive.newWriteToFile(csvFile, config)) {
archive.initialize(minMetadataIn);
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty()));
tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.empty()));
}
archive.finish(maxMetadataIn);
archive.finish(MAX_METADATA_IN);
}
assertEqualsDelimitedJson(expectedJson, Files.readString(csvFile));

14
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
Wyświetl plik

@ -76,7 +76,7 @@ class WriteableProtoStreamArchiveTest {
final var tile0 = new TileEncodingResult(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}, OptionalLong.empty());
final var tile1 = new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.of(1));
try (var archive = WriteableProtoStreamArchive.newWriteToFile(csvFile, defaultConfig)) {
archive.initialize(maxMetadataIn);
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(tile0);
tileWriter.write(tile1);
@ -86,7 +86,7 @@ class WriteableProtoStreamArchiveTest {
try (InputStream in = Files.newInputStream(csvFile)) {
assertEquals(
List.of(wrapInit(maxMetadataOut), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)),
List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)),
readAllEntries(in)
);
}
@ -105,7 +105,7 @@ class WriteableProtoStreamArchiveTest {
final var tile3 = new TileEncodingResult(TileCoord.ofXYZ(41, 42, 4), new byte[]{3}, OptionalLong.empty());
final var tile4 = new TileEncodingResult(TileCoord.ofXYZ(51, 52, 5), new byte[]{4}, OptionalLong.empty());
try (var archive = WriteableProtoStreamArchive.newWriteToFile(csvFilePrimary, defaultConfig)) {
archive.initialize(minMetadataIn);
archive.initialize();
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(tile0);
tileWriter.write(tile1);
@ -122,7 +122,7 @@ class WriteableProtoStreamArchiveTest {
try (InputStream in = Files.newInputStream(csvFilePrimary)) {
assertEquals(
List.of(wrapInit(minMetadataOut), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)),
List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)),
readAllEntries(in)
);
}
@ -167,10 +167,8 @@ class WriteableProtoStreamArchiveTest {
.build();
}
private static StreamArchiveProto.Entry wrapInit(StreamArchiveProto.Metadata metadata) {
return StreamArchiveProto.Entry.newBuilder()
.setInitialization(StreamArchiveProto.InitializationEntry.newBuilder().setMetadata(metadata).build())
.build();
private static StreamArchiveProto.Entry wrapInit() {
return StreamArchiveProto.Entry.newBuilder().build();
}
private static StreamArchiveProto.Entry wrapFinish(StreamArchiveProto.Metadata metadata) {

116
planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerAttrStatsTest.java
Wyświetl plik

@ -2,13 +2,8 @@ package com.onthegomap.planetiler.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.VectorTile;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.render.RenderedFeature;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.junit.jupiter.api.Test;
class LayerAttrStatsTest {
@ -17,109 +12,50 @@ class LayerAttrStatsTest {
@Test
void testEmptyLayerStats() {
assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{}), layerStats.getTileStats());
assertEquals(List.of(), layerStats.getTileStats());
}
@Test
void testEmptyLayerStatsOneLayer() {
layerStats.accept(new RenderedFeature(
TileCoord.ofXYZ(1, 2, 3),
new VectorTile.Feature(
"layer1",
1,
VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
Map.of("a", 1, "b", "string", "c", true)
),
1,
Optional.empty()
));
assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{
new LayerAttrStats.VectorLayer("layer1", Map.of(
"a", LayerAttrStats.FieldType.NUMBER,
"b", LayerAttrStats.FieldType.STRING,
"c", LayerAttrStats.FieldType.BOOLEAN
), 3, 3)
}), layerStats.getTileStats());
layerStats.accept("layer1", 3, "a", 1);
layerStats.accept("layer1", 3, "b", "string");
layerStats.accept("layer1", 3, "c", true);
assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of(
"a", LayerAttrStats.FieldType.NUMBER,
"b", LayerAttrStats.FieldType.STRING,
"c", LayerAttrStats.FieldType.BOOLEAN
), 3, 3)), layerStats.getTileStats());
}
@Test
void testEmptyLayerStatsTwoLayers() {
layerStats.accept(new RenderedFeature(
TileCoord.ofXYZ(1, 2, 3),
new VectorTile.Feature(
"layer1",
1,
VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
Map.of()
),
1,
Optional.empty()
));
layerStats.accept(new RenderedFeature(
TileCoord.ofXYZ(1, 2, 4),
new VectorTile.Feature(
"layer2",
1,
VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
Map.of("a", 1, "b", true, "c", true)
),
1,
Optional.empty()
));
layerStats.accept(new RenderedFeature(
TileCoord.ofXYZ(1, 2, 1),
new VectorTile.Feature(
"layer2",
1,
VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
Map.of("a", 1, "b", true, "c", 1)
),
1,
Optional.empty()
));
assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{
new LayerAttrStats.VectorLayer("layer1", Map.of(
), 3, 3),
layerStats.handlerForThread().forZoom(3).forLayer("layer1");
layerStats.accept("layer2", 4, "a", 1);
layerStats.accept("layer2", 4, "b", true);
layerStats.accept("layer2", 4, "c", true);
layerStats.accept("layer2", 1, "a", 1);
layerStats.accept("layer2", 1, "b", true);
layerStats.accept("layer2", 1, "c", 1);
assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of(
), 3, 3),
new LayerAttrStats.VectorLayer("layer2", Map.of(
"a", LayerAttrStats.FieldType.NUMBER,
"b", LayerAttrStats.FieldType.BOOLEAN,
"c", LayerAttrStats.FieldType.STRING
), 1, 4)
}), layerStats.getTileStats());
), 1, 4)), layerStats.getTileStats());
}
@Test
void testMergeFromMultipleThreads() throws InterruptedException {
Thread t1 = new Thread(() -> layerStats.accept(new RenderedFeature(
TileCoord.ofXYZ(1, 2, 3),
new VectorTile.Feature(
"layer1",
1,
VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
Map.of("a", 1)
),
1,
Optional.empty()
)));
layerStats.accept("layer1", 3, "a", true);
Thread t1 = new Thread(() -> layerStats.accept("layer1", 3, "a", 1));
t1.start();
Thread t2 = new Thread(() -> layerStats.accept(new RenderedFeature(
TileCoord.ofXYZ(1, 2, 4),
new VectorTile.Feature(
"layer1",
1,
VectorTile.encodeGeometry(GeoUtils.point(1, 2)),
Map.of("a", true)
),
1,
Optional.empty()
)));
Thread t2 = new Thread(() -> layerStats.accept("layer1", 4, "a", true));
t2.start();
t1.join();
t2.join();
assertEquals(Arrays.asList(new LayerAttrStats.VectorLayer[]{
new LayerAttrStats.VectorLayer("layer1", Map.of(
"a", LayerAttrStats.FieldType.STRING
), 3, 4)
}), layerStats.getTileStats());
assertEquals(List.of(new LayerAttrStats.VectorLayer("layer1", Map.of(
"a", LayerAttrStats.FieldType.STRING
), 3, 4)), layerStats.getTileStats());
}
}