mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

refactor VectorLayers from mbtiles into util.LayerStats [#98] (#456)

convex-tweaks
Brandon Liu 2023-01-26 10:15:43 +08:00 zatwierdzone przez GitHub
rodzic a0f8c67c78
commit f772180fb7
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
4 zmienionych plików z 97 dodań i 96 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

58
planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
Wyświetl plik

@ -27,13 +27,10 @@ import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.TreeMap;
import java.util.stream.Collectors;
@ -171,7 +168,7 @@ public final class Mbtiles implements TileArchive {
.setBoundsAndCenter(config.bounds().latLon())
.setMinzoom(config.minzoom())
.setMaxzoom(config.maxzoom())
.setJson(layerStats.getTileStats());
.setJson(new MetadataJson(layerStats.getTileStats()));
for (var entry : tileArchiveMetadata.planetilerSpecific().entrySet()) {
metadata.setMetadata(entry.getKey(), entry.getValue());
@ -400,10 +397,10 @@ public final class Mbtiles implements TileArchive {
* schema</a>
*/
public record MetadataJson(
@JsonProperty("vector_layers") List<VectorLayer> vectorLayers
@JsonProperty("vector_layers") List<LayerStats.VectorLayer> vectorLayers
) {
public MetadataJson(VectorLayer... layers) {
public MetadataJson(LayerStats.VectorLayer... layers) {
this(List.of(layers));
}
@ -422,55 +419,6 @@ public final class Mbtiles implements TileArchive {
throw new IllegalArgumentException("Unable to encode as string: " + this, e);
}
}
public enum FieldType {
@JsonProperty("Number")
NUMBER,
@JsonProperty("Boolean")
BOOLEAN,
@JsonProperty("String")
STRING;
/**
* Per the spec: attributes whose type varies between features SHOULD be listed as "String"
*/
public static FieldType merge(FieldType oldValue, FieldType newValue) {
return oldValue != newValue ? STRING : newValue;
}
}
public record VectorLayer(
@JsonProperty("id") String id,
@JsonProperty("fields") Map<String, FieldType> fields,
@JsonProperty("description") Optional<String> description,
@JsonProperty("minzoom") OptionalInt minzoom,
@JsonProperty("maxzoom") OptionalInt maxzoom
) {
public VectorLayer(String id, Map<String, FieldType> fields) {
this(id, fields, Optional.empty(), OptionalInt.empty(), OptionalInt.empty());
}
public VectorLayer(String id, Map<String, FieldType> fields, int minzoom, int maxzoom) {
this(id, fields, Optional.empty(), OptionalInt.of(minzoom), OptionalInt.of(maxzoom));
}
public static VectorLayer forLayer(String id) {
return new VectorLayer(id, new HashMap<>());
}
public VectorLayer withDescription(String newDescription) {
return new VectorLayer(id, fields, Optional.of(newDescription), minzoom, maxzoom);
}
public VectorLayer withMinzoom(int newMinzoom) {
return new VectorLayer(id, fields, description, OptionalInt.of(newMinzoom), maxzoom);
}
public VectorLayer withMaxzoom(int newMaxzoom) {
return new VectorLayer(id, fields, description, minzoom, OptionalInt.of(newMaxzoom));
}
}
}
/** Contents of a row of the tiles table, or in case of compact mode in the tiles view. */

86
planetiler-core/src/main/java/com/onthegomap/planetiler/util/LayerStats.java
Wyświetl plik

@ -1,10 +1,14 @@
package com.onthegomap.planetiler.util;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.onthegomap.planetiler.mbtiles.Mbtiles;
import com.onthegomap.planetiler.render.RenderedFeature;
import com.onthegomap.planetiler.writer.TileArchive;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.TreeMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
@ -12,11 +16,12 @@ import javax.annotation.concurrent.NotThreadSafe;
import javax.annotation.concurrent.ThreadSafe;
/**
* Tracks the type and zoom range of vector tile attributes that have been emitted by layer to populate the {@code json}
* attribute in the mbtiles output metadata.
* Tracks the feature attributes and zoom range of each layer to populate the archive output metadata.
* <p>
* To minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
* Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link TileArchive} formats. To
* minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
* thread-local handler that can update stats without contention.
* </p>
*
* @see Mbtiles.MetadataJson
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md#content">MBtiles spec</a>
@ -36,8 +41,8 @@ public class LayerStats implements Consumer<RenderedFeature> {
private final ThreadLocal<ThreadLocalHandler> layerStats = ThreadLocal
.withInitial(ThreadLocalHandler::new);
/** Returns stats on all features that have been emitted for the {@code json} mbtiles metadata value. */
public Mbtiles.MetadataJson getTileStats() {
/** Returns stats on all features that have been emitted as a list of {@link VectorLayer} objects. */
public List<VectorLayer> getTileStats() {
Map<String, StatsForLayer> layers = new TreeMap<>();
for (var threadLocal : threadLocals) {
for (StatsForLayer stats : threadLocal.layers.values()) {
@ -47,17 +52,64 @@ public class LayerStats implements Consumer<RenderedFeature> {
for (var entry : next.fields.entrySet()) {
// keep track of field type as a number/boolean/string but widen to string if multiple different
// types are encountered
prev.fields.merge(entry.getKey(), entry.getValue(), Mbtiles.MetadataJson.FieldType::merge);
prev.fields.merge(entry.getKey(), entry.getValue(), FieldType::merge);
}
return prev;
});
}
}
return new Mbtiles.MetadataJson(
layers.values().stream()
.map(stats -> new Mbtiles.MetadataJson.VectorLayer(stats.layer, stats.fields, stats.minzoom, stats.maxzoom))
.toList()
);
return layers.values().stream()
.map(stats -> new VectorLayer(stats.layer, stats.fields, stats.minzoom, stats.maxzoom))
.toList();
}
public enum FieldType {
@JsonProperty("Number")
NUMBER,
@JsonProperty("Boolean")
BOOLEAN,
@JsonProperty("String")
STRING;
/**
* Per the MBTiles spec: attributes whose type varies between features SHOULD be listed as "String"
*/
public static FieldType merge(FieldType oldValue, FieldType newValue) {
return oldValue != newValue ? STRING : newValue;
}
}
public record VectorLayer(
@JsonProperty("id") String id,
@JsonProperty("fields") Map<String, FieldType> fields,
@JsonProperty("description") Optional<String> description,
@JsonProperty("minzoom") OptionalInt minzoom,
@JsonProperty("maxzoom") OptionalInt maxzoom
) {
public VectorLayer(String id, Map<String, FieldType> fields) {
this(id, fields, Optional.empty(), OptionalInt.empty(), OptionalInt.empty());
}
public VectorLayer(String id, Map<String, FieldType> fields, int minzoom, int maxzoom) {
this(id, fields, Optional.empty(), OptionalInt.of(minzoom), OptionalInt.of(maxzoom));
}
public static VectorLayer forLayer(String id) {
return new VectorLayer(id, new HashMap<>());
}
public VectorLayer withDescription(String newDescription) {
return new VectorLayer(id, fields, Optional.of(newDescription), minzoom, maxzoom);
}
public VectorLayer withMinzoom(int newMinzoom) {
return new VectorLayer(id, fields, description, OptionalInt.of(newMinzoom), maxzoom);
}
public VectorLayer withMaxzoom(int newMaxzoom) {
return new VectorLayer(id, fields, description, minzoom, OptionalInt.of(newMaxzoom));
}
}
/** Accepts features from a single thread that will be combined across all threads in {@link #getTileStats()}. */
@ -79,17 +131,17 @@ public class LayerStats implements Consumer<RenderedFeature> {
String key = entry.getKey();
Object value = entry.getValue();
Mbtiles.MetadataJson.FieldType fieldType = null;
FieldType fieldType = null;
if (value instanceof Number) {
fieldType = Mbtiles.MetadataJson.FieldType.NUMBER;
fieldType = FieldType.NUMBER;
} else if (value instanceof Boolean) {
fieldType = Mbtiles.MetadataJson.FieldType.BOOLEAN;
fieldType = FieldType.BOOLEAN;
} else if (value != null) {
fieldType = Mbtiles.MetadataJson.FieldType.STRING;
fieldType = FieldType.STRING;
}
if (fieldType != null) {
// widen different types to string
stats.fields.merge(key, fieldType, Mbtiles.MetadataJson.FieldType::merge);
stats.fields.merge(key, fieldType, FieldType::merge);
}
}
}
@ -112,7 +164,7 @@ public class LayerStats implements Consumer<RenderedFeature> {
private static class StatsForLayer {
private final String layer;
private final Map<String, Mbtiles.MetadataJson.FieldType> fields = new HashMap<>();
private final Map<String, FieldType> fields = new HashMap<>();
private int minzoom = Integer.MAX_VALUE;
private int maxzoom = Integer.MIN_VALUE;

11
planetiler-core/src/test/java/com/onthegomap/planetiler/mbtiles/MbtilesTest.java
Wyświetl plik

@ -7,6 +7,7 @@ import com.google.common.math.IntMath;
import com.onthegomap.planetiler.TestUtils;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.util.LayerStats;
import com.onthegomap.planetiler.writer.TileEncodingResult;
import java.io.IOException;
import java.math.RoundingMode;
@ -204,17 +205,17 @@ class MbtilesTest {
@Test
void testFullMetadataJson() throws IOException {
testMetadataJson(new Mbtiles.MetadataJson(
new Mbtiles.MetadataJson.VectorLayer(
new LayerStats.VectorLayer(
"full",
Map.of(
"NUMBER_FIELD", Mbtiles.MetadataJson.FieldType.NUMBER,
"STRING_FIELD", Mbtiles.MetadataJson.FieldType.STRING,
"boolean field", Mbtiles.MetadataJson.FieldType.BOOLEAN
"NUMBER_FIELD", LayerStats.FieldType.NUMBER,
"STRING_FIELD", LayerStats.FieldType.STRING,
"boolean field", LayerStats.FieldType.BOOLEAN
)
).withDescription("full description")
.withMinzoom(0)
.withMaxzoom(5),
new Mbtiles.MetadataJson.VectorLayer(
new LayerStats.VectorLayer(
"partial",
Map.of()
)

38
planetiler-core/src/test/java/com/onthegomap/planetiler/util/LayerStatsTest.java
Wyświetl plik

@ -5,8 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.VectorTile;
import com.onthegomap.planetiler.geo.GeoUtils;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.mbtiles.Mbtiles;
import com.onthegomap.planetiler.render.RenderedFeature;
import java.util.Arrays;
import java.util.Map;
import java.util.Optional;
import org.junit.jupiter.api.Test;
@ -17,7 +17,7 @@ class LayerStatsTest {
@Test
void testEmptyLayerStats() {
assertEquals(new Mbtiles.MetadataJson(), layerStats.getTileStats());
assertEquals(Arrays.asList(new LayerStats.VectorLayer[]{}), layerStats.getTileStats());
}
@Test
@ -33,13 +33,13 @@ class LayerStatsTest {
1,
Optional.empty()
));
assertEquals(new Mbtiles.MetadataJson(
new Mbtiles.MetadataJson.VectorLayer("layer1", Map.of(
"a", Mbtiles.MetadataJson.FieldType.NUMBER,
"b", Mbtiles.MetadataJson.FieldType.STRING,
"c", Mbtiles.MetadataJson.FieldType.BOOLEAN
assertEquals(Arrays.asList(new LayerStats.VectorLayer[]{
new LayerStats.VectorLayer("layer1", Map.of(
"a", LayerStats.FieldType.NUMBER,
"b", LayerStats.FieldType.STRING,
"c", LayerStats.FieldType.BOOLEAN
), 3, 3)
), layerStats.getTileStats());
}), layerStats.getTileStats());
}
@Test
@ -77,15 +77,15 @@ class LayerStatsTest {
1,
Optional.empty()
));
assertEquals(new Mbtiles.MetadataJson(
new Mbtiles.MetadataJson.VectorLayer("layer1", Map.of(
assertEquals(Arrays.asList(new LayerStats.VectorLayer[]{
new LayerStats.VectorLayer("layer1", Map.of(
), 3, 3),
new Mbtiles.MetadataJson.VectorLayer("layer2", Map.of(
"a", Mbtiles.MetadataJson.FieldType.NUMBER,
"b", Mbtiles.MetadataJson.FieldType.BOOLEAN,
"c", Mbtiles.MetadataJson.FieldType.STRING
new LayerStats.VectorLayer("layer2", Map.of(
"a", LayerStats.FieldType.NUMBER,
"b", LayerStats.FieldType.BOOLEAN,
"c", LayerStats.FieldType.STRING
), 1, 4)
), layerStats.getTileStats());
}), layerStats.getTileStats());
}
@Test
@ -116,10 +116,10 @@ class LayerStatsTest {
t2.start();
t1.join();
t2.join();
assertEquals(new Mbtiles.MetadataJson(
new Mbtiles.MetadataJson.VectorLayer("layer1", Map.of(
"a", Mbtiles.MetadataJson.FieldType.STRING
assertEquals(Arrays.asList(new LayerStats.VectorLayer[]{
new LayerStats.VectorLayer("layer1", Map.of(
"a", LayerStats.FieldType.STRING
), 3, 4)
), layerStats.getTileStats());
}), layerStats.getTileStats());
}
}