kopia lustrzana https://github.com/onthegomap/planetiler
Porównaj commity
9 Commity
ebd4343b7e
...
0e6261f8a8
Autor | SHA1 | Data |
---|---|---|
Michael Barry | 0e6261f8a8 | |
dependabot[bot] | ed373ff3d3 | |
dependabot[bot] | 32f941f1b1 | |
dependabot[bot] | ef6fc85691 | |
Michael Barry | f69882addb | |
dependabot[bot] | 6c41c388f4 | |
dependabot[bot] | 689abb5ad7 | |
Mike Barry | 7c86699a04 | |
Mike Barry | b38b8b37a1 |
|
@ -30,7 +30,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.6.0</version>
|
||||
<version>3.7.1</version>
|
||||
<!-- for with-deps assembly descriptor -->
|
||||
<dependencies>
|
||||
<dependency>
|
||||
|
|
|
@ -20,7 +20,7 @@
|
|||
<log4j.version>2.23.1</log4j.version>
|
||||
<prometheus.version>0.16.0</prometheus.version>
|
||||
<protobuf.version>3.25.3</protobuf.version>
|
||||
<geopackage.version>6.6.4</geopackage.version>
|
||||
<geopackage.version>6.6.5</geopackage.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
|
|
@ -6,6 +6,8 @@ import com.onthegomap.planetiler.geo.GeoUtils;
|
|||
import com.onthegomap.planetiler.geo.GeometryException;
|
||||
import com.onthegomap.planetiler.geo.GeometryType;
|
||||
import com.onthegomap.planetiler.reader.SourceFeature;
|
||||
import com.onthegomap.planetiler.reader.osm.OsmElement;
|
||||
import com.onthegomap.planetiler.reader.osm.OsmSourceFeature;
|
||||
import com.onthegomap.planetiler.render.FeatureRenderer;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.util.CacheByZoom;
|
||||
|
@ -59,7 +61,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
* @return a feature that can be configured further.
|
||||
*/
|
||||
public Feature geometry(String layer, Geometry geometry) {
|
||||
Feature feature = new Feature(layer, geometry, source.id());
|
||||
Feature feature = new Feature(layer, geometry, source);
|
||||
output.add(feature);
|
||||
return feature;
|
||||
}
|
||||
|
@ -81,7 +83,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.worldGeometry());
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_point", "Error getting point geometry for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -101,7 +103,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.line());
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_line", "Error constructing line for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -121,7 +123,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.polygon());
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_polygon", "Error constructing polygon for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -136,7 +138,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.centroid());
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_centroid", "Error getting centroid for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -153,7 +155,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.centroidIfConvex());
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_centroid_if_convex", "Error constructing centroid if convex for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -169,7 +171,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.pointOnSurface());
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_point_on_surface", "Error constructing point on surface for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -191,7 +193,7 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
return geometry(layer, source.innermostPoint(tolerance));
|
||||
} catch (GeometryException e) {
|
||||
e.log(stats, "feature_innermost_point", "Error constructing innermost point for " + source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source.id());
|
||||
return new Feature(layer, EMPTY_GEOM, source);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -275,11 +277,21 @@ public class FeatureCollector implements Iterable<FeatureCollector.Feature> {
|
|||
|
||||
private String numPointsAttr = null;
|
||||
|
||||
private Feature(String layer, Geometry geom, long id) {
|
||||
private Feature(String layer, Geometry geom, SourceFeature source) {
|
||||
this.layer = layer;
|
||||
this.geom = geom;
|
||||
this.geometryType = GeometryType.typeOf(geom);
|
||||
this.id = id;
|
||||
if (source instanceof OsmSourceFeature osmSourceFeature) {
|
||||
long osmId = osmSourceFeature.originalElement().id();
|
||||
this.id = switch (osmSourceFeature.originalElement()) {
|
||||
case OsmElement.Node node -> node.id() * 10 + 1;
|
||||
case OsmElement.Way way -> way.id() * 10 + 2;
|
||||
case OsmElement.Relation relation -> relation.id() * 10 + 3;
|
||||
default -> osmId * 10;
|
||||
};
|
||||
} else {
|
||||
this.id = source.id();
|
||||
}
|
||||
if (geometryType == GeometryType.POINT) {
|
||||
minPixelSizeAtMaxZoom = 0;
|
||||
defaultMinPixelSize = 0;
|
||||
|
|
|
@ -266,6 +266,7 @@ public class TileArchiveWriter {
|
|||
boolean lastIsFill = false;
|
||||
List<TileSizeStats.LayerStats> lastLayerStats = null;
|
||||
boolean skipFilled = config.skipFilledTiles();
|
||||
var layerStatsSerializer = TileSizeStats.newThreadLocalSerializer();
|
||||
|
||||
var tileStatsUpdater = tileStats.threadLocalUpdater();
|
||||
var layerAttrStatsUpdater = layerAttrStats.handlerForThread();
|
||||
|
@ -320,7 +321,7 @@ public class TileArchiveWriter {
|
|||
if ((!skipFilled || !lastIsFill) && bytes != null) {
|
||||
tileStatsUpdater.recordTile(tileFeatures.tileCoord(), bytes.length, layerStats);
|
||||
List<String> layerStatsRows = config.outputLayerStats() ?
|
||||
TileSizeStats.formatOutputRows(tileFeatures.tileCoord(), bytes.length, layerStats) :
|
||||
layerStatsSerializer.formatOutputRows(tileFeatures.tileCoord(), bytes.length, layerStats) :
|
||||
List.of();
|
||||
result.add(
|
||||
new TileEncodingResult(
|
||||
|
|
|
@ -38,7 +38,7 @@ import vector_tile.VectorTileProto;
|
|||
* Utilities for extracting tile and layer size summaries from encoded vector tiles.
|
||||
* <p>
|
||||
* {@link #computeTileStats(VectorTileProto.Tile)} extracts statistics about each layer in a tile and
|
||||
* {@link #formatOutputRows(TileCoord, int, List)} formats them as row of a TSV file to write.
|
||||
* {@link TsvSerializer} formats them as row of a TSV file to write.
|
||||
* <p>
|
||||
* To generate a tsv.gz file with stats for each tile, you can add {@code --output-layerstats} option when generating an
|
||||
* archive, or run the following an existing archive:
|
||||
|
@ -52,13 +52,11 @@ import vector_tile.VectorTileProto;
|
|||
public class TileSizeStats {
|
||||
|
||||
private static final int BATCH_SIZE = 1_000;
|
||||
private static final CsvMapper MAPPER = new CsvMapper();
|
||||
private static final CsvSchema SCHEMA = MAPPER
|
||||
private static final CsvSchema SCHEMA = new CsvMapper()
|
||||
.schemaFor(OutputRow.class)
|
||||
.withoutHeader()
|
||||
.withColumnSeparator('\t')
|
||||
.withLineSeparator("\n");
|
||||
private static final ObjectWriter WRITER = MAPPER.writer(SCHEMA);
|
||||
|
||||
/** Returns the default path that a layerstats file should go relative to an existing archive. */
|
||||
public static Path getDefaultLayerstatsPath(Path archive) {
|
||||
|
@ -120,6 +118,7 @@ public class TileSizeStats {
|
|||
List<LayerStats> layerStats = null;
|
||||
|
||||
var updater = tileStats.threadLocalUpdater();
|
||||
var layerStatsSerializer = TileSizeStats.newThreadLocalSerializer();
|
||||
for (var batch : prev) {
|
||||
List<String> lines = new ArrayList<>(batch.tiles.size());
|
||||
for (var tile : batch.tiles) {
|
||||
|
@ -130,7 +129,7 @@ public class TileSizeStats {
|
|||
layerStats = computeTileStats(decoded);
|
||||
}
|
||||
updater.recordTile(tile.coord(), zipped.length, layerStats);
|
||||
lines.addAll(TileSizeStats.formatOutputRows(tile.coord(), zipped.length, layerStats));
|
||||
lines.addAll(layerStatsSerializer.formatOutputRows(tile.coord(), zipped.length, layerStats));
|
||||
}
|
||||
batch.stats.complete(lines);
|
||||
}
|
||||
|
@ -161,28 +160,32 @@ public class TileSizeStats {
|
|||
stats.printSummary();
|
||||
}
|
||||
|
||||
/** Returns the TSV rows to output for all the layers in a tile. */
|
||||
public static List<String> formatOutputRows(TileCoord tileCoord, int archivedBytes, List<LayerStats> layerStats)
|
||||
throws IOException {
|
||||
int hilbert = tileCoord.hilbertEncoded();
|
||||
List<String> result = new ArrayList<>(layerStats.size());
|
||||
for (var layer : layerStats) {
|
||||
result.add(lineToString(new OutputRow(
|
||||
tileCoord.z(),
|
||||
tileCoord.x(),
|
||||
tileCoord.y(),
|
||||
hilbert,
|
||||
archivedBytes,
|
||||
layer.layer,
|
||||
layer.layerBytes,
|
||||
layer.layerFeatures,
|
||||
layer.layerGeometries,
|
||||
layer.layerAttrBytes,
|
||||
layer.layerAttrKeys,
|
||||
layer.layerAttrValues
|
||||
)));
|
||||
}
|
||||
return result;
|
||||
/** Returns a {@link TsvSerializer} that can be used by a single thread to convert to CSV rows. */
|
||||
public static TsvSerializer newThreadLocalSerializer() {
|
||||
// CsvMapper is not entirely thread safe, and can end up with a BufferRecycler memory leak when writeValueAsString
|
||||
// is called billions of times from multiple threads, so we generate a new instance per serializing thread
|
||||
ObjectWriter writer = new CsvMapper().writer(SCHEMA);
|
||||
return (tileCoord, archivedBytes, layerStats) -> {
|
||||
int hilbert = tileCoord.hilbertEncoded();
|
||||
List<String> result = new ArrayList<>(layerStats.size());
|
||||
for (var layer : layerStats) {
|
||||
result.add(writer.writeValueAsString(new OutputRow(
|
||||
tileCoord.z(),
|
||||
tileCoord.x(),
|
||||
tileCoord.y(),
|
||||
hilbert,
|
||||
archivedBytes,
|
||||
layer.layer,
|
||||
layer.layerBytes,
|
||||
layer.layerFeatures,
|
||||
layer.layerGeometries,
|
||||
layer.layerAttrBytes,
|
||||
layer.layerAttrKeys,
|
||||
layer.layerAttrValues
|
||||
)));
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -195,11 +198,6 @@ public class TileSizeStats {
|
|||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE))));
|
||||
}
|
||||
|
||||
/** Returns {@code output} encoded as a TSV row string. */
|
||||
public static String lineToString(OutputRow output) throws IOException {
|
||||
return WRITER.writeValueAsString(output);
|
||||
}
|
||||
|
||||
/** Returns the header row for the output TSV file. */
|
||||
public static String headerRow() {
|
||||
return String.join(
|
||||
|
@ -240,6 +238,14 @@ public class TileSizeStats {
|
|||
return result;
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface TsvSerializer {
|
||||
|
||||
/** Returns the TSV rows to output for all the layers in a tile. */
|
||||
List<String> formatOutputRows(TileCoord tileCoord, int archivedBytes, List<LayerStats> layerStats)
|
||||
throws IOException;
|
||||
}
|
||||
|
||||
/** Model for the data contained in each row in the TSV. */
|
||||
@JsonPropertyOrder({
|
||||
"z",
|
||||
|
|
|
@ -875,7 +875,7 @@ class PlanetilerTests {
|
|||
feature(newPoint(128, 128), Map.of(
|
||||
"attr", "value",
|
||||
"name", "name value"
|
||||
))
|
||||
)).withId(11)
|
||||
)
|
||||
), results.tiles);
|
||||
}
|
||||
|
@ -964,7 +964,7 @@ class PlanetilerTests {
|
|||
feature(newLineString(128, 128, 192, 192), Map.of(
|
||||
"attr", "value",
|
||||
"name", "name value"
|
||||
))
|
||||
)).withId(32)
|
||||
)
|
||||
), results.tiles);
|
||||
}
|
||||
|
@ -1089,7 +1089,7 @@ class PlanetilerTests {
|
|||
"attr", "value",
|
||||
"name", "name value",
|
||||
"relname", "rel name"
|
||||
))
|
||||
)).withId(173)
|
||||
)
|
||||
), results.tiles);
|
||||
}
|
||||
|
|
|
@ -279,7 +279,9 @@ public class TestUtils {
|
|||
case UNKNOWN -> throw new IllegalArgumentException("cannot decompress \"UNKNOWN\"");
|
||||
};
|
||||
var decoded = VectorTile.decode(bytes).stream()
|
||||
.map(feature -> feature(decodeSilently(feature.geometry()), feature.layer(), feature.attrs())).toList();
|
||||
.map(
|
||||
feature -> feature(decodeSilently(feature.geometry()), feature.layer(), feature.attrs()).withId(feature.id()))
|
||||
.toList();
|
||||
tiles.put(tile.coord(), decoded);
|
||||
}
|
||||
return tiles;
|
||||
|
@ -467,12 +469,21 @@ public class TestUtils {
|
|||
public record ComparableFeature(
|
||||
GeometryComparision geometry,
|
||||
String layer,
|
||||
Map<String, Object> attrs
|
||||
Map<String, Object> attrs,
|
||||
Long id
|
||||
) {
|
||||
ComparableFeature(
|
||||
GeometryComparision geometry,
|
||||
String layer,
|
||||
Map<String, Object> attrs
|
||||
) {
|
||||
this(geometry, layer, attrs, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
return o == this || (o instanceof ComparableFeature other &&
|
||||
(id == null || other.id == null || id.equals(other.id)) &&
|
||||
geometry.equals(other.geometry) &&
|
||||
attrs.equals(other.attrs) &&
|
||||
(layer == null || other.layer == null || Objects.equals(layer, other.layer)));
|
||||
|
@ -484,6 +495,10 @@ public class TestUtils {
|
|||
result = 31 * result + attrs.hashCode();
|
||||
return result;
|
||||
}
|
||||
|
||||
ComparableFeature withId(long id) {
|
||||
return new ComparableFeature(geometry, layer, attrs, id);
|
||||
}
|
||||
}
|
||||
|
||||
public static ComparableFeature feature(Geometry geom, String layer, Map<String, Object> attrs) {
|
||||
|
|
|
@ -37,7 +37,7 @@ class TileSizeStatsTest {
|
|||
assertEquals(2, entry1.layerAttrKeys());
|
||||
assertEquals(2, entry1.layerAttrValues());
|
||||
|
||||
var formatted = TileSizeStats.formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
var formatted = TileSizeStats.newThreadLocalSerializer().formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
assertEquals(
|
||||
"""
|
||||
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_geometries layer_attr_bytes layer_attr_keys layer_attr_values
|
||||
|
@ -86,7 +86,7 @@ class TileSizeStatsTest {
|
|||
assertEquals("b", entry2.layer());
|
||||
assertEquals(1, entry2.layerFeatures());
|
||||
|
||||
var formatted = TileSizeStats.formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
var formatted = TileSizeStats.newThreadLocalSerializer().formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
assertEquals(
|
||||
"""
|
||||
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_geometries layer_attr_bytes layer_attr_keys layer_attr_values
|
||||
|
|
|
@ -89,7 +89,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.6.0</version>
|
||||
<version>3.7.1</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.onthegomap.planetiler</groupId>
|
||||
|
|
6
pom.xml
6
pom.xml
|
@ -284,7 +284,7 @@
|
|||
<plugin>
|
||||
<groupId>com.google.cloud.tools</groupId>
|
||||
<artifactId>jib-maven-plugin</artifactId>
|
||||
<version>3.4.1</version>
|
||||
<version>3.4.2</version>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
|
@ -333,7 +333,7 @@
|
|||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.11</version>
|
||||
<version>0.8.12</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
|
@ -394,7 +394,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-gpg-plugin</artifactId>
|
||||
<version>3.2.1</version>
|
||||
<version>3.2.2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>sign-artifacts</id>
|
||||
|
|
Ładowanie…
Reference in New Issue