kopia lustrzana https://github.com/onthegomap/planetiler
Porównaj commity
8 Commity
87353886f4
...
af1852217c
Autor | SHA1 | Data |
---|---|---|
Michael Barry | af1852217c | |
dependabot[bot] | 32f941f1b1 | |
dependabot[bot] | ef6fc85691 | |
Michael Barry | f69882addb | |
dependabot[bot] | 6c41c388f4 | |
dependabot[bot] | 689abb5ad7 | |
dependabot[bot] | da9d03c85d | |
dependabot[bot] | 1871e230ac |
|
@ -30,7 +30,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.6.0</version>
|
||||
<version>3.7.1</version>
|
||||
<!-- for with-deps assembly descriptor -->
|
||||
<dependencies>
|
||||
<dependency>
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
</parent>
|
||||
|
||||
<properties>
|
||||
<geotools.version>30.1</geotools.version>
|
||||
<geotools.version>31.0</geotools.version>
|
||||
<log4j.version>2.23.1</log4j.version>
|
||||
<prometheus.version>0.16.0</prometheus.version>
|
||||
<protobuf.version>3.25.3</protobuf.version>
|
||||
|
|
|
@ -266,6 +266,7 @@ public class TileArchiveWriter {
|
|||
boolean lastIsFill = false;
|
||||
List<TileSizeStats.LayerStats> lastLayerStats = null;
|
||||
boolean skipFilled = config.skipFilledTiles();
|
||||
var layerStatsSerializer = TileSizeStats.newThreadLocalSerializer();
|
||||
|
||||
var tileStatsUpdater = tileStats.threadLocalUpdater();
|
||||
var layerAttrStatsUpdater = layerAttrStats.handlerForThread();
|
||||
|
@ -320,7 +321,7 @@ public class TileArchiveWriter {
|
|||
if ((!skipFilled || !lastIsFill) && bytes != null) {
|
||||
tileStatsUpdater.recordTile(tileFeatures.tileCoord(), bytes.length, layerStats);
|
||||
List<String> layerStatsRows = config.outputLayerStats() ?
|
||||
TileSizeStats.formatOutputRows(tileFeatures.tileCoord(), bytes.length, layerStats) :
|
||||
layerStatsSerializer.formatOutputRows(tileFeatures.tileCoord(), bytes.length, layerStats) :
|
||||
List.of();
|
||||
result.add(
|
||||
new TileEncodingResult(
|
||||
|
|
|
@ -38,7 +38,7 @@ import vector_tile.VectorTileProto;
|
|||
* Utilities for extracting tile and layer size summaries from encoded vector tiles.
|
||||
* <p>
|
||||
* {@link #computeTileStats(VectorTileProto.Tile)} extracts statistics about each layer in a tile and
|
||||
* {@link #formatOutputRows(TileCoord, int, List)} formats them as row of a TSV file to write.
|
||||
* {@link TsvSerializer} formats them as row of a TSV file to write.
|
||||
* <p>
|
||||
* To generate a tsv.gz file with stats for each tile, you can add {@code --output-layerstats} option when generating an
|
||||
* archive, or run the following an existing archive:
|
||||
|
@ -52,13 +52,11 @@ import vector_tile.VectorTileProto;
|
|||
public class TileSizeStats {
|
||||
|
||||
private static final int BATCH_SIZE = 1_000;
|
||||
private static final CsvMapper MAPPER = new CsvMapper();
|
||||
private static final CsvSchema SCHEMA = MAPPER
|
||||
private static final CsvSchema SCHEMA = new CsvMapper()
|
||||
.schemaFor(OutputRow.class)
|
||||
.withoutHeader()
|
||||
.withColumnSeparator('\t')
|
||||
.withLineSeparator("\n");
|
||||
private static final ObjectWriter WRITER = MAPPER.writer(SCHEMA);
|
||||
|
||||
/** Returns the default path that a layerstats file should go relative to an existing archive. */
|
||||
public static Path getDefaultLayerstatsPath(Path archive) {
|
||||
|
@ -120,6 +118,7 @@ public class TileSizeStats {
|
|||
List<LayerStats> layerStats = null;
|
||||
|
||||
var updater = tileStats.threadLocalUpdater();
|
||||
var layerStatsSerializer = TileSizeStats.newThreadLocalSerializer();
|
||||
for (var batch : prev) {
|
||||
List<String> lines = new ArrayList<>(batch.tiles.size());
|
||||
for (var tile : batch.tiles) {
|
||||
|
@ -130,7 +129,7 @@ public class TileSizeStats {
|
|||
layerStats = computeTileStats(decoded);
|
||||
}
|
||||
updater.recordTile(tile.coord(), zipped.length, layerStats);
|
||||
lines.addAll(TileSizeStats.formatOutputRows(tile.coord(), zipped.length, layerStats));
|
||||
lines.addAll(layerStatsSerializer.formatOutputRows(tile.coord(), zipped.length, layerStats));
|
||||
}
|
||||
batch.stats.complete(lines);
|
||||
}
|
||||
|
@ -161,13 +160,16 @@ public class TileSizeStats {
|
|||
stats.printSummary();
|
||||
}
|
||||
|
||||
/** Returns the TSV rows to output for all the layers in a tile. */
|
||||
public static List<String> formatOutputRows(TileCoord tileCoord, int archivedBytes, List<LayerStats> layerStats)
|
||||
throws IOException {
|
||||
/** Returns a {@link TsvSerializer} that can be used by a single thread to convert to CSV rows. */
|
||||
public static TsvSerializer newThreadLocalSerializer() {
|
||||
// CsvMapper is not entirely thread safe, and can end up with a BufferRecycler memory leak when writeValueAsString
|
||||
// is called billions of times from multiple threads, so we generate a new instance per serializing thread
|
||||
ObjectWriter writer = new CsvMapper().writer(SCHEMA);
|
||||
return (tileCoord, archivedBytes, layerStats) -> {
|
||||
int hilbert = tileCoord.hilbertEncoded();
|
||||
List<String> result = new ArrayList<>(layerStats.size());
|
||||
for (var layer : layerStats) {
|
||||
result.add(lineToString(new OutputRow(
|
||||
result.add(writer.writeValueAsString(new OutputRow(
|
||||
tileCoord.z(),
|
||||
tileCoord.x(),
|
||||
tileCoord.y(),
|
||||
|
@ -183,6 +185,7 @@ public class TileSizeStats {
|
|||
)));
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -195,11 +198,6 @@ public class TileSizeStats {
|
|||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE))));
|
||||
}
|
||||
|
||||
/** Returns {@code output} encoded as a TSV row string. */
|
||||
public static String lineToString(OutputRow output) throws IOException {
|
||||
return WRITER.writeValueAsString(output);
|
||||
}
|
||||
|
||||
/** Returns the header row for the output TSV file. */
|
||||
public static String headerRow() {
|
||||
return String.join(
|
||||
|
@ -240,6 +238,14 @@ public class TileSizeStats {
|
|||
return result;
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface TsvSerializer {
|
||||
|
||||
/** Returns the TSV rows to output for all the layers in a tile. */
|
||||
List<String> formatOutputRows(TileCoord tileCoord, int archivedBytes, List<LayerStats> layerStats)
|
||||
throws IOException;
|
||||
}
|
||||
|
||||
/** Model for the data contained in each row in the TSV. */
|
||||
@JsonPropertyOrder({
|
||||
"z",
|
||||
|
|
|
@ -37,7 +37,7 @@ class TileSizeStatsTest {
|
|||
assertEquals(2, entry1.layerAttrKeys());
|
||||
assertEquals(2, entry1.layerAttrValues());
|
||||
|
||||
var formatted = TileSizeStats.formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
var formatted = TileSizeStats.newThreadLocalSerializer().formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
assertEquals(
|
||||
"""
|
||||
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_geometries layer_attr_bytes layer_attr_keys layer_attr_values
|
||||
|
@ -86,7 +86,7 @@ class TileSizeStatsTest {
|
|||
assertEquals("b", entry2.layer());
|
||||
assertEquals(1, entry2.layerFeatures());
|
||||
|
||||
var formatted = TileSizeStats.formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
var formatted = TileSizeStats.newThreadLocalSerializer().formatOutputRows(TileCoord.ofXYZ(1, 2, 3), 999, stats);
|
||||
assertEquals(
|
||||
"""
|
||||
z x y hilbert archived_tile_bytes layer layer_bytes layer_features layer_geometries layer_attr_bytes layer_attr_keys layer_attr_values
|
||||
|
|
|
@ -89,7 +89,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.6.0</version>
|
||||
<version>3.7.1</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.onthegomap.planetiler</groupId>
|
||||
|
|
8
pom.xml
8
pom.xml
|
@ -284,7 +284,7 @@
|
|||
<plugin>
|
||||
<groupId>com.google.cloud.tools</groupId>
|
||||
<artifactId>jib-maven-plugin</artifactId>
|
||||
<version>3.4.1</version>
|
||||
<version>3.4.2</version>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
|
@ -328,12 +328,12 @@
|
|||
<plugin>
|
||||
<groupId>org.sonarsource.scanner.maven</groupId>
|
||||
<artifactId>sonar-maven-plugin</artifactId>
|
||||
<version>3.10.0.2594</version>
|
||||
<version>3.11.0.3922</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.11</version>
|
||||
<version>0.8.12</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
|
@ -394,7 +394,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-gpg-plugin</artifactId>
|
||||
<version>3.2.1</version>
|
||||
<version>3.2.2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>sign-artifacts</id>
|
||||
|
|
Ładowanie…
Reference in New Issue