kopia lustrzana https://github.com/onthegomap/planetiler
tilestats
Mike Barry
rodzic
d7eb01faeb
commit
3391abfef4
|
|
@ -173,7 +173,8 @@ public record TileArchiveConfig(
|
|||
public enum Format {
|
||||
MBTILES("mbtiles"),
|
||||
PMTILES("pmtiles"),
|
||||
POSTGRES("postgres");
|
||||
POSTGRES("postgres"),
|
||||
STATS("stats");
|
||||
|
||||
private final String id;
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import com.onthegomap.planetiler.worker.WorkQueue;
|
|||
import com.onthegomap.planetiler.worker.Worker;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
|
|
@ -229,6 +230,7 @@ public class TileArchiveWriter {
|
|||
Long lastTileDataHash = null;
|
||||
boolean lastIsFill = false;
|
||||
boolean skipFilled = config.skipFilledTiles();
|
||||
long time = -1;
|
||||
|
||||
for (TileBatch batch : prev) {
|
||||
Queue<TileEncodingResult> result = new ArrayDeque<>(batch.size());
|
||||
|
|
@ -239,12 +241,15 @@ public class TileArchiveWriter {
|
|||
featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed());
|
||||
byte[] bytes, encoded;
|
||||
Long tileDataHash;
|
||||
boolean cached = false;
|
||||
if (tileFeatures.hasSameContents(last)) {
|
||||
bytes = lastBytes;
|
||||
encoded = lastEncoded;
|
||||
tileDataHash = lastTileDataHash;
|
||||
memoizedTiles.inc();
|
||||
cached = true;
|
||||
} else {
|
||||
long start = System.nanoTime();
|
||||
VectorTile en = tileFeatures.getVectorTileEncoder();
|
||||
if (skipFilled && (lastIsFill = en.containsOnlyFills())) {
|
||||
encoded = null;
|
||||
|
|
@ -267,6 +272,7 @@ public class TileArchiveWriter {
|
|||
tileDataHash = null;
|
||||
}
|
||||
lastTileDataHash = tileDataHash;
|
||||
time = System.nanoTime() - start;
|
||||
}
|
||||
if (skipFilled && lastIsFill) {
|
||||
continue;
|
||||
|
|
@ -276,8 +282,15 @@ public class TileArchiveWriter {
|
|||
totalTileSizesByZoom[zoom].incBy(encodedLength);
|
||||
maxTileSizesByZoom[zoom].accumulate(encodedLength);
|
||||
result.add(
|
||||
new TileEncodingResult(tileFeatures.tileCoord(), bytes,
|
||||
tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash))
|
||||
new TileEncodingResult(
|
||||
tileFeatures.tileCoord(),
|
||||
bytes,
|
||||
tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash),
|
||||
tileFeatures.getInputFeatureBytes(),
|
||||
tileFeatures.getNumFeaturesProcessed(),
|
||||
cached,
|
||||
Duration.ofNanos(time).toMillis()
|
||||
)
|
||||
);
|
||||
}
|
||||
// hand result off to writer
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
|||
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
|
||||
import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
|
||||
import com.onthegomap.planetiler.util.Pgtiles;
|
||||
import com.onthegomap.planetiler.util.TileStatsArchive;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */
|
||||
|
|
@ -47,6 +49,7 @@ public class TileArchives {
|
|||
.subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION)));
|
||||
case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath());
|
||||
case POSTGRES -> Pgtiles.writer(archive.uri(), options);
|
||||
case STATS -> new TileStatsArchive(archive.getLocalPath());
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -62,6 +65,7 @@ public class TileArchives {
|
|||
case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options);
|
||||
case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath());
|
||||
case POSTGRES -> Pgtiles.reader(archive.uri(), options);
|
||||
case STATS -> throw new UnsupportedEncodingException();
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,20 @@ public record TileEncodingResult(
|
|||
TileCoord coord,
|
||||
byte[] tileData,
|
||||
/** will always be empty in non-compact mode and might also be empty in compact mode */
|
||||
OptionalLong tileDataHash
|
||||
OptionalLong tileDataHash,
|
||||
long inputBytes,
|
||||
long inputFeatures,
|
||||
boolean cached,
|
||||
long time
|
||||
) {
|
||||
public TileEncodingResult(
|
||||
TileCoord coord,
|
||||
byte[] tileData,
|
||||
/** will always be empty in non-compact mode and might also be empty in compact mode */
|
||||
OptionalLong tileDataHash
|
||||
) {
|
||||
this(coord, tileData, tileDataHash, 0, 0, false, 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
|
|
|
|||
|
|
@ -50,7 +50,8 @@ public interface WriteableTileArchive extends Closeable {
|
|||
|
||||
// TODO: exists for compatibility reasons
|
||||
default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
|
||||
write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
|
||||
write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash(), 0,
|
||||
0, false, 0));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -355,6 +355,7 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
|
|||
private final TileCoord tileCoord;
|
||||
private final List<SortableFeature> entries = new ArrayList<>();
|
||||
private final AtomicLong numFeaturesProcessed = new AtomicLong(0);
|
||||
private final AtomicLong inputFeatureBytes = new AtomicLong(0);
|
||||
private LongLongHashMap counts = null;
|
||||
private byte lastLayer = Byte.MAX_VALUE;
|
||||
|
||||
|
|
@ -507,6 +508,7 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
|
|||
|
||||
void add(SortableFeature entry) {
|
||||
numFeaturesProcessed.incrementAndGet();
|
||||
inputFeatureBytes.addAndGet(entry.value().length);
|
||||
long key = entry.key();
|
||||
if (extractHasGroupFromKey(key)) {
|
||||
byte thisLayer = extractLayerIdFromKey(key);
|
||||
|
|
@ -535,5 +537,9 @@ public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>,
|
|||
", num entries=" + entries.size() +
|
||||
'}';
|
||||
}
|
||||
|
||||
public long getInputFeatureBytes() {
|
||||
return inputFeatureBytes.get();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import static com.onthegomap.planetiler.config.PlanetilerConfig.MAX_MAXZOOM;
|
|||
|
||||
import com.onthegomap.planetiler.util.Format;
|
||||
import com.onthegomap.planetiler.util.Hilbert;
|
||||
import java.util.regex.Pattern;
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
import org.locationtech.jts.geom.CoordinateXY;
|
||||
|
|
@ -26,6 +27,64 @@ import org.locationtech.jts.geom.Envelope;
|
|||
@Immutable
|
||||
public record TileCoord(int encoded, int x, int y, int z) implements Comparable<TileCoord> {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
|
||||
insertLinks("""
|
||||
| tile | inbytes | infeatures | outbytes | time |
|
||||
|------------|---------|------------|----------|-------|
|
||||
| 6/52/24 | 316,843 | 7,592 | 118,047 | 0.739 |
|
||||
| 6/50/26 | 263,488 | 5,587 | 101,577 | 0.668 |
|
||||
| 6/52/26 | 332,374 | 6,722 | 138,996 | 0.667 |
|
||||
| 10/308/381 | 604,776 | 6,533 | 362,984 | 0.662 |
|
||||
| 6/45/25 | 147,850 | 2,085 | 77,298 | 0.634 |
|
||||
| 6/51/26 | 221,077 | 4,312 | 89,517 | 0.63 |
|
||||
| 6/51/27 | 264,509 | 5,301 | 102,592 | 0.617 |
|
||||
| 6/15/22 | 124,147 | 1,480 | 70,605 | 0.597 |
|
||||
| 6/52/27 | 310,144 | 6,443 | 122,242 | 0.593 |
|
||||
| 1/0/0 | 149,781 | 3,906 | 63,429 | 0.586 |
|
||||
""");
|
||||
insertLinks("""
|
||||
| tile | inbytes | infeatures | outbytes | time |
|
||||
|--------------|---------|------------|----------|--------|
|
||||
| 7/68/33 | 574,106 | 3,310 | 253,432 | 48.02 |
|
||||
| 13/6527/4239 | 2,792,615 | 106,023 | 135,957 | 35.365 |
|
||||
| 13/6661/4261 | 3,111,509 | 114,647 | 130,210 | 32.91 |
|
||||
| 13/6661/4262 | 3,046,934 | 111,292 | 126,727 | 32.741 |
|
||||
| 13/6527/4240 | 2,344,566 | 88,393 | 116,492 | 29.108 |
|
||||
| 13/6527/4238 | 2,900,531 | 105,839 | 143,392 | 28.857 |
|
||||
| 13/6526/4236 | 3,025,297 | 106,137 | 137,592 | 28.829 |
|
||||
| 7/68/61 | 731,969 | 5,077 | 268,917 | 28.583 |
|
||||
| 13/6527/4237 | 2,843,197 | 99,475 | 144,424 | 28.142 |
|
||||
| 13/6607/4274 | 2,228,848 | 90,111 | 92,170 | 27.981 |
|
||||
""");
|
||||
insertLinks("""
|
||||
| tile | inbytes | infeatures | outbytes | time |
|
||||
|--------------|---------|------------|----------|--------|
|
||||
| 10/236/413 | 6,676,345 | 179,762 | 17,9050 | 16.77 |
|
||||
| 13/3037/4648 | 2,991,716 | 116,502 | 13,0376 | 13.434 |
|
||||
| 7/68/33 | 574,320 | 3,311 | 25,3391 | 11.826 |
|
||||
| 13/1436/3313 | 2,554,569 | 104,277 | 12,5866 | 11.517 |
|
||||
| 13/3038/4646 | 2,664,982 | 103,904 | 13,4165 | 11.47 |
|
||||
| 13/3037/4647 | 2,579,326 | 99,017 | 13,9578 | 11.2 |
|
||||
| 13/6527/4239 | 2,792,855 | 106,027 | 13,6069 | 11.08 |
|
||||
| 13/3036/4648 | 2,553,943 | 96,097 | 13,6788 | 11.064 |
|
||||
| 13/3039/4646 | 2,629,057 | 104,042 | 13,3678 | 11.02 |
|
||||
| 13/6661/4261 | 3,111,945 | 114,652 | 13,0269 | 10.987 |
|
||||
""");
|
||||
}
|
||||
|
||||
private static void insertLinks(String input) {
|
||||
var matcher = Pattern.compile("([0-9]+)/([0-9]+)/([0-9]+)").matcher(input);
|
||||
System.out.println(matcher.replaceAll(matchResult -> {
|
||||
int z = Integer.parseInt(matchResult.group(1));
|
||||
int x = Integer.parseInt(matchResult.group(2));
|
||||
int y = Integer.parseInt(matchResult.group(3));
|
||||
String url = TileCoord.ofXYZ(x, y, z).getDebugUrl();
|
||||
return "[" + z + "/" + x + "/" + y + "](" + url + ")";
|
||||
}));
|
||||
}
|
||||
|
||||
private static final int[] ZOOM_START_INDEX = new int[MAX_MAXZOOM + 1];
|
||||
|
||||
static {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
package com.onthegomap.planetiler.util;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class TileStatsArchive implements WriteableTileArchive {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TileStatsArchive.class);
|
||||
private final BufferedWriter writer;
|
||||
|
||||
public TileStatsArchive(Path output) throws IOException {
|
||||
this.writer = Files.newBufferedWriter(output, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deduplicates() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileOrder tileOrder() {
|
||||
return TileOrder.TMS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileWriter newTileWriter() {
|
||||
return new BulkLoader();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
writer.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(TileArchiveMetadata metadata) {
|
||||
try {
|
||||
writer.write("z\tx\ty\tinbytes\tinfeatures\tcached\toutbytes\ttime\n");
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private class BulkLoader implements TileWriter {
|
||||
|
||||
@Override
|
||||
public void write(TileEncodingResult encodingResult) {
|
||||
try {
|
||||
writer.write("%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d%n".formatted(
|
||||
encodingResult.coord().z(),
|
||||
encodingResult.coord().x(),
|
||||
encodingResult.coord().y(),
|
||||
encodingResult.inputBytes(),
|
||||
encodingResult.inputFeatures(),
|
||||
encodingResult.cached() ? 1 : 0,
|
||||
encodingResult.tileData().length,
|
||||
encodingResult.time()
|
||||
));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {}
|
||||
}
|
||||
}
|
||||
Ładowanie…
Reference in New Issue