kopia lustrzana https://github.com/onthegomap/planetiler
Make OSM polygon construction deterministic (#788)
Michael Barry
GitHub
rodzic
36977d5388
commit
062528b1ee
|
|
@ -7,6 +7,7 @@ import com.carrotsearch.hppc.LongIntHashMap;
|
|||
import com.carrotsearch.hppc.LongLongHashMap;
|
||||
import com.carrotsearch.hppc.LongObjectHashMap;
|
||||
import com.carrotsearch.hppc.ObjectIntHashMap;
|
||||
import com.carrotsearch.hppc.SortedIterationLongObjectHashMap;
|
||||
|
||||
/**
|
||||
* Static factory method for <a href="https://github.com/carrotsearch/hppc">High Performance Primitive Collections</a>.
|
||||
|
|
@ -40,4 +41,8 @@ public class Hppc {
|
|||
public static LongByteMap newLongByteHashMap() {
|
||||
return new LongByteHashMap(10, 0.75);
|
||||
}
|
||||
|
||||
public static <T> SortedIterationLongObjectHashMap<T> sortedView(LongObjectHashMap<T> input) {
|
||||
return new SortedIterationLongObjectHashMap<>(input, Long::compare);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,10 +24,10 @@ import java.nio.file.StandardOpenOption;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.TreeMap;
|
||||
import java.util.function.LongSupplier;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -147,7 +147,8 @@ public final class WriteablePmtiles implements WriteableTileArchive {
|
|||
}
|
||||
try {
|
||||
Directories directories = makeDirectories(entries);
|
||||
var otherMetadata = new LinkedHashMap<>(tileArchiveMetadata.toMap());
|
||||
// use treemap to ensure consistent ouput between runs
|
||||
var otherMetadata = new TreeMap<>(tileArchiveMetadata.toMap());
|
||||
|
||||
// exclude keys included in top-level header
|
||||
otherMetadata.remove(TileArchiveMetadata.CENTER_KEY);
|
||||
|
|
|
|||
|
|
@ -14,15 +14,14 @@
|
|||
package com.onthegomap.planetiler.reader.osm;
|
||||
|
||||
import com.carrotsearch.hppc.LongArrayList;
|
||||
import com.carrotsearch.hppc.LongObjectMap;
|
||||
import com.carrotsearch.hppc.LongObjectHashMap;
|
||||
import com.carrotsearch.hppc.ObjectIntMap;
|
||||
import com.carrotsearch.hppc.cursors.LongObjectCursor;
|
||||
import com.onthegomap.planetiler.collection.Hppc;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.geo.GeometryException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import org.locationtech.jts.geom.Coordinate;
|
||||
|
|
@ -41,8 +40,9 @@ import org.locationtech.jts.geom.prep.PreparedPolygon;
|
|||
* Multipolygon way members have an "inner" and "outer" role, but they can be incorrectly specified, so instead
|
||||
* determine the nesting order and alternate outer/inner/outer/inner... from the outermost ring inwards.
|
||||
* <p>
|
||||
* This class is ported to Java from https://github.com/omniscale/imposm3/blob/master/geom/multipolygon.go and
|
||||
* https://github.com/omniscale/imposm3/blob/master/geom/ring.go
|
||||
* This class is ported to Java from
|
||||
* <a href="https://github.com/omniscale/imposm3/blob/master/geom/multipolygon.go">imposm3 multipolygon.go</a> and
|
||||
* <a href="https://github.com/omniscale/imposm3/blob/master/geom/ring.go">imposm3 ring.go</a>
|
||||
*/
|
||||
public class OsmMultipolygon {
|
||||
/*
|
||||
|
|
@ -62,7 +62,8 @@ public class OsmMultipolygon {
|
|||
private final Polygon geom;
|
||||
private final double area;
|
||||
private Ring containedBy = null;
|
||||
private final Set<Ring> holes = new HashSet<>();
|
||||
// use linked hash set to ensure same input always produces same output
|
||||
private final Set<Ring> holes = new LinkedHashSet<>();
|
||||
|
||||
private Ring(Polygon geom) {
|
||||
this.geom = geom;
|
||||
|
|
@ -163,7 +164,7 @@ public class OsmMultipolygon {
|
|||
boolean fix
|
||||
) throws GeometryException {
|
||||
try {
|
||||
if (rings.size() == 0) {
|
||||
if (rings.isEmpty()) {
|
||||
throw new GeometryException.Verbose("osm_invalid_multipolygon_empty",
|
||||
"error building multipolygon " + osmId + ": no rings to process");
|
||||
}
|
||||
|
|
@ -175,7 +176,7 @@ public class OsmMultipolygon {
|
|||
}
|
||||
polygons.sort(BY_AREA_DESCENDING);
|
||||
Set<Ring> shells = groupParentChildShells(polygons);
|
||||
if (shells.size() == 0) {
|
||||
if (shells.isEmpty()) {
|
||||
throw new GeometryException.Verbose("osm_invalid_multipolygon_not_closed",
|
||||
"error building multipolygon " + osmId + ": multipolygon not closed");
|
||||
} else if (shells.size() == 1) {
|
||||
|
|
@ -227,7 +228,8 @@ public class OsmMultipolygon {
|
|||
}
|
||||
|
||||
private static Set<Ring> groupParentChildShells(List<Ring> polygons) {
|
||||
Set<Ring> shells = new HashSet<>();
|
||||
// use linked hash set to ensure the same input always produces the same output
|
||||
Set<Ring> shells = new LinkedHashSet<>();
|
||||
int numPolygons = polygons.size();
|
||||
if (numPolygons == 0) {
|
||||
return shells;
|
||||
|
|
@ -313,7 +315,7 @@ public class OsmMultipolygon {
|
|||
}
|
||||
|
||||
static List<LongArrayList> connectPolygonSegments(List<LongArrayList> outer) {
|
||||
LongObjectMap<LongArrayList> endpointIndex = Hppc.newLongObjectHashMap(outer.size() * 2);
|
||||
LongObjectHashMap<LongArrayList> endpointIndex = Hppc.newLongObjectHashMap(outer.size() * 2);
|
||||
List<LongArrayList> completeRings = new ArrayList<>(outer.size());
|
||||
|
||||
for (LongArrayList ids : outer) {
|
||||
|
|
@ -366,12 +368,11 @@ public class OsmMultipolygon {
|
|||
}
|
||||
}
|
||||
|
||||
for (LongObjectCursor<LongArrayList> cursor : endpointIndex) {
|
||||
LongArrayList value = cursor.value;
|
||||
if (value.size() >= 4) {
|
||||
if (value.get(0) == value.get(value.size() - 1) || cursor.key == value.get(0)) {
|
||||
completeRings.add(value);
|
||||
}
|
||||
// iterate in sorted order to ensure the same input always produces the same output
|
||||
for (var entry : Hppc.sortedView(endpointIndex)) {
|
||||
LongArrayList value = entry.value;
|
||||
if (value.size() >= 4 && (value.get(0) == value.get(value.size() - 1) || entry.key == value.get(0))) {
|
||||
completeRings.add(value);
|
||||
}
|
||||
}
|
||||
return completeRings;
|
||||
|
|
|
|||
|
|
@ -592,7 +592,7 @@ public class OsmReader implements Closeable, MemoryEstimator.HasEstimate {
|
|||
* @param role "role" of the relation member
|
||||
* @param relation user-provided data about the relation from pass1
|
||||
*/
|
||||
public record RelationMember<T extends OsmRelationInfo> (String role, T relation) {}
|
||||
public record RelationMember<T extends OsmRelationInfo>(String role, T relation) {}
|
||||
|
||||
/** Raw relation membership data that gets encoded/decoded into a long. */
|
||||
private record RelationMembership(String role, long relationId) {}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
package com.onthegomap.planetiler.util;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import com.onthegomap.planetiler.VectorTile;
|
||||
import com.onthegomap.planetiler.archive.Tile;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveConfig;
|
||||
import com.onthegomap.planetiler.archive.TileArchives;
|
||||
import com.onthegomap.planetiler.archive.TileCompression;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.GeometryException;
|
||||
import com.onthegomap.planetiler.geo.GeometryType;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
|
||||
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.io.IOException;
|
||||
|
|
@ -18,6 +24,9 @@ import java.util.Set;
|
|||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.Supplier;
|
||||
import org.locationtech.jts.geom.Geometry;
|
||||
import org.locationtech.jts.geom.MultiPolygon;
|
||||
import org.locationtech.jts.geom.Polygon;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import vector_tile.VectorTileProto;
|
||||
|
|
@ -38,15 +47,17 @@ public class CompareArchives {
|
|||
private final Map<String, Map<String, Long>> diffsByLayer = new ConcurrentHashMap<>();
|
||||
private final TileArchiveConfig input1;
|
||||
private final TileArchiveConfig input2;
|
||||
private final boolean verbose;
|
||||
|
||||
private CompareArchives(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2) {
|
||||
private CompareArchives(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2, boolean verbose) {
|
||||
this.verbose = verbose;
|
||||
this.input1 = archiveConfig1;
|
||||
this.input2 = archiveConfig2;
|
||||
}
|
||||
|
||||
public static Result compare(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2,
|
||||
PlanetilerConfig config) {
|
||||
return new CompareArchives(archiveConfig1, archiveConfig2).getResult(config);
|
||||
PlanetilerConfig config, boolean verbose) {
|
||||
return new CompareArchives(archiveConfig1, archiveConfig2, verbose).getResult(config);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
|
@ -58,12 +69,13 @@ public class CompareArchives {
|
|||
String inputString1 = args[args.length - 2];
|
||||
String inputString2 = args[args.length - 1];
|
||||
var arguments = Arguments.fromArgsOrConfigFile(Arrays.copyOf(args, args.length - 2));
|
||||
var verbose = arguments.getBoolean("verbose", "log each tile diff", false);
|
||||
var config = PlanetilerConfig.from(arguments);
|
||||
var input1 = TileArchiveConfig.from(inputString1);
|
||||
var input2 = TileArchiveConfig.from(inputString2);
|
||||
|
||||
try {
|
||||
var result = compare(input1, input2, config);
|
||||
var result = compare(input1, input2, config, verbose);
|
||||
|
||||
var format = Format.defaultInstance();
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
|
|
@ -88,12 +100,11 @@ public class CompareArchives {
|
|||
}
|
||||
|
||||
private Result getResult(PlanetilerConfig config) {
|
||||
final TileCompression compression2;
|
||||
final TileCompression compression1;
|
||||
if (!input1.format().equals(input2.format())) {
|
||||
throw new IllegalArgumentException(
|
||||
"input1 and input2 must have the same format, got " + input1.format() + " and " +
|
||||
input2.format());
|
||||
LOGGER.warn("archive1 and archive2 have different formats, got {} and {}", input1.format(), input2.format());
|
||||
}
|
||||
final TileCompression compression;
|
||||
try (
|
||||
var reader1 = TileArchives.newReader(input1, config);
|
||||
var reader2 = TileArchives.newReader(input2, config);
|
||||
|
|
@ -107,18 +118,34 @@ public class CompareArchives {
|
|||
archive2: {}
|
||||
""", reader1.metadata(), reader2.metadata());
|
||||
}
|
||||
compression = metadata1 == null ? TileCompression.UNKNOWN : metadata1.tileCompression();
|
||||
TileCompression compression2 = metadata2 == null ? TileCompression.UNKNOWN : metadata2.tileCompression();
|
||||
if (compression != compression2) {
|
||||
throw new IllegalArgumentException(
|
||||
"input1 and input2 must have the same compression, got " + compression + " and " +
|
||||
compression2);
|
||||
if (reader1 instanceof ReadablePmtiles pmt1 && reader2 instanceof ReadablePmtiles pmt2) {
|
||||
var header1 = pmt1.getHeader();
|
||||
var header2 = pmt2.getHeader();
|
||||
if (!Objects.equals(header1, header2)) {
|
||||
LOGGER.warn("""
|
||||
archive1 and archive2 have different pmtiles headers
|
||||
archive1: {}
|
||||
archive2: {}
|
||||
""", header1, header2);
|
||||
}
|
||||
}
|
||||
compression1 = metadata1 == null ? TileCompression.UNKNOWN : metadata1.tileCompression();
|
||||
compression2 = metadata2 == null ? TileCompression.UNKNOWN : metadata2.tileCompression();
|
||||
if (compression1 != compression2) {
|
||||
LOGGER.warn(
|
||||
"input1 and input2 must have the same compression, got {} and {} - will compare decompressed tile contents instead",
|
||||
compression1, compression2);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
|
||||
var order = input1.format().preferredOrder();
|
||||
var order2 = input2.format().preferredOrder();
|
||||
if (order != order2) {
|
||||
throw new IllegalArgumentException(
|
||||
"Archive orders must be the same to compare, got " + order + " and " + order2);
|
||||
}
|
||||
var stats = config.arguments().getStats();
|
||||
var total = new AtomicLong(0);
|
||||
var diffs = new AtomicLong(0);
|
||||
|
|
@ -160,22 +187,39 @@ public class CompareArchives {
|
|||
})
|
||||
.addBuffer("diffs", 50_000, 1_000)
|
||||
.sinkTo("process", config.featureProcessThreads(), prev -> {
|
||||
boolean sameCompression = compression1 == compression2;
|
||||
for (var diff : prev) {
|
||||
var a = diff.a();
|
||||
var b = diff.b();
|
||||
total.incrementAndGet();
|
||||
if (a == null) {
|
||||
recordTileDiff("archive 1 missing tile");
|
||||
recordTileDiff(b.coord(), "archive 1 missing tile");
|
||||
diffs.incrementAndGet();
|
||||
} else if (b == null) {
|
||||
recordTileDiff("archive 2 missing tile");
|
||||
recordTileDiff(a.coord(), "archive 2 missing tile");
|
||||
diffs.incrementAndGet();
|
||||
} else if (!Arrays.equals(a.bytes(), b.bytes())) {
|
||||
recordTileDiff("different contents");
|
||||
diffs.incrementAndGet();
|
||||
var proto1 = decode(a.bytes(), compression);
|
||||
var proto2 = decode(b.bytes(), compression);
|
||||
compareTiles(proto1, proto2);
|
||||
} else if (sameCompression) {
|
||||
if (!Arrays.equals(a.bytes(), b.bytes())) {
|
||||
recordTileDiff(a.coord(), "different contents");
|
||||
diffs.incrementAndGet();
|
||||
compareTiles(
|
||||
a.coord(),
|
||||
decode(decompress(a.bytes(), compression1)),
|
||||
decode(decompress(b.bytes(), compression2))
|
||||
);
|
||||
}
|
||||
} else { // different compression
|
||||
var decompressed1 = decompress(a.bytes(), compression1);
|
||||
var decompressed2 = decompress(b.bytes(), compression2);
|
||||
if (!Arrays.equals(decompressed1, decompressed2)) {
|
||||
recordTileDiff(a.coord(), "different decompressed contents");
|
||||
diffs.incrementAndGet();
|
||||
compareTiles(
|
||||
a.coord(),
|
||||
decode(decompressed1),
|
||||
decode(decompressed2)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -189,104 +233,157 @@ public class CompareArchives {
|
|||
return new Result(total.get(), diffs.get(), diffTypes, diffsByLayer);
|
||||
}
|
||||
|
||||
private void compareTiles(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
|
||||
compareLayerNames(proto1, proto2);
|
||||
private void compareTiles(TileCoord coord, VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
|
||||
compareLayerNames(coord, proto1, proto2);
|
||||
for (int i = 0; i < proto1.getLayersCount() && i < proto2.getLayersCount(); i++) {
|
||||
var layer1 = proto1.getLayers(i);
|
||||
var layer2 = proto2.getLayers(i);
|
||||
compareLayer(layer1, layer2);
|
||||
compareLayer(coord, layer1, layer2);
|
||||
}
|
||||
}
|
||||
|
||||
private void compareLayer(VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) {
|
||||
private void compareLayer(TileCoord coord, VectorTileProto.Tile.Layer layer1, VectorTileProto.Tile.Layer layer2) {
|
||||
String name = layer1.getName();
|
||||
compareValues(name, "version", layer1.getVersion(), layer2.getVersion());
|
||||
compareValues(name, "extent", layer1.getExtent(), layer2.getExtent());
|
||||
compareList(name, "keys list", layer1.getKeysList(), layer2.getKeysList());
|
||||
compareList(name, "values list", layer1.getValuesList(), layer2.getValuesList());
|
||||
if (compareValues(name, "features count", layer1.getFeaturesCount(), layer2.getFeaturesCount())) {
|
||||
compareValues(coord, name, "version", layer1.getVersion(), layer2.getVersion());
|
||||
compareValues(coord, name, "extent", layer1.getExtent(), layer2.getExtent());
|
||||
compareList(coord, name, "keys list", layer1.getKeysList(), layer2.getKeysList());
|
||||
compareList(coord, name, "values list", layer1.getValuesList(), layer2.getValuesList());
|
||||
if (compareValues(coord, name, "features count", layer1.getFeaturesCount(), layer2.getFeaturesCount())) {
|
||||
var ids1 = layer1.getFeaturesList().stream().map(f -> f.getId()).toList();
|
||||
var ids2 = layer2.getFeaturesList().stream().map(f -> f.getId()).toList();
|
||||
if (compareValues(name, "feature ids", Set.of(ids1), Set.of(ids2)) &&
|
||||
compareValues(name, "feature order", ids1, ids2)) {
|
||||
if (compareValues(coord, name, "feature ids", Set.of(ids1), Set.of(ids2)) &&
|
||||
compareValues(coord, name, "feature order", ids1, ids2)) {
|
||||
for (int i = 0; i < layer1.getFeaturesCount() && i < layer2.getFeaturesCount(); i++) {
|
||||
var feature1 = layer1.getFeatures(i);
|
||||
var feature2 = layer2.getFeatures(i);
|
||||
compareFeature(name, feature1, feature2);
|
||||
compareFeature(coord, name, feature1, feature2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void compareFeature(String layer, VectorTileProto.Tile.Feature feature1,
|
||||
private void compareFeature(TileCoord coord, String layer, VectorTileProto.Tile.Feature feature1,
|
||||
VectorTileProto.Tile.Feature feature2) {
|
||||
compareValues(layer, "feature id", feature1.getId(), feature2.getId());
|
||||
compareValues(layer, "feature type", feature1.getType(), feature2.getType());
|
||||
compareValues(layer, "feature geometry", feature1.getGeometryList(), feature2.getGeometryList());
|
||||
compareValues(layer, "feature tags", feature1.getTagsCount(), feature2.getTagsCount());
|
||||
compareValues(coord, layer, "feature id", feature1.getId(), feature2.getId());
|
||||
compareGeometry(coord, layer, feature1, feature2);
|
||||
compareValues(coord, layer, "feature tags", feature1.getTagsCount(), feature2.getTagsCount());
|
||||
}
|
||||
|
||||
private void compareLayerNames(VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
|
||||
private void compareGeometry(TileCoord coord, String layer, VectorTileProto.Tile.Feature feature1,
|
||||
VectorTileProto.Tile.Feature feature2) {
|
||||
if (compareValues(coord, layer, "feature type", feature1.getType(), feature2.getType())) {
|
||||
var geomType = feature1.getType();
|
||||
if (!compareValues(coord, layer, "feature " + geomType.toString().toLowerCase() + " geometry commands",
|
||||
feature1.getGeometryList(), feature2.getGeometryList())) {
|
||||
var geom1 =
|
||||
new VectorTile.VectorGeometry(Ints.toArray(feature1.getGeometryList()), GeometryType.valueOf(geomType), 0);
|
||||
var geom2 =
|
||||
new VectorTile.VectorGeometry(Ints.toArray(feature2.getGeometryList()), GeometryType.valueOf(geomType), 0);
|
||||
try {
|
||||
compareGeometry(coord, layer, geom1.decode(), geom2.decode());
|
||||
} catch (GeometryException e) {
|
||||
LOGGER.error("Error decoding geometry", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void compareGeometry(TileCoord coord, String layer, Geometry geom1, Geometry geom2) {
|
||||
String geometryType = geom1.getGeometryType();
|
||||
compareValues(coord, layer, "feature JTS geometry type", geom1.getGeometryType(), geom2.getGeometryType());
|
||||
compareValues(coord, layer, "feature num geometries", geom1.getNumGeometries(), geom2.getNumGeometries());
|
||||
if (geom1 instanceof MultiPolygon) {
|
||||
for (int i = 0; i < geom1.getNumGeometries(); i++) {
|
||||
comparePolygon(coord, layer, geometryType, (Polygon) geom1.getGeometryN(i), (Polygon) geom2.getGeometryN(i));
|
||||
}
|
||||
} else if (geom1 instanceof Polygon p1 && geom2 instanceof Polygon p2) {
|
||||
comparePolygon(coord, layer, geometryType, p1, p2);
|
||||
}
|
||||
}
|
||||
|
||||
private void comparePolygon(TileCoord coord, String layer, String geomType, Polygon p1, Polygon p2) {
|
||||
compareValues(coord, layer, geomType + " exterior ring geometry", p1.getExteriorRing(), p2.getExteriorRing());
|
||||
if (compareValues(coord, layer, geomType + " num interior rings", p1.getNumInteriorRing(),
|
||||
p2.getNumInteriorRing())) {
|
||||
for (int i = 0; i < p1.getNumInteriorRing(); i++) {
|
||||
compareValues(coord, layer, geomType + " interior ring geometry", p1.getInteriorRingN(i),
|
||||
p2.getInteriorRingN(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void compareLayerNames(TileCoord coord, VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) {
|
||||
var layers1 = proto1.getLayersList().stream().map(d -> d.getName()).toList();
|
||||
var layers2 = proto2.getLayersList().stream().map(d -> d.getName()).toList();
|
||||
compareListDetailed("tile layers", layers1, layers2);
|
||||
compareListDetailed(coord, "tile layers", layers1, layers2);
|
||||
}
|
||||
|
||||
private <T> boolean compareList(String layer, String name, List<T> value1, List<T> value2) {
|
||||
return compareValues(layer, name + " unique values", Set.copyOf(value1), Set.copyOf(value2)) &&
|
||||
compareValues(layer, name + " order", value1, value2);
|
||||
private <T> boolean compareList(TileCoord coord, String layer, String name, List<T> value1, List<T> value2) {
|
||||
return compareValues(coord, layer, name + " unique values", Set.copyOf(value1), Set.copyOf(value2)) &&
|
||||
compareValues(coord, layer, name + " order", value1, value2);
|
||||
}
|
||||
|
||||
private <T> void compareListDetailed(String name, List<T> value1, List<T> value2) {
|
||||
private <T> void compareListDetailed(TileCoord coord, String name, List<T> value1, List<T> value2) {
|
||||
if (!Objects.equals(value1, value2)) {
|
||||
boolean missing = false;
|
||||
for (var layer : value1) {
|
||||
if (!value2.contains(layer)) {
|
||||
recordTileDiff(name + " 2 missing " + layer);
|
||||
recordTileDiff(coord, name + " 2 missing " + layer);
|
||||
missing = true;
|
||||
}
|
||||
}
|
||||
for (var layer : value2) {
|
||||
if (!value1.contains(layer)) {
|
||||
recordTileDiff(name + " 1 missing " + layer);
|
||||
recordTileDiff(coord, name + " 1 missing " + layer);
|
||||
missing = true;
|
||||
}
|
||||
}
|
||||
if (!missing) {
|
||||
recordTileDiff(name + " different order");
|
||||
recordTileDiff(coord, name + " different order");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private <T> boolean compareValues(String layer, String name, T value1, T value2) {
|
||||
private <T> boolean compareValues(TileCoord coord, String layer, String name, T value1, T value2) {
|
||||
if (!Objects.equals(value1, value2)) {
|
||||
recordLayerDiff(layer, name);
|
||||
recordLayerDiff(coord, layer, name);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private VectorTileProto.Tile decode(byte[] bytes, TileCompression tileCompression) throws IOException {
|
||||
byte[] decompressed = switch (tileCompression) {
|
||||
private byte[] decompress(byte[] bytes, TileCompression tileCompression) throws IOException {
|
||||
return switch (tileCompression) {
|
||||
case GZIP -> Gzip.gunzip(bytes);
|
||||
case NONE -> bytes;
|
||||
case UNKNOWN -> throw new IllegalArgumentException("Unknown compression");
|
||||
};
|
||||
return VectorTileProto.Tile.parseFrom(decompressed);
|
||||
}
|
||||
|
||||
private void recordLayerDiff(String layer, String issue) {
|
||||
private VectorTileProto.Tile decode(byte[] decompressedTile) throws IOException {
|
||||
return VectorTileProto.Tile.parseFrom(decompressedTile);
|
||||
}
|
||||
|
||||
private void recordLayerDiff(TileCoord coord, String layer, String issue) {
|
||||
var layerDiffs = diffsByLayer.get(layer);
|
||||
if (layerDiffs == null) {
|
||||
layerDiffs = diffsByLayer.computeIfAbsent(layer, k -> new ConcurrentHashMap<>());
|
||||
}
|
||||
layerDiffs.merge(issue, 1L, Long::sum);
|
||||
if (verbose) {
|
||||
LOGGER.debug("{} layer {} {}", coord, layer, issue);
|
||||
}
|
||||
}
|
||||
|
||||
private void recordTileDiff(String issue) {
|
||||
private void recordTileDiff(TileCoord coord, String issue) {
|
||||
diffTypes.merge(issue, 1L, Long::sum);
|
||||
if (verbose) {
|
||||
LOGGER.debug("{} {}", coord, issue);
|
||||
}
|
||||
}
|
||||
|
||||
public record Result(long total, long tileDiffs, Map<String, Long> diffTypes,
|
||||
Map<String, Map<String, Long>> diffsByLayer) {}
|
||||
public record Result(
|
||||
long total, long tileDiffs, Map<String, Long> diffTypes,
|
||||
Map<String, Map<String, Long>> diffsByLayer
|
||||
) {}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package com.onthegomap.planetiler.util;
|
|||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
|
@ -97,10 +96,6 @@ public class LayerAttrStats {
|
|||
this(id, fields, Optional.empty(), OptionalInt.of(minzoom), OptionalInt.of(maxzoom));
|
||||
}
|
||||
|
||||
public static VectorLayer forLayer(String id) {
|
||||
return new VectorLayer(id, new HashMap<>());
|
||||
}
|
||||
|
||||
public VectorLayer withDescription(String newDescription) {
|
||||
return new VectorLayer(id, fields, Optional.of(newDescription), minzoom, maxzoom);
|
||||
}
|
||||
|
|
@ -174,7 +169,8 @@ public class LayerAttrStats {
|
|||
private static class StatsForLayer {
|
||||
|
||||
private final String layer;
|
||||
private final Map<String, FieldType> fields = new HashMap<>();
|
||||
// use TreeMap to ensure the same output always appears the same in an archive
|
||||
private final Map<String, FieldType> fields = new TreeMap<>();
|
||||
private int minzoom = Integer.MAX_VALUE;
|
||||
private int maxzoom = Integer.MIN_VALUE;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import com.onthegomap.planetiler.Profile;
|
|||
import com.onthegomap.planetiler.archive.TileArchiveConfig;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
|
||||
|
|
@ -73,7 +74,8 @@ class CompareArchivesTest {
|
|||
var result = CompareArchives.compare(
|
||||
TileArchiveConfig.from(aPath.toString()),
|
||||
TileArchiveConfig.from(bPath.toString()),
|
||||
config
|
||||
config,
|
||||
false
|
||||
);
|
||||
assertEquals(new CompareArchives.Result(
|
||||
5, 4, Map.of(
|
||||
|
|
@ -88,4 +90,58 @@ class CompareArchivesTest {
|
|||
)
|
||||
), result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCompareArchivesDifferentCompression() throws IOException {
|
||||
var aPath = path.resolve("a.pmtiles");
|
||||
var bPath = path.resolve("b.pmtiles");
|
||||
byte[] a1 = new byte[]{0xa, 0x2};
|
||||
byte[] b1 = Gzip.gzip(a1);
|
||||
byte[] a2 = tile1;
|
||||
byte[] b2 = Gzip.gzip(tile2);
|
||||
try (
|
||||
var a = WriteablePmtiles.newWriteToFile(aPath);
|
||||
var b = WriteablePmtiles.newWriteToFile(bPath);
|
||||
) {
|
||||
a.initialize();
|
||||
b.initialize();
|
||||
try (
|
||||
var aWriter = a.newTileWriter();
|
||||
var bWriter = b.newTileWriter()
|
||||
) {
|
||||
aWriter
|
||||
.write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty()));
|
||||
aWriter
|
||||
.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a2, OptionalLong.empty()));
|
||||
aWriter
|
||||
.write(new TileEncodingResult(TileOrder.HILBERT.decode(4), a1, OptionalLong.empty()));
|
||||
bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(1), b1, OptionalLong.empty()));
|
||||
bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), b2, OptionalLong.empty()));
|
||||
bWriter.write(new TileEncodingResult(TileOrder.HILBERT.decode(3), b1, OptionalLong.empty()));
|
||||
bWriter
|
||||
.write(new TileEncodingResult(TileOrder.HILBERT.decode(4), b1, OptionalLong.empty()));
|
||||
}
|
||||
a.finish(new TileArchiveMetadata(new Profile.NullProfile(),
|
||||
PlanetilerConfig.from(Arguments.fromArgs("--tile-compression=none"))));
|
||||
b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
|
||||
}
|
||||
var result = CompareArchives.compare(
|
||||
TileArchiveConfig.from(aPath.toString()),
|
||||
TileArchiveConfig.from(bPath.toString()),
|
||||
config,
|
||||
false
|
||||
);
|
||||
assertEquals(new CompareArchives.Result(
|
||||
5, 4, Map.of(
|
||||
"archive 2 missing tile", 1L,
|
||||
"archive 1 missing tile", 2L,
|
||||
"different decompressed contents", 1L
|
||||
), Map.of(
|
||||
"layer1", Map.of(
|
||||
"values list unique values", 1L,
|
||||
"feature ids", 1L
|
||||
)
|
||||
)
|
||||
), result);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue