diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java index d3a0b4ed..12d7836c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java @@ -13,15 +13,21 @@ import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.stats.ProgressLoggers; +import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.worker.Worker; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import org.locationtech.jts.geom.Geometry; @@ -45,6 +51,7 @@ public class CompareArchives { private static final Logger LOGGER = LoggerFactory.getLogger(CompareArchives.class); private final Map diffTypes = new ConcurrentHashMap<>(); private final Map> diffsByLayer = new ConcurrentHashMap<>(); + private final List archiveDiffs = new CopyOnWriteArrayList<>(); private final TileArchiveConfig input1; private final TileArchiveConfig input2; private final boolean verbose; @@ -55,6 +62,9 @@ public class CompareArchives { this.input2 = archiveConfig2; } + /** + * @throws FatalComparisonFailure if a comparison failure is encountered that prevents comparing the whole archives. + */ public static Result compare(TileArchiveConfig archiveConfig1, TileArchiveConfig archiveConfig2, PlanetilerConfig config, boolean verbose) { return new CompareArchives(archiveConfig1, archiveConfig2, verbose).getResult(config); @@ -70,6 +80,7 @@ public class CompareArchives { String inputString2 = args[args.length - 1]; var arguments = Arguments.fromArgsOrConfigFile(Arrays.copyOf(args, args.length - 2)); var verbose = arguments.getBoolean("verbose", "log each tile diff", false); + var strict = arguments.getBoolean("strict", "set to false to only fail on tile diffs", true); var config = PlanetilerConfig.from(arguments); var input1 = TileArchiveConfig.from(inputString1); var input2 = TileArchiveConfig.from(inputString2); @@ -80,61 +91,55 @@ public class CompareArchives { var format = Format.defaultInstance(); if (LOGGER.isInfoEnabled()) { LOGGER.info("Detailed diffs:"); - for (var entry : result.diffsByLayer.entrySet()) { + for (var entry : result.tileDiffsByLayer.entrySet()) { LOGGER.info(" \"{}\" layer", entry.getKey()); for (var layerEntry : entry.getValue().entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { LOGGER.info(" {}: {}", layerEntry.getKey(), format.integer(layerEntry.getValue())); } } - for (var entry : result.diffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { + for (var entry : result.tileDiffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue())); } + for (var diffType : result.archiveDiffs) { + LOGGER.info(" {}", diffType); + } LOGGER.info("Total tiles: {}", format.integer(result.total)); - LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs), + LOGGER.info("Tile diffs: {} ({} of all tiles)", format.integer(result.tileDiffs), format.percent(result.tileDiffs * 1d / result.total)); } - } catch (IllegalArgumentException e) { + System.exit((result.tileDiffs > 0 || (strict && !result.archiveDiffs.isEmpty())) ? 1 : 0); + } catch (FatalComparisonFailure e) { LOGGER.error("Error comparing archives {}", e.getMessage()); System.exit(1); } } + public static class FatalComparisonFailure extends IllegalArgumentException { + FatalComparisonFailure(String message) { + super(message); + } + } + private Result getResult(PlanetilerConfig config) { final TileCompression compression2; final TileCompression compression1; - if (!input1.format().equals(input2.format())) { - LOGGER.warn("archive1 and archive2 have different formats, got {} and {}", input1.format(), input2.format()); - } + compareArchive("format", input1.format(), input2.format()); try ( var reader1 = TileArchives.newReader(input1, config); var reader2 = TileArchives.newReader(input2, config); ) { var metadata1 = reader1.metadata(); var metadata2 = reader2.metadata(); - if (!Objects.equals(metadata1, metadata2)) { - LOGGER.warn(""" - archive1 and archive2 have different metadata - archive1: {} - archive2: {} - """, reader1.metadata(), reader2.metadata()); - } + compareArchive("metadata", metadata1, metadata2); if (reader1 instanceof ReadablePmtiles pmt1 && reader2 instanceof ReadablePmtiles pmt2) { var header1 = pmt1.getHeader(); var header2 = pmt2.getHeader(); - if (!Objects.equals(header1, header2)) { - LOGGER.warn(""" - archive1 and archive2 have different pmtiles headers - archive1: {} - archive2: {} - """, header1, header2); - } + compareArchive("pmtiles header", header1, header2); } compression1 = metadata1 == null ? TileCompression.UNKNOWN : metadata1.tileCompression(); compression2 = metadata2 == null ? TileCompression.UNKNOWN : metadata2.tileCompression(); - if (compression1 != compression2) { - LOGGER.warn( - "input1 and input2 must have the same compression, got {} and {} - will compare decompressed tile contents instead", - compression1, compression2); + if (!compareArchive("tile compression", compression1, compression2)) { + LOGGER.warn("Will compare decompressed tile contents instead"); } } catch (IOException e) { throw new UncheckedIOException(e); @@ -143,7 +148,7 @@ public class CompareArchives { var order = input1.format().preferredOrder(); var order2 = input2.format().preferredOrder(); if (order != order2) { - throw new IllegalArgumentException( + throw new FatalComparisonFailure( "Archive orders must be the same to compare, got " + order + " and " + order2); } var stats = config.arguments().getStats(); @@ -232,7 +237,73 @@ public class CompareArchives { .newLine() .addProcessStats(); loggers.awaitAndLog(pipeline.done(), config.logInterval()); - return new Result(total.get(), diffs.get(), diffTypes, diffsByLayer); + if (archiveDiffs.isEmpty() && diffs.get() == 0) { + var path1 = input1.getLocalPath(); + var path2 = input2.getLocalPath(); + if (path1 != null && path2 != null && Files.isRegularFile(path1) && Files.isRegularFile(path2)) { + LOGGER.info("No diffs so far, comparing bytes in {} vs. {}", path1, path2); + compareFiles(path1, path2, config); + } + } + return new Result(total.get(), diffs.get(), archiveDiffs, diffTypes, diffsByLayer); + } + + private void compareFiles(Path path1, Path path2, PlanetilerConfig config) { + long size = FileUtils.fileSize(path1); + if (compareArchive("archive size", size, FileUtils.fileSize(path2))) { + AtomicLong bytesRead = new AtomicLong(0); + var worker = new Worker("compare", Stats.inMemory(), 1, () -> { + byte[] bytes1 = new byte[8192]; + byte[] bytes2 = new byte[8192]; + long n = 0; + try ( + var is1 = Files.newInputStream(path1, StandardOpenOption.READ); + var is2 = Files.newInputStream(path2, StandardOpenOption.READ) + ) { + do { + int len = is1.read(bytes1); + int len2 = is2.read(bytes2, 0, len); + if (len2 != len) { + String message = "Expected to read %s bytes from %s but got %s".formatted(len, path2, len2); + archiveDiffs.add(message); + LOGGER.warn(message); + return; + } + int mismatch = Arrays.mismatch(bytes1, bytes2); + if (mismatch >= 0 && mismatch < len) { + archiveDiffs.add("mismatch at byte %s".formatted(mismatch + n)); + LOGGER.warn("Archives mismatch ay byte {}", mismatch + n); + return; + } + n += len; + bytesRead.set(n); + } while (n < size); + } + LOGGER.info("No mismatches! Analyzed {} / {} bytes", n, size); + }); + + var logger = ProgressLoggers.create() + .addStorageRatePercentCounter("bytes", size, bytesRead::get, true) + .newLine() + .addThreadPoolStats("compare", worker) + .newLine() + .addProcessStats(); + + worker.awaitAndLog(logger, config.logInterval()); + } + } + + private boolean compareArchive(String name, T a, T b) { + if (Objects.equals(a, b)) { + return true; + } + LOGGER.warn(""" + archive1 and archive2 have different {} + archive1: {} + archive2: {} + """, name, a, b); + archiveDiffs.add(name); + return false; } private void compareTiles(TileCoord coord, VectorTileProto.Tile proto1, VectorTileProto.Tile proto2) { @@ -358,7 +429,7 @@ public class CompareArchives { return switch (tileCompression) { case GZIP -> Gzip.gunzip(bytes); case NONE -> bytes; - case UNKNOWN -> throw new IllegalArgumentException("Unknown compression"); + case UNKNOWN -> throw new FatalComparisonFailure("Unknown compression"); }; } @@ -385,7 +456,10 @@ public class CompareArchives { } public record Result( - long total, long tileDiffs, Map diffTypes, - Map> diffsByLayer + long total, + long tileDiffs, + List archiveDiffs, + Map tileDiffTypes, + Map> tileDiffsByLayer ) {} } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java index 6a9c1f99..8993c902 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java @@ -12,6 +12,7 @@ import com.onthegomap.planetiler.geo.TileOrder; import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; import java.io.IOException; import java.nio.file.Path; +import java.util.List; import java.util.Map; import java.util.OptionalLong; import org.junit.jupiter.api.Test; @@ -78,7 +79,9 @@ class CompareArchivesTest { false ); assertEquals(new CompareArchives.Result( - 5, 4, Map.of( + 5, 4, List.of( + "pmtiles header" + ), Map.of( "archive 2 missing tile", 1L, "archive 1 missing tile", 2L, "different contents", 1L @@ -132,7 +135,11 @@ class CompareArchivesTest { false ); assertEquals(new CompareArchives.Result( - 5, 4, Map.of( + 5, 4, List.of( + "metadata", + "pmtiles header", + "tile compression" + ), Map.of( "archive 2 missing tile", 1L, "archive 1 missing tile", 2L, "different decompressed contents", 1L @@ -144,4 +151,42 @@ class CompareArchivesTest { ) ), result); } + + @Test + void testCompareArchivesSame() throws IOException { + var aPath = path.resolve("a.pmtiles"); + var bPath = path.resolve("b.pmtiles"); + byte[] a1 = new byte[]{0xa, 0x2}; + try ( + var a = WriteablePmtiles.newWriteToFile(aPath); + var b = WriteablePmtiles.newWriteToFile(bPath); + ) { + a.initialize(); + b.initialize(); + try ( + var aWriter = a.newTileWriter(); + var bWriter = b.newTileWriter() + ) { + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty())); + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty())); + bWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty())); + bWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty())); + } + a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config)); + b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config)); + } + var result = CompareArchives.compare( + TileArchiveConfig.from(aPath.toString()), + TileArchiveConfig.from(bPath.toString()), + config, + false + ); + assertEquals(new CompareArchives.Result( + 2, 0, List.of(), Map.of(), Map.of() + ), result); + } } diff --git a/scripts/test-release.sh b/scripts/test-release.sh index 179c75b7..b015746b 100755 --- a/scripts/test-release.sh +++ b/scripts/test-release.sh @@ -14,24 +14,27 @@ fi echo "Test java build" echo "::group::OpenMapTiles monaco (java)" -rm -f data/out.mbtiles -java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/out.mbtiles -./scripts/check-monaco.sh data/out.mbtiles +rm -f data/out*.mbtiles +# vary threads to stress-test determinism check +java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles --threads=32 +./scripts/check-monaco.sh data/jar-monaco.mbtiles echo "::endgroup::" echo "::group::Example (java)" -rm -f data/out.mbtiles -java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/out.mbtiles -./scripts/check-mbtiles.sh data/out.mbtiles +java -jar planetiler-dist/target/*with-deps.jar example-toilets --download --area=monaco --output=data/jar-example.mbtiles +./scripts/check-mbtiles.sh data/jar-example.mbtiles echo "::endgroup::" echo "::endgroup::" echo "::group::OpenMapTiles monaco (docker)" -rm -f data/out.mbtiles -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/out.mbtiles -./scripts/check-monaco.sh data/out.mbtiles +# vary threads to stress-test determinism check +docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles --threads=4 +./scripts/check-monaco.sh data/docker-monaco.mbtiles echo "::endgroup::" echo "::group::Example (docker)" -rm -f data/out.mbtiles -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/out.mbtiles -./scripts/check-mbtiles.sh data/out.mbtiles +docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" example-toilets --area=monaco --output=data/docker-example.mbtiles +./scripts/check-mbtiles.sh data/docker-example.mbtiles +echo "::endgroup::" + +echo "::group::Compare" +java -jar planetiler-dist/target/*with-deps.jar compare data/jar-monaco.mbtiles data/docker-monaco.mbtiles echo "::endgroup::"