diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java index 6c39408e..76e1147f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CompareArchives.java @@ -13,9 +13,14 @@ import com.onthegomap.planetiler.geo.GeometryType; import com.onthegomap.planetiler.geo.TileCoord; import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.stats.ProgressLoggers; +import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.worker.Worker; import com.onthegomap.planetiler.worker.WorkerPipeline; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -95,8 +100,11 @@ public class CompareArchives { for (var entry : result.tileDiffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) { LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue())); } + for (var diffType : result.archiveDiffs) { + LOGGER.info(" {}", diffType); + } LOGGER.info("Total tiles: {}", format.integer(result.total)); - LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs), + LOGGER.info("Tile diffs: {} ({} of all tiles)", format.integer(result.tileDiffs), format.percent(result.tileDiffs * 1d / result.total)); } System.exit((result.tileDiffs > 0 || (strict && !result.archiveDiffs.isEmpty())) ? 1 : 0); @@ -227,9 +235,62 @@ public class CompareArchives { .newLine() .addProcessStats(); loggers.awaitAndLog(pipeline.done(), config.logInterval()); + if (archiveDiffs.isEmpty() && diffs.get() == 0) { + var path1 = input1.getLocalPath(); + var path2 = input2.getLocalPath(); + if (path1 != null && path2 != null && Files.isRegularFile(path1) && Files.isRegularFile(path2)) { + LOGGER.info("No diffs so far, comparing bytes in {} vs. {}", path1, path2); + compareFiles(path1, path2, config); + } + } return new Result(total.get(), diffs.get(), archiveDiffs, diffTypes, diffsByLayer); } + private void compareFiles(Path path1, Path path2, PlanetilerConfig config) { + long size = FileUtils.fileSize(path1); + if (compareArchive("archive size", size, FileUtils.fileSize(path2))) { + AtomicLong bytesRead = new AtomicLong(0); + var worker = new Worker("compare", Stats.inMemory(), 1, () -> { + byte[] bytes1 = new byte[8192]; + byte[] bytes2 = new byte[8192]; + long n = 0; + try ( + var is1 = Files.newInputStream(path1, StandardOpenOption.READ); + var is2 = Files.newInputStream(path2, StandardOpenOption.READ) + ) { + do { + int len = is1.read(bytes1); + int len2 = is2.read(bytes2, 0, len); + if (len2 != len) { + String message = "Expected to read %s bytes from %s but got %s".formatted(len, path2, len2); + archiveDiffs.add(message); + LOGGER.warn(message); + return; + } + int mismatch = Arrays.mismatch(bytes1, bytes2); + if (mismatch >= 0 && mismatch < len) { + archiveDiffs.add("mismatch at byte %s".formatted(mismatch + n)); + LOGGER.warn("Archives mismatch ay byte {}", mismatch + n); + return; + } + n += len; + bytesRead.set(n); + } while (n < size); + } + LOGGER.info("No mismatches! Analyzed {} / {} bytes", n, size); + }); + + var logger = ProgressLoggers.create() + .addStorageRatePercentCounter("bytes", size, bytesRead::get, true) + .newLine() + .addThreadPoolStats("compare", worker) + .newLine() + .addProcessStats(); + + worker.awaitAndLog(logger, config.logInterval()); + } + } + private boolean compareArchive(String name, T a, T b) { if (Objects.equals(a, b)) { return true; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java index 542f6a78..8993c902 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CompareArchivesTest.java @@ -151,4 +151,42 @@ class CompareArchivesTest { ) ), result); } + + @Test + void testCompareArchivesSame() throws IOException { + var aPath = path.resolve("a.pmtiles"); + var bPath = path.resolve("b.pmtiles"); + byte[] a1 = new byte[]{0xa, 0x2}; + try ( + var a = WriteablePmtiles.newWriteToFile(aPath); + var b = WriteablePmtiles.newWriteToFile(bPath); + ) { + a.initialize(); + b.initialize(); + try ( + var aWriter = a.newTileWriter(); + var bWriter = b.newTileWriter() + ) { + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty())); + aWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty())); + bWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty())); + bWriter + .write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty())); + } + a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config)); + b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config)); + } + var result = CompareArchives.compare( + TileArchiveConfig.from(aPath.toString()), + TileArchiveConfig.from(bPath.toString()), + config, + false + ); + assertEquals(new CompareArchives.Result( + 2, 0, List.of(), Map.of(), Map.of() + ), result); + } } diff --git a/scripts/test-release.sh b/scripts/test-release.sh index b951cdf1..b015746b 100755 --- a/scripts/test-release.sh +++ b/scripts/test-release.sh @@ -15,7 +15,8 @@ fi echo "Test java build" echo "::group::OpenMapTiles monaco (java)" rm -f data/out*.mbtiles -java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles +# vary threads to stress-test determinism check +java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles --threads=32 ./scripts/check-monaco.sh data/jar-monaco.mbtiles echo "::endgroup::" echo "::group::Example (java)" @@ -25,7 +26,8 @@ echo "::endgroup::" echo "::endgroup::" echo "::group::OpenMapTiles monaco (docker)" -docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles +# vary threads to stress-test determinism check +docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles --threads=4 ./scripts/check-monaco.sh data/docker-monaco.mbtiles echo "::endgroup::" echo "::group::Example (docker)"