kopia lustrzana https://github.com/onthegomap/planetiler
check bytes too
rodzic
4dced7b22c
commit
2532f0f9a6
|
@ -13,9 +13,14 @@ import com.onthegomap.planetiler.geo.GeometryType;
|
||||||
import com.onthegomap.planetiler.geo.TileCoord;
|
import com.onthegomap.planetiler.geo.TileCoord;
|
||||||
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
|
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
|
||||||
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
||||||
|
import com.onthegomap.planetiler.stats.Stats;
|
||||||
|
import com.onthegomap.planetiler.worker.Worker;
|
||||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.UncheckedIOException;
|
import java.io.UncheckedIOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -95,8 +100,11 @@ public class CompareArchives {
|
||||||
for (var entry : result.tileDiffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
|
for (var entry : result.tileDiffTypes.entrySet().stream().sorted(Map.Entry.comparingByValue()).toList()) {
|
||||||
LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
|
LOGGER.info(" {}: {}", entry.getKey(), format.integer(entry.getValue()));
|
||||||
}
|
}
|
||||||
|
for (var diffType : result.archiveDiffs) {
|
||||||
|
LOGGER.info(" {}", diffType);
|
||||||
|
}
|
||||||
LOGGER.info("Total tiles: {}", format.integer(result.total));
|
LOGGER.info("Total tiles: {}", format.integer(result.total));
|
||||||
LOGGER.info("Total diffs: {} ({} of all tiles)", format.integer(result.tileDiffs),
|
LOGGER.info("Tile diffs: {} ({} of all tiles)", format.integer(result.tileDiffs),
|
||||||
format.percent(result.tileDiffs * 1d / result.total));
|
format.percent(result.tileDiffs * 1d / result.total));
|
||||||
}
|
}
|
||||||
System.exit((result.tileDiffs > 0 || (strict && !result.archiveDiffs.isEmpty())) ? 1 : 0);
|
System.exit((result.tileDiffs > 0 || (strict && !result.archiveDiffs.isEmpty())) ? 1 : 0);
|
||||||
|
@ -227,9 +235,62 @@ public class CompareArchives {
|
||||||
.newLine()
|
.newLine()
|
||||||
.addProcessStats();
|
.addProcessStats();
|
||||||
loggers.awaitAndLog(pipeline.done(), config.logInterval());
|
loggers.awaitAndLog(pipeline.done(), config.logInterval());
|
||||||
|
if (archiveDiffs.isEmpty() && diffs.get() == 0) {
|
||||||
|
var path1 = input1.getLocalPath();
|
||||||
|
var path2 = input2.getLocalPath();
|
||||||
|
if (path1 != null && path2 != null && Files.isRegularFile(path1) && Files.isRegularFile(path2)) {
|
||||||
|
LOGGER.info("No diffs so far, comparing bytes in {} vs. {}", path1, path2);
|
||||||
|
compareFiles(path1, path2, config);
|
||||||
|
}
|
||||||
|
}
|
||||||
return new Result(total.get(), diffs.get(), archiveDiffs, diffTypes, diffsByLayer);
|
return new Result(total.get(), diffs.get(), archiveDiffs, diffTypes, diffsByLayer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void compareFiles(Path path1, Path path2, PlanetilerConfig config) {
|
||||||
|
long size = FileUtils.fileSize(path1);
|
||||||
|
if (compareArchive("archive size", size, FileUtils.fileSize(path2))) {
|
||||||
|
AtomicLong bytesRead = new AtomicLong(0);
|
||||||
|
var worker = new Worker("compare", Stats.inMemory(), 1, () -> {
|
||||||
|
byte[] bytes1 = new byte[8192];
|
||||||
|
byte[] bytes2 = new byte[8192];
|
||||||
|
long n = 0;
|
||||||
|
try (
|
||||||
|
var is1 = Files.newInputStream(path1, StandardOpenOption.READ);
|
||||||
|
var is2 = Files.newInputStream(path2, StandardOpenOption.READ)
|
||||||
|
) {
|
||||||
|
do {
|
||||||
|
int len = is1.read(bytes1);
|
||||||
|
int len2 = is2.read(bytes2, 0, len);
|
||||||
|
if (len2 != len) {
|
||||||
|
String message = "Expected to read %s bytes from %s but got %s".formatted(len, path2, len2);
|
||||||
|
archiveDiffs.add(message);
|
||||||
|
LOGGER.warn(message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int mismatch = Arrays.mismatch(bytes1, bytes2);
|
||||||
|
if (mismatch >= 0 && mismatch < len) {
|
||||||
|
archiveDiffs.add("mismatch at byte %s".formatted(mismatch + n));
|
||||||
|
LOGGER.warn("Archives mismatch ay byte {}", mismatch + n);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
n += len;
|
||||||
|
bytesRead.set(n);
|
||||||
|
} while (n < size);
|
||||||
|
}
|
||||||
|
LOGGER.info("No mismatches! Analyzed {} / {} bytes", n, size);
|
||||||
|
});
|
||||||
|
|
||||||
|
var logger = ProgressLoggers.create()
|
||||||
|
.addStorageRatePercentCounter("bytes", size, bytesRead::get, true)
|
||||||
|
.newLine()
|
||||||
|
.addThreadPoolStats("compare", worker)
|
||||||
|
.newLine()
|
||||||
|
.addProcessStats();
|
||||||
|
|
||||||
|
worker.awaitAndLog(logger, config.logInterval());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private <T> boolean compareArchive(String name, T a, T b) {
|
private <T> boolean compareArchive(String name, T a, T b) {
|
||||||
if (Objects.equals(a, b)) {
|
if (Objects.equals(a, b)) {
|
||||||
return true;
|
return true;
|
||||||
|
|
|
@ -151,4 +151,42 @@ class CompareArchivesTest {
|
||||||
)
|
)
|
||||||
), result);
|
), result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testCompareArchivesSame() throws IOException {
|
||||||
|
var aPath = path.resolve("a.pmtiles");
|
||||||
|
var bPath = path.resolve("b.pmtiles");
|
||||||
|
byte[] a1 = new byte[]{0xa, 0x2};
|
||||||
|
try (
|
||||||
|
var a = WriteablePmtiles.newWriteToFile(aPath);
|
||||||
|
var b = WriteablePmtiles.newWriteToFile(bPath);
|
||||||
|
) {
|
||||||
|
a.initialize();
|
||||||
|
b.initialize();
|
||||||
|
try (
|
||||||
|
var aWriter = a.newTileWriter();
|
||||||
|
var bWriter = b.newTileWriter()
|
||||||
|
) {
|
||||||
|
aWriter
|
||||||
|
.write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty()));
|
||||||
|
aWriter
|
||||||
|
.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty()));
|
||||||
|
bWriter
|
||||||
|
.write(new TileEncodingResult(TileOrder.HILBERT.decode(0), a1, OptionalLong.empty()));
|
||||||
|
bWriter
|
||||||
|
.write(new TileEncodingResult(TileOrder.HILBERT.decode(2), a1, OptionalLong.empty()));
|
||||||
|
}
|
||||||
|
a.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
|
||||||
|
b.finish(new TileArchiveMetadata(new Profile.NullProfile(), config));
|
||||||
|
}
|
||||||
|
var result = CompareArchives.compare(
|
||||||
|
TileArchiveConfig.from(aPath.toString()),
|
||||||
|
TileArchiveConfig.from(bPath.toString()),
|
||||||
|
config,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assertEquals(new CompareArchives.Result(
|
||||||
|
2, 0, List.of(), Map.of(), Map.of()
|
||||||
|
), result);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,8 @@ fi
|
||||||
echo "Test java build"
|
echo "Test java build"
|
||||||
echo "::group::OpenMapTiles monaco (java)"
|
echo "::group::OpenMapTiles monaco (java)"
|
||||||
rm -f data/out*.mbtiles
|
rm -f data/out*.mbtiles
|
||||||
java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles
|
# vary threads to stress-test determinism check
|
||||||
|
java -jar planetiler-dist/target/*with-deps.jar --download --area=monaco --output=data/jar-monaco.mbtiles --threads=32
|
||||||
./scripts/check-monaco.sh data/jar-monaco.mbtiles
|
./scripts/check-monaco.sh data/jar-monaco.mbtiles
|
||||||
echo "::endgroup::"
|
echo "::endgroup::"
|
||||||
echo "::group::Example (java)"
|
echo "::group::Example (java)"
|
||||||
|
@ -25,7 +26,8 @@ echo "::endgroup::"
|
||||||
|
|
||||||
echo "::endgroup::"
|
echo "::endgroup::"
|
||||||
echo "::group::OpenMapTiles monaco (docker)"
|
echo "::group::OpenMapTiles monaco (docker)"
|
||||||
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles
|
# vary threads to stress-test determinism check
|
||||||
|
docker run -v "$(pwd)/data":/data ghcr.io/onthegomap/planetiler:"${version}" --area=monaco --output=data/docker-monaco.mbtiles --threads=4
|
||||||
./scripts/check-monaco.sh data/docker-monaco.mbtiles
|
./scripts/check-monaco.sh data/docker-monaco.mbtiles
|
||||||
echo "::endgroup::"
|
echo "::endgroup::"
|
||||||
echo "::group::Example (docker)"
|
echo "::group::Example (docker)"
|
||||||
|
|
Ładowanie…
Reference in New Issue