From 264b3515c452fa8bccc21846908ae3650c4152b1 Mon Sep 17 00:00:00 2001 From: bbilger Date: Sat, 6 Jan 2024 20:58:49 +0100 Subject: [PATCH 1/3] add tile-copy utility to copy tiles from one archive into another e.g. mbtiles to files, pmtiles to mbtiles, ... --- .../com/onthegomap/planetiler/Planetiler.java | 34 +-- .../planetiler/archive/TileArchiveConfig.java | 41 ++- .../archive/TileArchiveMetadata.java | 5 + .../planetiler/archive/TileArchiveWriter.java | 10 +- .../planetiler/archive/TileArchives.java | 43 +++- .../planetiler/archive/TileCompression.java | 5 - .../planetiler/archive/TileCopy.java | 236 ++++++++++++++++++ .../archive/TileEncodingResult.java | 6 +- .../planetiler/config/CommonConfigs.java | 31 +++ .../planetiler/config/PlanetilerConfig.java | 15 +- .../files/WriteableFilesArchive.java | 11 +- .../planetiler/mbtiles/Mbtiles.java | 7 +- .../onthegomap/planetiler/mbtiles/Verify.java | 7 +- .../planetiler/pmtiles/WriteablePmtiles.java | 9 +- .../planetiler/stream/CsvBinaryEncoding.java | 50 ++++ .../stream/JsonStreamArchiveEntry.java | 46 ++++ .../planetiler/stream/ReadableCsvArchive.java | 108 ++++++++ .../stream/ReadableJsonStreamArchive.java | 82 ++++++ .../stream/ReadableProtoStreamArchive.java | 160 ++++++++++++ .../stream/ReadableStreamArchive.java | 94 +++++++ .../stream/StreamArchiveConfig.java | 8 +- .../planetiler/stream/StreamArchiveUtils.java | 50 ++++ .../stream/WriteableCsvArchive.java | 58 +---- .../stream/WriteableJsonStreamArchive.java | 92 +------ .../stream/WriteableProtoStreamArchive.java | 12 +- .../planetiler/util/CloseableIterator.java | 40 +++ .../com/onthegomap/planetiler/util/Gzip.java | 26 +- .../onthegomap/planetiler/util/Hashing.java | 3 + .../src/main/proto/stream_archive_proto.proto | 16 +- .../planetiler/PlanetilerTests.java | 14 +- .../com/onthegomap/planetiler/TestUtils.java | 2 +- .../planetiler/archive/TileCopyTest.java | 200 +++++++++++++++ .../stream/InMemoryStreamArchive.java | 119 --------- .../stream/ReadableCsvStreamArchiveTest.java | 75 ++++++ .../stream/ReadableJsonStreamArchiveTest.java | 48 ++++ .../ReadableProtoStreamArchiveTest.java | 60 +++++ .../stream/WriteableCsvArchiveTest.java | 6 +- .../WriteableProtoStreamArchiveTest.java | 20 +- .../util/CloseableIteratorTest.java | 36 +++ .../java/com/onthegomap/planetiler/Main.java | 5 +- 40 files changed, 1512 insertions(+), 378 deletions(-) create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java delete mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java create mode 100644 planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java index b2503d76..09ac706f 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java @@ -41,7 +41,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Function; -import java.util.stream.IntStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -656,18 +655,8 @@ public class Planetiler { System.exit(0); } else if (onlyDownloadSources) { // don't check files if not generating map - } else if (config.append()) { - if (!output.format().supportsAppend()) { - throw new IllegalArgumentException("cannot append to " + output.format().id()); - } - if (!output.exists()) { - throw new IllegalArgumentException(output.uri() + " must exist when appending"); - } - } else if (overwrite || config.force()) { - output.delete(); - } else if (output.exists()) { - throw new IllegalArgumentException( - output.uri() + " already exists, use the --force argument to overwrite or --append."); + } else { + output.setup(config.force() || overwrite, config.append(), config.tileWriteThreads()); } Path layerStatsPath = arguments.file("layer_stats", "layer stats output path", @@ -677,23 +666,6 @@ public class Planetiler { if (config.tileWriteThreads() < 1) { throw new IllegalArgumentException("require tile_write_threads >= 1"); } - if (config.tileWriteThreads() > 1) { - if (!output.format().supportsConcurrentWrites()) { - throw new IllegalArgumentException(output.format() + " doesn't support concurrent writes"); - } - IntStream.range(1, config.tileWriteThreads()) - .mapToObj(output::getPathForMultiThreadedWriter) - .forEach(p -> { - if (!config.append() && (overwrite || config.force())) { - FileUtils.delete(p); - } - if (config.append() && !output.exists(p)) { - throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending"); - } else if (!config.append() && output.exists(p)) { - throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending"); - } - }); - } LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri()); @@ -718,7 +690,7 @@ public class Planetiler { // in case any temp files are left from a previous run... FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath); Files.createDirectories(tmpDir); - FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath()); + FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath); if (!toDownload.isEmpty()) { download(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java index d4bb6118..0e9f8f39 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -16,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.stream.IntStream; import java.util.stream.Stream; /** @@ -158,7 +159,7 @@ public record TileArchiveConfig( /** * Returns the local base path for this archive, for which directories should be pre-created for. */ - public Path getLocalBasePath() { + Path getLocalBasePath() { Path p = getLocalPath(); if (format() == Format.FILES) { p = FilesArchiveUtils.cleanBasePath(p); @@ -166,7 +167,6 @@ public record TileArchiveConfig( return p; } - /** * Deletes the archive if possible. */ @@ -187,7 +187,7 @@ public record TileArchiveConfig( * @param p path to the archive * @return {@code true} if the archive already exists, {@code false} otherwise. */ - public boolean exists(Path p) { + private boolean exists(Path p) { if (p == null) { return false; } @@ -229,6 +229,41 @@ public record TileArchiveConfig( }; } + public void setup(boolean force, boolean append, int tileWriteThreads) { + if (append) { + if (!format().supportsAppend()) { + throw new IllegalArgumentException("cannot append to " + format().id()); + } + if (!exists()) { + throw new IllegalArgumentException(uri() + " must exist when appending"); + } + } else if (force) { + delete(); + } else if (exists()) { + throw new IllegalArgumentException(uri() + " already exists, use the --force argument to overwrite or --append."); + } + + if (tileWriteThreads > 1) { + if (!format().supportsConcurrentWrites()) { + throw new IllegalArgumentException(format() + " doesn't support concurrent writes"); + } + IntStream.range(1, tileWriteThreads) + .mapToObj(this::getPathForMultiThreadedWriter) + .forEach(p -> { + if (!append && force) { + FileUtils.delete(p); + } + if (append && !exists(p)) { + throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending"); + } else if (!append && exists(p)) { + throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending"); + } + }); + } + + FileUtils.createParentDirectories(getLocalBasePath()); + } + public enum Format { MBTILES("mbtiles", false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */, diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java index fc0d4eca..eeb3f0d0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java @@ -170,6 +170,11 @@ public record TileArchiveMetadata( maxzoom, json, others, tileCompression); } + public TileArchiveMetadata withTileCompression(TileCompression tileCompression) { + return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, minzoom, + maxzoom, json, others, tileCompression); + } + /* * few workarounds to make collect unknown fields to others work, * because @JsonAnySetter does not yet work on constructor/creator arguments diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index bba4bd63..b6f7d403 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -261,7 +261,7 @@ public class TileArchiveWriter { * To optimize emitting many identical consecutive tiles (like large ocean areas), memoize output to avoid * recomputing if the input hasn't changed. */ - byte[] lastBytes = null, lastEncoded = null; + byte[] lastBytes = null; Long lastTileDataHash = null; boolean lastIsFill = false; List lastLayerStats = null; @@ -276,24 +276,22 @@ public class TileArchiveWriter { for (int i = 0; i < batch.in.size(); i++) { FeatureGroup.TileFeatures tileFeatures = batch.in.get(i); featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed()); - byte[] bytes, encoded; + byte[] bytes; List layerStats; Long tileDataHash; if (tileFeatures.hasSameContents(last)) { bytes = lastBytes; - encoded = lastEncoded; tileDataHash = lastTileDataHash; layerStats = lastLayerStats; memoizedTiles.inc(); } else { VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater); if (skipFilled && (lastIsFill = tile.containsOnlyFills())) { - encoded = null; layerStats = null; bytes = null; } else { var proto = tile.toProto(); - encoded = proto.toByteArray(); + var encoded = proto.toByteArray(); bytes = switch (config.tileCompression()) { case GZIP -> gzip(encoded); case NONE -> encoded; @@ -307,7 +305,6 @@ public class TileArchiveWriter { } } lastLayerStats = layerStats; - lastEncoded = encoded; lastBytes = bytes; last = tileFeatures; if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) { @@ -326,7 +323,6 @@ public class TileArchiveWriter { new TileEncodingResult( tileFeatures.tileCoord(), bytes, - encoded.length, tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash), layerStatsRows ) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java index 8ee03b4f..638922e3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java @@ -1,17 +1,23 @@ package com.onthegomap.planetiler.archive; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.CommonConfigs; import com.onthegomap.planetiler.config.PlanetilerConfig; import com.onthegomap.planetiler.files.ReadableFilesArchive; import com.onthegomap.planetiler.files.WriteableFilesArchive; import com.onthegomap.planetiler.mbtiles.Mbtiles; import com.onthegomap.planetiler.pmtiles.ReadablePmtiles; import com.onthegomap.planetiler.pmtiles.WriteablePmtiles; +import com.onthegomap.planetiler.stream.ReadableCsvArchive; +import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive; +import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive; import com.onthegomap.planetiler.stream.StreamArchiveConfig; import com.onthegomap.planetiler.stream.WriteableCsvArchive; import com.onthegomap.planetiler.stream.WriteableJsonStreamArchive; import com.onthegomap.planetiler.stream.WriteableProtoStreamArchive; import java.io.IOException; import java.nio.file.Path; +import java.util.function.Supplier; /** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */ public class TileArchives { @@ -37,45 +43,58 @@ public class TileArchives { return newReader(TileArchiveConfig.from(archive), config); } + public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config) + throws IOException { + return newWriter(archive, config.arguments()); + } + /** * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}. * * @throws IOException if an error occurs creating the resource. */ - public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config) + public static WriteableTileArchive newWriter(TileArchiveConfig archive, Arguments baseArguments) throws IOException { - var options = archive.applyFallbacks(config.arguments()); + var options = archive.applyFallbacks(baseArguments); var format = archive.format(); return switch (format) { case MBTILES -> // pass-through legacy arguments for fallback - Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments() + Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(baseArguments .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION))); case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath()); case CSV, TSV -> WriteableCsvArchive.newWriteToFile(format, archive.getLocalPath(), - new StreamArchiveConfig(config, options)); + new StreamArchiveConfig(baseArguments, options)); case PROTO, PBF -> WriteableProtoStreamArchive.newWriteToFile(archive.getLocalPath(), - new StreamArchiveConfig(config, options)); + new StreamArchiveConfig(baseArguments, options)); case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(), - new StreamArchiveConfig(config, options)); - case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, config.force() || config.append()); + new StreamArchiveConfig(baseArguments, options)); + case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, + CommonConfigs.appendToArchive(baseArguments) || CommonConfigs.force(baseArguments)); }; } + public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config) + throws IOException { + return newReader(archive, config.arguments()); + } + /** * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}. * * @throws IOException if an error occurs opening the resource. */ - public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config) + public static ReadableTileArchive newReader(TileArchiveConfig archive, Arguments baseArguments) throws IOException { - var options = archive.applyFallbacks(config.arguments()); + var options = archive.applyFallbacks(baseArguments); + Supplier streamArchiveConfig = () -> new StreamArchiveConfig(baseArguments, options); return switch (archive.format()) { case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options); case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath()); - case CSV, TSV -> throw new UnsupportedOperationException("reading CSV is not supported"); - case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported"); - case JSON -> throw new UnsupportedOperationException("reading JSON is not supported"); + case CSV, TSV -> + ReadableCsvArchive.newReader(archive.format(), archive.getLocalPath(), streamArchiveConfig.get()); + case PROTO, PBF -> ReadableProtoStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get()); + case JSON -> ReadableJsonStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get()); case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath(), options); }; } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java index 2e8b5bd7..4123c105 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java @@ -54,10 +54,5 @@ public enum TileCompression { public TileCompression deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { return findById(p.getValueAsString()).orElse(TileCompression.UNKNOWN); } - - @Override - public TileCompression getNullValue(DeserializationContext ctxt) { - return TileCompression.GZIP; - } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java new file mode 100644 index 00000000..03d8a139 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java @@ -0,0 +1,236 @@ +package com.onthegomap.planetiler.archive; + +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.CommonConfigs; +import com.onthegomap.planetiler.stats.Counter; +import com.onthegomap.planetiler.stats.ProcessInfo; +import com.onthegomap.planetiler.stats.ProgressLoggers; +import com.onthegomap.planetiler.stats.Stats; +import com.onthegomap.planetiler.util.Gzip; +import com.onthegomap.planetiler.util.Hashing; +import com.onthegomap.planetiler.worker.WorkerPipeline; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.function.Function; +import java.util.function.UnaryOperator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility to copy/convert tiles and metadata from one archive into another. + *

+ * Example usages: + * + * 

+ * --input=tiles.mbtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
+ * --input=tiles.pmtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles/
+ * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
+ * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
+ * --input=tiles.mbtiles --output=tiles.proto
+ *
+ */ +public class TileCopy { + + private static final Logger LOGGER = LoggerFactory.getLogger(TileCopy.class); + + private final TileCopyConfig config; + + private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter(); + + + TileCopy(TileCopyConfig config) { + this.config = config; + } + + public void run() throws IOException { + + if (!config.inArchive().exists()) { + throw new IllegalArgumentException("the input archive does not exist"); + } + + config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads()); + + final var loggers = ProgressLoggers.create() + .addRateCounter("tiles", tilesWrittenOverall::get); + + try ( + var reader = TileArchives.newReader(config.inArchive(), config.inArguments()); + var writer = TileArchives.newWriter(config.outArchive(), config.outArguments()) + ) { + + final TileArchiveMetadata inMetadata = getInMetadata(reader); + final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata); + + writer.initialize(); + try ( + var rawTiles = reader.getAllTiles(); + var it = config.skipEmpty() ? rawTiles.filter(t -> t.bytes() != null && t.bytes().length > 0) : rawTiles + ) { + + final var tileConverter = tileConverter(inMetadata.tileCompression(), outMetadata.tileCompression(), writer); + var pipeline = WorkerPipeline.start("archive", config.stats()) + .readFrom("tiles", () -> it) + .addBuffer("buffer", config.queueSize()) + .sinkTo("write", config.tileWriterThreads(), itt -> tileWriter(writer, itt, tileConverter)); + + final var f = pipeline.done().thenRun(() -> writer.finish(outMetadata)); + + loggers.awaitAndLog(f, config.logInterval()); + } + } + } + + private void tileWriter(WriteableTileArchive archive, Iterable itt, + Function tileConverter) { + + final Counter tilesWritten = tilesWrittenOverall.counterForThread(); + try (var tileWriter = archive.newTileWriter()) { + for (Tile t : itt) { + tileWriter.write(tileConverter.apply(t)); + tilesWritten.inc(); + } + } + } + + private static Function tileConverter(TileCompression inCompression, + TileCompression outCompression, WriteableTileArchive writer) { + + final UnaryOperator bytesReEncoder = bytesReEncoder(inCompression, outCompression); + final Function hasher = + writer.deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) : + b -> OptionalLong.empty(); + + return t -> new TileEncodingResult(t.coord(), bytesReEncoder.apply(t.bytes()), hasher.apply(t.bytes())); + } + + private static UnaryOperator bytesReEncoder(TileCompression inCompression, TileCompression outCompression) { + if (inCompression == outCompression) { + return UnaryOperator.identity(); + } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) { + return Gzip::gunzip; + } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) { + return Gzip::gzip; + } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.GZIP) { + return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b); + } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.NONE) { + return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b; + } else { + throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression); + } + } + + private TileArchiveMetadata getInMetadata(ReadableTileArchive reader) { + TileArchiveMetadata inMetadata = config.inMetadata(); + if (inMetadata == null) { + inMetadata = reader.metadata(); + if (inMetadata == null) { + LOGGER.atWarn() + .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata"); + inMetadata = fallbackMetadata(); + } + } + if (inMetadata.tileCompression() == null) { + inMetadata = inMetadata.withTileCompression(config.inCompression()); + } + + return inMetadata; + } + + private TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata) { + if (config.outCompression() == TileCompression.UNKNWON && inMetadata.tileCompression() == TileCompression.UNKNWON) { + return inMetadata.withTileCompression(TileCompression.GZIP); + } else if (config.outCompression() != TileCompression.UNKNWON) { + return inMetadata.withTileCompression(config.outCompression()); + } else { + return inMetadata; + } + } + + private static TileArchiveMetadata fallbackMetadata() { + return new TileArchiveMetadata( + "unknown", + null, + null, + null, + null, + TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf + null, + null, + null, + null, + new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers + Map.of(), + null + ); + } + + record TileCopyConfig( + TileArchiveConfig inArchive, + TileArchiveConfig outArchive, + Arguments inArguments, + Arguments outArguments, + TileCompression inCompression, + TileCompression outCompression, + int tileWriterThreads, + Duration logInterval, + Stats stats, + int queueSize, + boolean append, + boolean force, + TileArchiveMetadata inMetadata, + boolean skipEmpty + ) { + static TileCopyConfig fromArguments(Arguments baseArguments) { + + final Arguments inArguments = baseArguments.withPrefix("in"); + final Arguments outArguments = baseArguments.withPrefix("out"); + final Arguments baseOrOutArguments = outArguments.orElse(baseArguments); + + final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null); + final TileArchiveMetadata inMetadata; + if (inMetadataPath != null) { + try { + inMetadata = + TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + inMetadata = null; + } + + return new TileCopyConfig( + TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")), + TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")), + inArguments, + outArguments, + getTileCompressionArg(inArguments, "the input tile compression"), + getTileCompressionArg(outArguments, "the output tile compression"), + CommonConfigs.tileWriterThreads(baseOrOutArguments), + CommonConfigs.logInterval(baseArguments), + baseArguments.getStats(), + Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)), + CommonConfigs.appendToArchive(baseOrOutArguments), + CommonConfigs.force(baseOrOutArguments), + inMetadata, + baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", false) + ); + } + } + + private static TileCompression getTileCompressionArg(Arguments args, String description) { + return args.getObject("tile_compression", description, TileCompression.UNKNWON, + v -> TileCompression.findById(v).orElseThrow()); + } + + public static void main(String[] args) throws IOException { + new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run(); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java index 8716c214..c6f58896 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java @@ -5,12 +5,10 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.OptionalLong; -import javax.annotation.Nonnull; public record TileEncodingResult( TileCoord coord, - @Nonnull byte[] tileData, - int rawTileSize, + byte[] tileData, /* will always be empty in non-compact mode and might also be empty in compact mode */ OptionalLong tileDataHash, List layerStats @@ -20,7 +18,7 @@ public record TileEncodingResult( byte[] tileData, OptionalLong tileDataHash ) { - this(coord, tileData, tileData.length, tileDataHash, List.of()); + this(coord, tileData, tileDataHash, List.of()); } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java new file mode 100644 index 00000000..1e30db59 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java @@ -0,0 +1,31 @@ +package com.onthegomap.planetiler.config; + +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import java.time.Duration; +import java.util.stream.Stream; + +public final class CommonConfigs { + private CommonConfigs() {} + + public static boolean force(Arguments arguments) { + return arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false); + } + + public static boolean appendToArchive(Arguments arguments) { + return arguments.getBoolean("append", + "append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values()) + .filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(), + false); + } + + public static int tileWriterThreads(Arguments arguments) { + return arguments.getInteger("tile_write_threads", + "number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values()) + .filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(), + 1); + } + + public static Duration logInterval(Arguments arguments) { + return arguments.getDuration("loginterval", "time between logs", "10s"); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java index 066afbc8..321bd282 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java @@ -1,6 +1,5 @@ package com.onthegomap.planetiler.config; -import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.archive.TileCompression; import com.onthegomap.planetiler.collection.LongLongMap; import com.onthegomap.planetiler.collection.Storage; @@ -132,19 +131,13 @@ public record PlanetilerConfig( featureProcessThreads, arguments.getInteger("feature_read_threads", "number of threads to use when reading features at tile write time", threads < 32 ? 1 : 2), - arguments.getInteger("tile_write_threads", - "number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values()) - .filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(), - 1), - arguments.getDuration("loginterval", "time between logs", "10s"), + CommonConfigs.tileWriterThreads(arguments), + CommonConfigs.logInterval(arguments), minzoom, maxzoom, renderMaxzoom, - arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false), - arguments.getBoolean("append", - "append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values()) - .filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(), - false), + CommonConfigs.force(arguments), + CommonConfigs.appendToArchive(arguments), arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false), arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true), arguments.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks", diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java index ee4d70c0..b8b9c47b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java @@ -159,12 +159,17 @@ public class WriteableFilesArchive implements WriteableTileArchive { } lastCheckedFolder = folder; try { - Files.write(file, data); + if (data == null) { + Files.createFile(file); + } else { + Files.write(file, data); + } } catch (IOException e) { throw new UncheckedIOException(e); } - - bytesWritten.incBy(data.length); + if (data != null) { + bytesWritten.incBy(data.length); + } } @Override diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java index e68e1f7f..e1e9449a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java @@ -6,6 +6,7 @@ import com.onthegomap.planetiler.archive.ReadableTileArchive; import com.onthegomap.planetiler.archive.Tile; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer; +import com.onthegomap.planetiler.archive.TileCompression; import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.archive.WriteableTileArchive; import com.onthegomap.planetiler.config.Arguments; @@ -847,7 +848,11 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive */ public TileArchiveMetadata get() { Map map = new HashMap<>(getAll()); - return TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class); + var metadata = TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class); + if (metadata.tileCompression() == null) { + metadata = metadata.withTileCompression(TileCompression.GZIP); + } + return metadata; } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java index 99a43f15..d0a18ed7 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java @@ -6,7 +6,6 @@ import com.onthegomap.planetiler.VectorTile; import com.onthegomap.planetiler.geo.GeometryException; import com.onthegomap.planetiler.geo.TileCoord; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -92,11 +91,7 @@ public class Verify { } private static List decode(byte[] zipped) { - try { - return VectorTile.decode(gunzip(zipped)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + return VectorTile.decode(gunzip(zipped)); } /** diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java index 795a50f0..e95b1d7d 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java @@ -57,8 +57,7 @@ public final class WriteablePmtiles implements WriteableTileArchive { this.bytesWritten = bytesWritten; } - private static Directories makeDirectoriesWithLeaves(List subEntries, int leafSize, int attemptNum) - throws IOException { + private static Directories makeDirectoriesWithLeaves(List subEntries, int leafSize, int attemptNum) { LOGGER.info("Building directories with {} entries per leaf, attempt {}...", leafSize, attemptNum); ArrayList rootEntries = new ArrayList<>(); ByteArrayList leavesOutputStream = new ByteArrayList(); @@ -91,9 +90,8 @@ public final class WriteablePmtiles implements WriteableTileArchive { * * @param entries a sorted ObjectArrayList of all entries in the tileset. * @return byte arrays of the root and all leaf directories, and the # of leaves. - * @throws IOException if compression fails */ - static Directories makeDirectories(List entries) throws IOException { + static Directories makeDirectories(List entries) { int maxEntriesRootOnly = 16384; int attemptNum = 1; if (entries.size() < maxEntriesRootOnly) { @@ -302,6 +300,9 @@ public final class WriteablePmtiles implements WriteableTileArchive { long offset; OptionalLong tileDataHashOpt = encodingResult.tileDataHash(); var data = encodingResult.tileData(); + if (data == null) { + return; + } TileCoord coord = encodingResult.coord(); long tileId = coord.hilbertEncoded(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java new file mode 100644 index 00000000..506aa02d --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java @@ -0,0 +1,50 @@ +package com.onthegomap.planetiler.stream; + +import com.google.common.base.Suppliers; +import java.util.Base64; +import java.util.HexFormat; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; + +enum CsvBinaryEncoding { + + BASE64("base64", () -> Base64.getEncoder()::encodeToString, () -> Base64.getDecoder()::decode), + HEX("hex", () -> HexFormat.of()::formatHex, () -> HexFormat.of()::parseHex); + + private final String id; + private final Supplier> encoder; + private final Supplier> decoder; + + private CsvBinaryEncoding(String id, Supplier> encoder, + Supplier> decoder) { + this.id = id; + this.encoder = Suppliers.memoize(encoder::get); + this.decoder = Suppliers.memoize(decoder::get); + } + + String encode(byte[] b) { + return encoder.get().apply(b); + } + + byte[] decode(String s) { + return decoder.get().apply(s); + } + + static List ids() { + return Stream.of(CsvBinaryEncoding.values()).map(CsvBinaryEncoding::id).toList(); + } + + static CsvBinaryEncoding fromId(String id) { + return Stream.of(CsvBinaryEncoding.values()) + .filter(de -> de.id().equals(id)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "unexpected binary encoding - expected one of " + ids() + " but got " + id)); + } + + String id() { + return id; + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java new file mode 100644 index 00000000..2f0808f7 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java @@ -0,0 +1,46 @@ +package com.onthegomap.planetiler.stream; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import java.util.Arrays; +import java.util.Objects; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = JsonStreamArchiveEntry.TileEntry.class, name = "tile"), + @JsonSubTypes.Type(value = JsonStreamArchiveEntry.InitializationEntry.class, name = "initialization"), + @JsonSubTypes.Type(value = JsonStreamArchiveEntry.FinishEntry.class, name = "finish") +}) +sealed interface JsonStreamArchiveEntry { + record TileEntry(int x, int y, int z, byte[] encodedData) implements JsonStreamArchiveEntry { + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(encodedData); + result = prime * result + Objects.hash(x, y, z); + return result; + } + + @Override + public boolean equals(Object obj) { + return this == obj || (obj instanceof JsonStreamArchiveEntry.TileEntry tileEntry && + Arrays.equals(encodedData, tileEntry.encodedData) && x == tileEntry.x && y == tileEntry.y && z == tileEntry.z); + } + + @Override + public String toString() { + return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]"; + } + } + + record InitializationEntry() implements JsonStreamArchiveEntry {} + + + record FinishEntry(TileArchiveMetadata metadata) implements JsonStreamArchiveEntry {} +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java new file mode 100644 index 00000000..f6df44e4 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java @@ -0,0 +1,108 @@ +package com.onthegomap.planetiler.stream; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Scanner; +import java.util.function.Function; +import java.util.regex.Pattern; + +/** + * Reads tiles from a CSV file. Counterpart to {@link WriteableCsvArchive}. + *

+ * Supported arguments: + *

+ *
column_separator
+ *
The column separator e.g. ",", ";", "\t"
+ *
line_separator
+ *
The line separator e.g. "\n", "\r", "\r\n"
+ *
+ * + * @see WriteableCsvArchive + */ +public class ReadableCsvArchive extends ReadableStreamArchive { + + private final Pattern columnSeparatorPattern; + private final Pattern lineSeparatorPattern; + private final Function tileDataDecoder; + + private ReadableCsvArchive(TileArchiveConfig.Format format, Path basePath, StreamArchiveConfig config) { + super(basePath, config); + this.columnSeparatorPattern = + Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format))); + this.lineSeparatorPattern = + Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format))); + final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions()); + this.tileDataDecoder = binaryEncoding::decode; + } + + public static ReadableCsvArchive newReader(TileArchiveConfig.Format format, Path basePath, + StreamArchiveConfig config) { + return new ReadableCsvArchive(format, basePath, config); + } + + @Override + CloseableIterator createIterator() { + try { + @SuppressWarnings("java:S2095") final Scanner s = + new Scanner(basePath.toFile()).useDelimiter(lineSeparatorPattern); + return new CloseableIterator<>() { + @Override + public void close() { + s.close(); + } + + @Override + public boolean hasNext() { + return s.hasNext(); + } + + @Override + public String next() { + return s.next(); + } + }; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + } + + @Override + Optional mapEntryToTile(String entry) { + final String[] splits = columnSeparatorPattern.split(entry); + final byte[] bytes; + if (splits.length == 4) { + bytes = tileDataDecoder.apply(splits[3].strip()); + } else if (splits.length == 3) { + bytes = null; + } else { + throw new InvalidCsvFormat(entry.length() > 20 ? entry.substring(0, 20) + "..." : entry); + } + return Optional.of(new Tile( + TileCoord.ofXYZ( + Integer.parseInt(splits[0].strip()), + Integer.parseInt(splits[1].strip()), + Integer.parseInt(splits[2].strip()) + ), + bytes + )); + } + + @Override + Optional mapEntryToMetadata(String entry) { + return Optional.empty(); + } + + static class InvalidCsvFormat extends RuntimeException { + InvalidCsvFormat(String message) { + super(message); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java new file mode 100644 index 00000000..640adf0b --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java @@ -0,0 +1,82 @@ +package com.onthegomap.planetiler.stream; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +/** + * Reads tiles and metadata from a delimited JSON file. Counterpart to {@link WriteableJsonStreamArchive}. + * + * @see WriteableJsonStreamArchive + */ +public class ReadableJsonStreamArchive extends ReadableStreamArchive { + + private ReadableJsonStreamArchive(Path basePath, StreamArchiveConfig config) { + super(basePath, config); + } + + public static ReadableJsonStreamArchive newReader(Path basePath, StreamArchiveConfig config) { + return new ReadableJsonStreamArchive(basePath, config); + } + + @Override + CloseableIterator createIterator() { + BufferedReader reader = null; + try { + reader = Files.newBufferedReader(basePath); + final var readerFinal = reader; + final var it = StreamArchiveUtils.jsonMapperJsonStreamArchive + .readerFor(JsonStreamArchiveEntry.class) + .readValues(readerFinal); + return new CloseableIterator<>() { + @Override + public void close() { + try { + readerFinal.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public JsonStreamArchiveEntry next() { + return it.next(); + } + }; + } catch (IOException e) { + closeSilentlyOnError(reader); + throw new UncheckedIOException(e); + } + } + + @Override + Optional mapEntryToTile(JsonStreamArchiveEntry entry) { + if (entry instanceof JsonStreamArchiveEntry.TileEntry tileEntry) { + return Optional.of(new Tile( + TileCoord.ofXYZ(tileEntry.x(), tileEntry.y(), tileEntry.z()), + tileEntry.encodedData() + )); + } + return Optional.empty(); + } + + @Override + Optional mapEntryToMetadata(JsonStreamArchiveEntry entry) { + if (entry instanceof JsonStreamArchiveEntry.FinishEntry finishEntry) { + return Optional.ofNullable(finishEntry.metadata()); + } + return Optional.empty(); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java new file mode 100644 index 00000000..98fb9681 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java @@ -0,0 +1,160 @@ +package com.onthegomap.planetiler.stream; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.proto.StreamArchiveProto; +import com.onthegomap.planetiler.util.CloseableIterator; +import com.onthegomap.planetiler.util.LayerAttrStats; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Envelope; + +/** + * Reads tiles and metadata from a delimited protobuf file. Counterpart to {@link WriteableProtoStreamArchive}. + * + * @see WriteableProtoStreamArchive + */ +public class ReadableProtoStreamArchive extends ReadableStreamArchive { + + private ReadableProtoStreamArchive(Path basePath, StreamArchiveConfig config) { + super(basePath, config); + } + + public static ReadableProtoStreamArchive newReader(Path basePath, StreamArchiveConfig config) { + return new ReadableProtoStreamArchive(basePath, config); + } + + @Override + CloseableIterator createIterator() { + try { + @SuppressWarnings("java:S2095") var in = new FileInputStream(basePath.toFile()); + return new CloseableIterator<>() { + private StreamArchiveProto.Entry nextValue; + + @Override + public void close() { + closeUnchecked(in); + } + + @Override + public boolean hasNext() { + if (nextValue != null) { + return true; + } + try { + nextValue = StreamArchiveProto.Entry.parseDelimitedFrom(in); + return nextValue != null; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public StreamArchiveProto.Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final StreamArchiveProto.Entry returnValue = nextValue; + nextValue = null; + return returnValue; + } + }; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + Optional mapEntryToTile(StreamArchiveProto.Entry entry) { + if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.TILE) { + return Optional.empty(); + } + final StreamArchiveProto.TileEntry tileEntry = entry.getTile(); + return Optional.of(new Tile( + TileCoord.ofXYZ(tileEntry.getX(), tileEntry.getY(), tileEntry.getZ()), + tileEntry.getEncodedData().toByteArray() + )); + } + + @Override + Optional mapEntryToMetadata(StreamArchiveProto.Entry entry) { + if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.FINISH) { + return Optional.empty(); + } + final StreamArchiveProto.Metadata metadata = entry.getFinish().getMetadata(); + return Optional.of(new TileArchiveMetadata( + StringUtils.trimToNull(metadata.getName()), + StringUtils.trimToNull(metadata.getDescription()), + StringUtils.trimToNull(metadata.getAttribution()), + StringUtils.trimToNull(metadata.getVersion()), + StringUtils.trimToNull(metadata.getType()), + StringUtils.trimToNull(metadata.getFormat()), + deserializeEnvelope(metadata.hasBounds() ? metadata.getBounds() : null), + deserializeCoordinate(metadata.hasCenter() ? metadata.getCenter() : null), + metadata.hasMinZoom() ? metadata.getMinZoom() : null, + metadata.hasMaxZoom() ? metadata.getMaxZoom() : null, + extractMetadataJson(metadata), + metadata.getOthersMap(), + deserializeTileCompression(metadata.getTileCompression()) + )); + } + + private Envelope deserializeEnvelope(StreamArchiveProto.Envelope s) { + return s == null ? null : new Envelope(s.getMinX(), s.getMaxX(), s.getMinY(), s.getMaxY()); + } + + private Coordinate deserializeCoordinate(StreamArchiveProto.Coordinate s) { + if (s == null) { + return null; + } + return s.hasZ() ? new Coordinate(s.getX(), s.getY(), s.getZ()) : new CoordinateXY(s.getX(), s.getY()); + } + + private TileCompression deserializeTileCompression(StreamArchiveProto.TileCompression s) { + return switch (s) { + case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNWON; + case TILE_COMPRESSION_GZIP -> TileCompression.GZIP; + case TILE_COMPRESSION_NONE -> TileCompression.NONE; + }; + } + + private TileArchiveMetadata.TileArchiveMetadataJson extractMetadataJson(StreamArchiveProto.Metadata s) { + final List vl = deserializeVectorLayers(s.getVectorLayersList()); + return vl.isEmpty() ? null : new TileArchiveMetadata.TileArchiveMetadataJson(vl); + } + + private List deserializeVectorLayers(List s) { + return s.stream() + .map(vl -> new LayerAttrStats.VectorLayer( + vl.getId(), + vl.getFieldsMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> deserializeFieldType(e.getValue()))), + Optional.ofNullable(StringUtils.trimToNull(vl.getDescription())), + vl.hasMinZoom() ? OptionalInt.of(vl.getMinZoom()) : OptionalInt.empty(), + vl.hasMaxZoom() ? OptionalInt.of(vl.getMaxZoom()) : OptionalInt.empty() + )) + .toList(); + } + + private LayerAttrStats.FieldType deserializeFieldType(StreamArchiveProto.VectorLayer.FieldType s) { + return switch (s) { + case FIELD_TYPE_UNSPECIFIED, UNRECOGNIZED -> throw new IllegalArgumentException("unknown type"); + case FIELD_TYPE_NUMBER -> LayerAttrStats.FieldType.NUMBER; + case FIELD_TYPE_BOOLEAN -> LayerAttrStats.FieldType.BOOLEAN; + case FIELD_TYPE_STRING -> LayerAttrStats.FieldType.STRING; + }; + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java new file mode 100644 index 00000000..f6f9ecff --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java @@ -0,0 +1,94 @@ +package com.onthegomap.planetiler.stream; + +import com.google.common.base.Suppliers; +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.util.CloseableIterator; +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Supplier; + +abstract class ReadableStreamArchive implements ReadableTileArchive { + + private final Supplier cachedMetadata = Suppliers.memoize(this::loadMetadata); + + final Path basePath; + final StreamArchiveConfig config; + + ReadableStreamArchive(Path basePath, StreamArchiveConfig config) { + this.basePath = basePath; + this.config = config; + } + + @Override + public final byte[] getTile(TileCoord coord) { + return getAllTiles().stream().filter(c -> c.coord().equals(coord)).map(Tile::bytes).findFirst().orElse(null); + } + + @Override + public final byte[] getTile(int x, int y, int z) { + return getTile(TileCoord.ofXYZ(x, y, z)); + } + + @Override + public final CloseableIterator getAllTileCoords() { + return getAllTiles().map(Tile::coord); + } + + @Override + public final CloseableIterator getAllTiles() { + return createIterator() + .map(this::mapEntryToTile) + .filter(Optional::isPresent) + .map(Optional::get); + } + + @Override + public final TileArchiveMetadata metadata() { + return cachedMetadata.get(); + } + + private TileArchiveMetadata loadMetadata() { + try (var it = createIterator()) { + return it.stream().map(this::mapEntryToMetadata).flatMap(Optional::stream).findFirst().orElse(null); + } + } + + @Override + public void close() throws IOException { + // nothing to close + } + + abstract CloseableIterator createIterator(); + + abstract Optional mapEntryToTile(E entry); + + abstract Optional mapEntryToMetadata(E entry); + + void closeSilentlyOnError(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (Exception ignored) { + // ignore + } + } + + @SuppressWarnings("java:S112") + void closeUnchecked(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java index 6f85ccb5..08b8967c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java @@ -1,10 +1,10 @@ package com.onthegomap.planetiler.stream; import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.PlanetilerConfig; +import com.onthegomap.planetiler.config.CommonConfigs; -public record StreamArchiveConfig(boolean appendToFile, Arguments moreOptions) { - public StreamArchiveConfig(PlanetilerConfig planetilerConfig, Arguments moreOptions) { - this(planetilerConfig.append(), moreOptions); +public record StreamArchiveConfig(boolean appendToFile, Arguments formatOptions) { + public StreamArchiveConfig(Arguments baseArguments, Arguments formatOptions) { + this(CommonConfigs.appendToArchive(baseArguments), formatOptions); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java index 8d23ef6f..b1ff2e20 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java @@ -1,5 +1,8 @@ package com.onthegomap.planetiler.stream; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.google.common.net.UrlEscapers; import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.config.Arguments; @@ -12,8 +15,25 @@ import org.apache.commons.text.StringEscapeUtils; public final class StreamArchiveUtils { + /** + * exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases + * => allow to output tiles, only + */ + private static final String JSON_OPTION_WRITE_TILES_ONLY = "tiles_only"; + + private static final String JSON_OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator"; + + static final String CSV_OPTION_COLUMN_SEPARATOR = "column_separator"; + static final String CSV_OPTION_LINE_SEPARATOR = "line_separator"; + static final String CSV_OPTION_BINARY_ENCODING = "binary_encoding"; + private static final Pattern quotedPattern = Pattern.compile("^'(.+?)'$"); + static final JsonMapper jsonMapperJsonStreamArchive = JsonMapper.builder() + .serializationInclusion(JsonInclude.Include.NON_ABSENT) + .addModule(new Jdk8Module()) + .build(); + private StreamArchiveUtils() {} public static Path constructIndexedPath(Path basePath, int index) { @@ -39,6 +59,36 @@ public final class StreamArchiveUtils { .translateEscapes(); } + static String jsonOptionRootValueSeparator(Arguments formatOptions) { + return getEscapedString(formatOptions, TileArchiveConfig.Format.JSON, + JSON_OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " ")); + } + + static boolean jsonOptionWriteTilesOnly(Arguments formatOptions) { + return formatOptions.getBoolean(JSON_OPTION_WRITE_TILES_ONLY, "write tiles, only", false); + } + + static String csvOptionColumnSeparator(Arguments formatOptions, TileArchiveConfig.Format format) { + final String defaultColumnSeparator = switch (format) { + case CSV -> "','"; + case TSV -> "'\\t'"; + default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id()); + }; + return getEscapedString(formatOptions, format, + CSV_OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " ")); + } + + static String csvOptionLineSeparator(Arguments formatOptions, TileArchiveConfig.Format format) { + return StreamArchiveUtils.getEscapedString(formatOptions, format, + CSV_OPTION_LINE_SEPARATOR, "line separator", "'\\n'", List.of("\n", "\r\n")); + } + + static CsvBinaryEncoding csvOptionBinaryEncoding(Arguments formatOptions) { + return CsvBinaryEncoding.fromId(formatOptions.getString(CSV_OPTION_BINARY_ENCODING, + "binary (tile) data encoding - one of " + CsvBinaryEncoding.ids(), "base64")); + } + + private static String escapeJava(String s) { if (!s.trim().equals(s)) { s = "'" + s + "'"; diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java index 5f9d549e..be0bc513 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java @@ -11,11 +11,7 @@ import java.io.UncheckedIOException; import java.io.Writer; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.Base64; -import java.util.HexFormat; -import java.util.List; import java.util.function.Function; -import java.util.stream.Stream; /** * Writes tile data into a CSV file (or pipe). @@ -67,31 +63,16 @@ import java.util.stream.Stream; */ public final class WriteableCsvArchive extends WriteableStreamArchive { - static final String OPTION_COLUMN_SEPARATOR = "column_separator"; - static final String OPTION_LINE_SEPARTATOR = "line_separator"; - static final String OPTION_BINARY_ENCODING = "binary_encoding"; - private final String columnSeparator; private final String lineSeparator; private final Function tileDataEncoder; private WriteableCsvArchive(TileArchiveConfig.Format format, Path p, StreamArchiveConfig config) { super(p, config); - final String defaultColumnSeparator = switch (format) { - case CSV -> "','"; - case TSV -> "'\\t'"; - default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id()); - }; - this.columnSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format, - OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " ")); - this.lineSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format, - OPTION_LINE_SEPARTATOR, "line separator", "'\\n'", List.of("\n", "\r\n")); - final BinaryEncoding binaryEncoding = BinaryEncoding.fromId(config.moreOptions().getString(OPTION_BINARY_ENCODING, - "binary (tile) data encoding - one of " + BinaryEncoding.ids(), "base64")); - this.tileDataEncoder = switch (binaryEncoding) { - case BASE64 -> Base64.getEncoder()::encodeToString; - case HEX -> HexFormat.of()::formatHex; - }; + this.columnSeparator = StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format); + this.lineSeparator = StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format); + final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions()); + this.tileDataEncoder = binaryEncoding::encode; } public static WriteableCsvArchive newWriteToFile(TileArchiveConfig.Format format, Path path, @@ -131,7 +112,8 @@ public final class WriteableCsvArchive extends WriteableStreamArchive { @Override public void write(TileEncodingResult encodingResult) { final TileCoord coord = encodingResult.coord(); - final String tileDataEncoded = tileDataEncoder.apply(encodingResult.tileData()); + final byte[] data = encodingResult.tileData(); + final String tileDataEncoded = data == null ? "" : tileDataEncoder.apply(encodingResult.tileData()); try { // x | y | z | encoded data writer.write("%d%s%d%s%d%s%s%s".formatted(coord.x(), columnSeparator, coord.y(), columnSeparator, coord.z(), @@ -150,32 +132,4 @@ public final class WriteableCsvArchive extends WriteableStreamArchive { } } } - - private enum BinaryEncoding { - - BASE64("base64"), - HEX("hex"); - - private final String id; - - private BinaryEncoding(String id) { - this.id = id; - } - - static List ids() { - return Stream.of(BinaryEncoding.values()).map(BinaryEncoding::id).toList(); - } - - static BinaryEncoding fromId(String id) { - return Stream.of(BinaryEncoding.values()) - .filter(de -> de.id().equals(id)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "unexpected binary encoding - expected one of " + ids() + " but got " + id)); - } - - String id() { - return id; - } - } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java index 1fad9f0b..671a8676 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java @@ -1,14 +1,8 @@ package com.onthegomap.planetiler.stream; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.onthegomap.planetiler.archive.TileArchiveConfig; import com.onthegomap.planetiler.archive.TileArchiveMetadata; import com.onthegomap.planetiler.archive.TileEncodingResult; import com.onthegomap.planetiler.geo.TileCoord; @@ -19,41 +13,26 @@ import java.io.OutputStreamWriter; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Writes JSON-serialized tile data as well as meta data into file(s). The entries are of type - * {@link WriteableJsonStreamArchive.Entry} are separated by newline (by default). + * {@link JsonStreamArchiveEntry} are separated by newline (by default). */ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { private static final Logger LOGGER = LoggerFactory.getLogger(WriteableJsonStreamArchive.class); - /** - * exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases - * => allow to output tiles, only - */ - private static final String OPTION_WRITE_TILES_ONLY = "tiles_only"; - - private static final String OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator"; - - static final JsonMapper jsonMapper = JsonMapper.builder() - .serializationInclusion(Include.NON_ABSENT) - .addModule(new Jdk8Module()) - .build(); + private static final JsonMapper jsonMapper = StreamArchiveUtils.jsonMapperJsonStreamArchive; private final boolean writeTilesOnly; private final String rootValueSeparator; private WriteableJsonStreamArchive(Path p, StreamArchiveConfig config) { super(p, config); - this.writeTilesOnly = config.moreOptions().getBoolean(OPTION_WRITE_TILES_ONLY, "write tiles, only", false); - this.rootValueSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), TileArchiveConfig.Format.JSON, - OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " ")); + this.writeTilesOnly = StreamArchiveUtils.jsonOptionWriteTilesOnly(config.formatOptions()); + this.rootValueSeparator = StreamArchiveUtils.jsonOptionRootValueSeparator(config.formatOptions()); } public static WriteableJsonStreamArchive newWriteToFile(Path path, StreamArchiveConfig config) { @@ -70,7 +49,7 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { if (writeTilesOnly) { return; } - writeEntryFlush(new InitializationEntry()); + writeEntryFlush(new JsonStreamArchiveEntry.InitializationEntry()); } @Override @@ -78,13 +57,13 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { if (writeTilesOnly) { return; } - writeEntryFlush(new FinishEntry(metadata)); + writeEntryFlush(new JsonStreamArchiveEntry.FinishEntry(metadata)); } - private void writeEntryFlush(Entry entry) { + private void writeEntryFlush(JsonStreamArchiveEntry entry) { try (var out = new OutputStreamWriter(getPrimaryOutputStream(), StandardCharsets.UTF_8.newEncoder())) { jsonMapper - .writerFor(Entry.class) + .writerFor(JsonStreamArchiveEntry.class) .withoutFeatures(JsonGenerator.Feature.AUTO_CLOSE_TARGET) .writeValue(out, entry); out.write(rootValueSeparator); @@ -104,7 +83,8 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { this.rootValueSeparator = rootValueSeparator; try { this.jsonWriter = - jsonMapper.writerFor(Entry.class).withRootValueSeparator(rootValueSeparator).writeValues(outputStream); + jsonMapper.writerFor(JsonStreamArchiveEntry.class).withRootValueSeparator(rootValueSeparator) + .writeValues(outputStream); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -114,7 +94,8 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { public void write(TileEncodingResult encodingResult) { final TileCoord coord = encodingResult.coord(); try { - jsonWriter.write(new TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData())); + jsonWriter + .write(new JsonStreamArchiveEntry.TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData())); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -150,53 +131,4 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive { } } - - @JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "type") - @JsonSubTypes({ - @Type(value = TileEntry.class, name = "tile"), - @Type(value = InitializationEntry.class, name = "initialization"), - @Type(value = FinishEntry.class, name = "finish") - }) - sealed interface Entry { - - } - - - record TileEntry(int x, int y, int z, byte[] encodedData) implements Entry { - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(encodedData); - result = prime * result + Objects.hash(x, y, z); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof TileEntry)) { - return false; - } - TileEntry other = (TileEntry) obj; - return Arrays.equals(encodedData, other.encodedData) && x == other.x && y == other.y && z == other.z; - } - - @Override - public String toString() { - return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]"; - } - } - - record InitializationEntry() implements Entry {} - - - record FinishEntry(TileArchiveMetadata metadata) implements Entry {} - } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java index 54566948..a618ffe2 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java @@ -158,15 +158,17 @@ public final class WriteableProtoStreamArchive extends WriteableStreamArchive { @Override public void write(TileEncodingResult encodingResult) { final TileCoord coord = encodingResult.coord(); - final StreamArchiveProto.TileEntry tile = StreamArchiveProto.TileEntry.newBuilder() + final byte[] data = encodingResult.tileData(); + StreamArchiveProto.TileEntry.Builder tileBuilder = StreamArchiveProto.TileEntry.newBuilder() .setZ(coord.z()) .setX(coord.x()) - .setY(coord.y()) - .setEncodedData(ByteString.copyFrom(encodingResult.tileData())) - .build(); + .setY(coord.y()); + if (data != null) { + tileBuilder = tileBuilder.setEncodedData(ByteString.copyFrom(encodingResult.tileData())); + } final StreamArchiveProto.Entry entry = StreamArchiveProto.Entry.newBuilder() - .setTile(tile) + .setTile(tileBuilder.build()) .build(); try { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java index 4efe6257..6b44b21a 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java @@ -2,8 +2,10 @@ package com.onthegomap.planetiler.util; import java.io.Closeable; import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.Spliterators; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -56,4 +58,42 @@ public interface CloseableIterator extends Closeable, Iterator { } }; } + + default CloseableIterator filter(Predicate predicate) { + final var parent = this; + return new CloseableIterator<>() { + private T nextValue; + + @Override + public void close() { + parent.close(); + } + + @Override + public boolean hasNext() { + if (nextValue != null) { + return true; + } + while (parent.hasNext()) { + final T parentNext = parent.next(); + if (predicate.test(parentNext)) { + nextValue = parentNext; + break; + } + } + return nextValue != null; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final T returnValue = nextValue; + nextValue = null; + return returnValue; + } + }; + } + } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java index 3e341208..56884ed9 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java @@ -3,22 +3,42 @@ package com.onthegomap.planetiler.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -public class Gzip { +public final class Gzip { - public static byte[] gzip(byte[] in) throws IOException { + private Gzip() {} + + @SuppressWarnings("java:S1168") // null in, null out + public static byte[] gzip(byte[] in) { + if (in == null) { + return null; + } var bos = new ByteArrayOutputStream(in.length); try (var gzipOS = new GZIPOutputStream(bos)) { gzipOS.write(in); + } catch (IOException e) { + throw new UncheckedIOException(e); } return bos.toByteArray(); } - public static byte[] gunzip(byte[] zipped) throws IOException { + @SuppressWarnings("java:S1168") // null in, null out + public static byte[] gunzip(byte[] zipped) { + if (zipped == null) { + return null; + } try (var is = new GZIPInputStream(new ByteArrayInputStream(zipped))) { return is.readAllBytes(); + } catch (IOException e) { + throw new UncheckedIOException(e); } } + + public static boolean isZipped(byte[] in) { + return in != null && in.length > 2 && in[0] == (byte) GZIPInputStream.GZIP_MAGIC && + in[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8); + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java index 45bc476e..a5b69663 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java @@ -61,6 +61,9 @@ public final class Hashing { */ public static long fnv1a64(long initHash, byte... data) { long hash = initHash; + if (data == null) { + return hash; + } for (byte datum : data) { hash ^= (datum & 0xff); hash *= FNV1_PRIME_64; diff --git a/planetiler-core/src/main/proto/stream_archive_proto.proto b/planetiler-core/src/main/proto/stream_archive_proto.proto index 1df9f81f..374ab453 100644 --- a/planetiler-core/src/main/proto/stream_archive_proto.proto +++ b/planetiler-core/src/main/proto/stream_archive_proto.proto @@ -21,7 +21,7 @@ message InitializationEntry { } message FinishEntry { - Metadata metadata = 1; + optional Metadata metadata = 1; } message Metadata { @@ -32,10 +32,10 @@ message Metadata { string version = 4; string type = 5; string format = 6; - Envelope bounds = 7; - Coordinate center = 8; - int32 min_zoom = 9; - int32 max_zoom = 10; + optional Envelope bounds = 7; + optional Coordinate center = 8; + optional int32 min_zoom = 9; + optional int32 max_zoom = 10; repeated VectorLayer vector_layers = 11; map others = 12; TileCompression tile_compression = 13; @@ -51,15 +51,15 @@ message Envelope { message Coordinate { double x = 1; double y = 2; - double z = 3; + optional double z = 3; } message VectorLayer { string id = 1; map fields = 2; string description = 3; - int32 min_zoom = 4; - int32 max_zoom = 5; + optional int32 min_zoom = 4; + optional int32 max_zoom = 5; enum FieldType { FIELD_TYPE_UNSPECIFIED = 0; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java index 610af5d9..6e9cdbe4 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java @@ -31,7 +31,10 @@ import com.onthegomap.planetiler.reader.osm.OsmElement; import com.onthegomap.planetiler.reader.osm.OsmReader; import com.onthegomap.planetiler.reader.osm.OsmRelationInfo; import com.onthegomap.planetiler.stats.Stats; -import com.onthegomap.planetiler.stream.InMemoryStreamArchive; +import com.onthegomap.planetiler.stream.ReadableCsvArchive; +import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive; +import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive; +import com.onthegomap.planetiler.stream.StreamArchiveConfig; import com.onthegomap.planetiler.util.BuildInfo; import com.onthegomap.planetiler.util.Gzip; import com.onthegomap.planetiler.util.TileSizeStats; @@ -2040,11 +2043,12 @@ class PlanetilerTests { final ReadableTileArchiveFactory readableTileArchiveFactory = switch (format) { case MBTILES -> Mbtiles::newReadOnlyDatabase; - case CSV -> p -> InMemoryStreamArchive.fromCsv(p, ","); - case TSV -> p -> InMemoryStreamArchive.fromCsv(p, "\t"); - case JSON -> InMemoryStreamArchive::fromJson; + case CSV, TSV -> + p -> ReadableCsvArchive.newReader(format, p, new StreamArchiveConfig(false, Arguments.of())); + case JSON -> p -> ReadableJsonStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of())); case PMTILES -> ReadablePmtiles::newReadFromFile; - case PROTO, PBF -> InMemoryStreamArchive::fromProtobuf; + case PROTO, PBF -> + p -> ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of())); case FILES -> p -> ReadableFilesArchive.newReader(p, Arguments.of()); }; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 603d28e5..1d11e550 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -770,7 +770,7 @@ public class TestUtils { if (!failures.isEmpty()) { fail(String.join(System.lineSeparator(), failures)); } - } catch (GeometryException | IOException e) { + } catch (GeometryException e) { fail(e); } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java new file mode 100644 index 00000000..c3e5b70d --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java @@ -0,0 +1,200 @@ +package com.onthegomap.planetiler.archive; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.util.Gzip; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; + +class TileCopyTest { + + private static final String ARCHIVE_0_JSON_BASE = """ + {"type":"initialization"} + {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="} + {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="} + {"type":"finish","metadata":%s} + """; + + private static final String ARCHIVE_0_CSV_COMPRESSION_NONE = """ + 0,0,0,AA== + 1,2,3,AQ== + """; + + private static final String EXTERNAL_METADATA = "{\"name\": \"blub\"}"; + + @ParameterizedTest(name = "{index} - {0}") + @ArgumentsSource(TestArgs.class) + void testSimple(String testName, String archiveDataIn, String archiveDataOut, Map arguments, + @TempDir Path tempDir) throws Exception { + + final Path archiveInPath = tempDir.resolve(archiveDataIn.contains("{") ? "in.json" : "in.csv"); + final Path archiveOutPath = tempDir.resolve(archiveDataOut.contains("{") ? "out.json" : "out.csv"); + final Path inMetadataPath = tempDir.resolve("metadata.json"); + + Files.writeString(archiveInPath, archiveDataIn); + Files.writeString(inMetadataPath, EXTERNAL_METADATA); + + arguments = new LinkedHashMap<>(arguments); + arguments.replace("in_metadata", inMetadataPath.toString()); + + final Arguments args = Arguments.of(Map.of( + "input", archiveInPath.toString(), + "output", archiveOutPath.toString() + )).orElse(Arguments.of(arguments)); + + new TileCopy(TileCopy.TileCopyConfig.fromArguments(args)).run(); + + if (archiveDataOut.contains("{")) { + final List expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList(); + final List actualLines = Files.readAllLines(archiveOutPath); + assertEquals(expectedLines.size(), actualLines.size()); + for (int i = 0; i < expectedLines.size(); i++) { + TestUtils.assertSameJson(expectedLines.get(i), actualLines.get(i)); + } + } else { + assertEquals(archiveDataOut, Files.readString(archiveOutPath)); + } + } + + private static String compressBase64(String archiveIn) { + final Base64.Encoder encoder = Base64.getEncoder(); + for (int i = 0; i <= 1; i++) { + archiveIn = archiveIn.replace( + encoder.encodeToString(new byte[]{(byte) i}), + encoder.encodeToString(Gzip.gzip(new byte[]{(byte) i})) + ); + } + return archiveIn; + } + + private static String replaceBase64(String archiveIn, String replacement) { + final Base64.Encoder encoder = Base64.getEncoder(); + for (int i = 0; i <= 1; i++) { + archiveIn = archiveIn.replace( + encoder.encodeToString(new byte[]{(byte) i}), + replacement + ); + } + return archiveIn; + } + + private static class TestArgs implements ArgumentsProvider { + + @Override + public Stream provideArguments(ExtensionContext context) { + return Stream.of( + argsOf( + "json(w/o meta, compression:none) to csv(compression:none)", + ARCHIVE_0_JSON_BASE.formatted("null"), + ARCHIVE_0_CSV_COMPRESSION_NONE, + Map.of("out_tile_compression", "none") + ), + argsOf( + "json(w/o meta, compression:none) to csv(compression:gzip)", + ARCHIVE_0_JSON_BASE.formatted("null"), + compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE), + Map.of("out_tile_compression", "gzip") + ), + argsOf( + "json(w/o meta, compression:gzip) to csv(compression:none)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")), + ARCHIVE_0_CSV_COMPRESSION_NONE, + Map.of("out_tile_compression", "none") + ), + argsOf( + "json(w/o meta, compression:gzip) to csv(compression:gzip)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")), + compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE), + Map.of("out_tile_compression", "gzip") + ), + argsOf( + "json(w/o meta, compression:gzip) to csv(compression:gzip)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")), + compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE), + Map.of("out_tile_compression", "gzip") + ), + argsOf( + "json(w/ meta, compression:gzip) to csv(compression:none)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)), + ARCHIVE_0_CSV_COMPRESSION_NONE, + Map.of("out_tile_compression", "none") + ), + argsOf( + "json(w/ meta, compression:gzip) to json(w/ meta, compression:gzip)", + compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)), + compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)), + Map.of() + ), + argsOf( + "csv to json - use fallback metadata", + ARCHIVE_0_CSV_COMPRESSION_NONE, + ARCHIVE_0_JSON_BASE.formatted( + "{\"name\":\"unknown\",\"format\":\"pbf\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"), + Map.of("out_tile_compression", "none") + ), + argsOf( + "csv to json - use external metadata", + ARCHIVE_0_CSV_COMPRESSION_NONE, + ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"none\"}"), + Map.of("out_tile_compression", "none", "in_metadata", "blub") + ), + argsOf( + "csv to json - null handling", + replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), + replaceBase64(ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"gzip\"}"), "null") + .replace(",\"encodedData\":\"null\"", ""), + Map.of("in_metadata", "blub") + ), + argsOf( + "json to csv - null handling", + replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null") + .replace(",\"encodedData\":\"null\"", ""), + replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), + Map.of() + ), + argsOf( + "csv to csv - empty skipping on", + """ + 0,0,0, + 1,2,3,AQ== + """, + """ + 1,2,3,AQ== + """, + Map.of("skip_empty", "true", "out_tile_compression", "none") + ), + argsOf( + "csv to csv - empty skipping off", + """ + 0,0,0, + 1,2,3,AQ== + """, + """ + 0,0,0, + 1,2,3,AQ== + """, + Map.of("skip_empty", "false", "out_tile_compression", "none") + ) + ); + } + + private static org.junit.jupiter.params.provider.Arguments argsOf(String testName, String archiveDataIn, + String archiveDataOut, Map arguments) { + return org.junit.jupiter.params.provider.Arguments.of(testName, archiveDataIn, archiveDataOut, arguments); + } + } + +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java deleted file mode 100644 index 1d30afb1..00000000 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.onthegomap.planetiler.stream; - -import com.onthegomap.planetiler.archive.ReadableTileArchive; -import com.onthegomap.planetiler.archive.TileArchiveMetadata; -import com.onthegomap.planetiler.archive.TileEncodingResult; -import com.onthegomap.planetiler.geo.TileCoord; -import com.onthegomap.planetiler.proto.StreamArchiveProto; -import com.onthegomap.planetiler.util.CloseableIterator; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Base64; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.OptionalLong; - -public class InMemoryStreamArchive implements ReadableTileArchive { - - private final List tileEncodings; - private final TileArchiveMetadata metadata; - - private InMemoryStreamArchive(List tileEncodings, TileArchiveMetadata metadata) { - this.tileEncodings = tileEncodings; - this.metadata = metadata; - } - - public static InMemoryStreamArchive fromCsv(Path p, String columnSepatator) throws IOException { - var base64Decoder = Base64.getDecoder(); - final List tileEncodings = new ArrayList<>(); - try (var reader = Files.newBufferedReader(p)) { - String line; - while ((line = reader.readLine()) != null) { - final String[] splits = line.split(columnSepatator); - final TileCoord tileCoord = TileCoord.ofXYZ(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]), - Integer.parseInt(splits[2])); - tileEncodings.add(new TileEncodingResult(tileCoord, base64Decoder.decode(splits[3]), OptionalLong.empty())); - } - } - return new InMemoryStreamArchive(tileEncodings, null); - } - - public static InMemoryStreamArchive fromProtobuf(Path p) throws IOException { - final List tileEncodings = new ArrayList<>(); - try (var in = Files.newInputStream(p)) { - StreamArchiveProto.Entry entry; - while ((entry = StreamArchiveProto.Entry.parseDelimitedFrom(in)) != null) { - if (entry.getEntryCase() == StreamArchiveProto.Entry.EntryCase.TILE) { - final StreamArchiveProto.TileEntry tileProto = entry.getTile(); - final TileCoord tileCoord = TileCoord.ofXYZ(tileProto.getX(), tileProto.getY(), tileProto.getZ()); - tileEncodings - .add(new TileEncodingResult(tileCoord, tileProto.getEncodedData().toByteArray(), OptionalLong.empty())); - } - } - } - return new InMemoryStreamArchive(tileEncodings, null /* could add once the format is finalized*/); - } - - public static InMemoryStreamArchive fromJson(Path p) throws IOException { - final List tileEncodings = new ArrayList<>(); - final TileArchiveMetadata[] metadata = new TileArchiveMetadata[]{null}; - try (var reader = Files.newBufferedReader(p)) { - WriteableJsonStreamArchive.jsonMapper - .readerFor(WriteableJsonStreamArchive.Entry.class) - .readValues(reader) - .forEachRemaining(entry -> { - if (entry instanceof WriteableJsonStreamArchive.TileEntry te) { - final TileCoord tileCoord = TileCoord.ofXYZ(te.x(), te.y(), te.z()); - tileEncodings.add(new TileEncodingResult(tileCoord, te.encodedData(), OptionalLong.empty())); - } else if (entry instanceof WriteableJsonStreamArchive.FinishEntry fe) { - metadata[0] = fe.metadata(); - } - }); - } - return new InMemoryStreamArchive(tileEncodings, Objects.requireNonNull(metadata[0])); - } - - @Override - public void close() throws IOException {} - - @Override - public byte[] getTile(int x, int y, int z) { - - final TileCoord coord = TileCoord.ofXYZ(x, y, z); - - return tileEncodings.stream() - .filter(ter -> ter.coord().equals(coord)).findFirst() - .map(TileEncodingResult::tileData) - .orElse(null); - } - - @Override - public CloseableIterator getAllTileCoords() { - - final Iterator it = tileEncodings.iterator(); - - return new CloseableIterator() { - @Override - public TileCoord next() { - return it.next().coord(); - } - - @Override - public boolean hasNext() { - return it.hasNext(); - } - - @Override - public void close() {} - }; - } - - @Override - public TileArchiveMetadata metadata() { - return metadata; - } - -} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java new file mode 100644 index 00000000..31258995 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java @@ -0,0 +1,75 @@ +package com.onthegomap.planetiler.stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.geo.TileCoord; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.apache.commons.text.StringEscapeUtils; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class ReadableCsvStreamArchiveTest { + + + @ParameterizedTest + @CsvSource(delimiter = '$', textBlock = """ + ,$\\n$false$BASE64 + ,$\\r$false$BASE64 + ,$\\r\\n$false$BASE64 + ,$x$false$BASE64 + ;$\\n$false$BASE64 + {$\\n$false$BASE64 + ,${$false$BASE64 + ,$\\n$false$HEX + ,$\\n$true$BASE64 + """) + void testSimple(String columnSeparator, String lineSeparator, boolean pad, CsvBinaryEncoding encoding, + @TempDir Path tempDir) throws IOException { + + final Path csvFile = tempDir.resolve("in.csv"); + final String csv = + """ + 0,0,0,AA== + 1,2,3,AQ== + """ + .replace("\n", StringEscapeUtils.unescapeJava(lineSeparator)) + .replace(",", columnSeparator + (pad ? " " : "")) + .replace("AA==", encoding == CsvBinaryEncoding.BASE64 ? "AA==" : "00") + .replace("AQ==", encoding == CsvBinaryEncoding.BASE64 ? "AQ==" : "01"); + + Files.writeString(csvFile, csv); + final StreamArchiveConfig config = new StreamArchiveConfig( + false, + Arguments.of(Map.of( + StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, columnSeparator, + StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, lineSeparator, + StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, encoding.id() + )) + ); + + final List expectedTiles = List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) + ); + + try (var reader = ReadableCsvArchive.newReader(TileArchiveConfig.Format.CSV, csvFile, config)) { + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertNull(reader.metadata()); + assertNull(reader.metadata()); + assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3))); + assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0)); + assertNull(reader.getTile(4, 5, 6)); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java new file mode 100644 index 00000000..ab728f8e --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java @@ -0,0 +1,48 @@ +package com.onthegomap.planetiler.stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.onthegomap.planetiler.TestUtils; +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.geo.TileCoord; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ReadableJsonStreamArchiveTest { + + @Test + void testSimple(@TempDir Path tempDir) throws IOException { + + final Path jsonFile = tempDir.resolve("in.json"); + final String json = """ + {"type":"initialization"} + {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="} + {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="} + {"type":"finish","metadata":%s} + """.formatted(TestUtils.MAX_METADATA_SERIALIZED); + + Files.writeString(jsonFile, json); + final StreamArchiveConfig config = new StreamArchiveConfig(false, Arguments.of()); + + final List expectedTiles = List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) + ); + try (var reader = ReadableJsonStreamArchive.newReader(jsonFile, config)) { + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata()); + assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata()); + assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3))); + assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0)); + assertNull(reader.getTile(4, 5, 6)); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java new file mode 100644 index 00000000..7681f98e --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java @@ -0,0 +1,60 @@ +package com.onthegomap.planetiler.stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.protobuf.ByteString; +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.proto.StreamArchiveProto; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class ReadableProtoStreamArchiveTest { + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testSimple(boolean maxMetaData, @TempDir Path tempDir) throws IOException { + + final StreamArchiveProto.Metadata metadataSerialized = maxMetaData ? + WriteableProtoStreamArchiveTest.maxMetadataSerialized : WriteableProtoStreamArchiveTest.minMetadataSerialized; + + final TileArchiveMetadata metadataDeserialized = maxMetaData ? + WriteableProtoStreamArchiveTest.maxMetadataDeserialized : WriteableProtoStreamArchiveTest.minMetadataDeserialized; + + + final Path p = tempDir.resolve("out.proto"); + try (var out = Files.newOutputStream(p)) { + StreamArchiveProto.Entry.newBuilder().setInitialization( + StreamArchiveProto.InitializationEntry.newBuilder() + ).build().writeDelimitedTo(out); + StreamArchiveProto.Entry.newBuilder().setTile( + StreamArchiveProto.TileEntry.newBuilder() + .setX(0).setY(0).setZ(0).setEncodedData(ByteString.copyFrom(new byte[]{0})) + ).build().writeDelimitedTo(out); + StreamArchiveProto.Entry.newBuilder().setTile( + StreamArchiveProto.TileEntry.newBuilder() + .setX(1).setY(2).setZ(3).setEncodedData(ByteString.copyFrom(new byte[]{1})) + ).build().writeDelimitedTo(out); + StreamArchiveProto.Entry.newBuilder().setFinish( + StreamArchiveProto.FinishEntry.newBuilder() + .setMetadata(metadataSerialized) + ).build().writeDelimitedTo(out); + } + final List expectedTiles = List.of( + new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), + new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) + ); + try (var reader = ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()))) { + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + assertEquals(metadataDeserialized, reader.metadata()); + } + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java index 48d05956..e69d08e5 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java @@ -113,7 +113,7 @@ class WriteableCsvArchiveTest { void testColumnSeparator(@TempDir Path tempDir) throws IOException { final StreamArchiveConfig config = - new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_COLUMN_SEPARATOR, "' '"))); + new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, "' '"))); final String expectedCsv = """ @@ -128,7 +128,7 @@ class WriteableCsvArchiveTest { void testLineSeparator(@TempDir Path tempDir) throws IOException { final StreamArchiveConfig config = - new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_LINE_SEPARTATOR, "'\\r'"))); + new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, "'\\r'"))); final String expectedCsv = """ @@ -143,7 +143,7 @@ class WriteableCsvArchiveTest { void testHexEncoding(@TempDir Path tempDir) throws IOException { final StreamArchiveConfig config = - new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_BINARY_ENCODING, "hex"))); + new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, "hex"))); final String expectedCsv = """ diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java index 934a62bf..846739ac 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java @@ -28,8 +28,8 @@ import org.locationtech.jts.geom.Envelope; class WriteableProtoStreamArchiveTest { - private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null); - private static final TileArchiveMetadata maxMetadataIn = + static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null); + static final TileArchiveMetadata maxMetadataDeserialized = new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3), new Coordinate(1.3, 3.7, 1.0), 2, 3, TileArchiveMetadata.TileArchiveMetadataJson.create( @@ -45,7 +45,7 @@ class WriteableProtoStreamArchiveTest { ), Map.of("a", "b", "c", "d"), TileCompression.GZIP); - private static final StreamArchiveProto.Metadata maxMetadataOut = StreamArchiveProto.Metadata.newBuilder() + static final StreamArchiveProto.Metadata maxMetadataSerialized = StreamArchiveProto.Metadata.newBuilder() .setName("name").setDescription("description").setAttribution("attribution").setVersion("version") .setType("type").setFormat("format") .setBounds(StreamArchiveProto.Envelope.newBuilder().setMinX(0).setMaxX(1).setMinY(2).setMaxY(3).build()) @@ -64,10 +64,10 @@ class WriteableProtoStreamArchiveTest { .setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_GZIP) .build(); - private static final TileArchiveMetadata minMetadataIn = - new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null, + static final TileArchiveMetadata minMetadataDeserialized = + new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, Map.of(), TileCompression.NONE); - private static final StreamArchiveProto.Metadata minMetadataOut = StreamArchiveProto.Metadata.newBuilder() + static final StreamArchiveProto.Metadata minMetadataSerialized = StreamArchiveProto.Metadata.newBuilder() .setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_NONE) .build(); @@ -83,12 +83,12 @@ class WriteableProtoStreamArchiveTest { tileWriter.write(tile0); tileWriter.write(tile1); } - archive.finish(minMetadataIn); + archive.finish(minMetadataDeserialized); } try (InputStream in = Files.newInputStream(csvFile)) { assertEquals( - List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)), + List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataSerialized)), readAllEntries(in) ); } @@ -119,12 +119,12 @@ class WriteableProtoStreamArchiveTest { try (var tileWriter = archive.newTileWriter()) { tileWriter.write(tile4); } - archive.finish(maxMetadataIn); + archive.finish(maxMetadataDeserialized); } try (InputStream in = Files.newInputStream(csvFilePrimary)) { assertEquals( - List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)), + List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataSerialized)), readAllEntries(in) ); } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java new file mode 100644 index 00000000..33f16c90 --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java @@ -0,0 +1,36 @@ +package com.onthegomap.planetiler.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; + +class CloseableIteratorTest { + + @Test + void testFilter() { + assertEquals( + List.of(2, 4), + CloseableIterator.of(Stream.of(1, 2, 3, 4, 5, 6)) + .filter(i -> i == 2 || i == 4) + .stream() + .toList() + ); + assertEquals( + List.of(), + CloseableIterator.of(Stream.of(100, 99, 98)) + .filter(i -> i == 2 || i == 4) + .stream() + .toList() + ); + assertEquals( + List.of(), + CloseableIterator.of(Stream.of()) + .filter(i -> i == 2 || i == 4) + .stream() + .toList() + ); + } + +} diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index becc425b..ecdb3300 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -2,6 +2,7 @@ package com.onthegomap.planetiler; import static java.util.Map.entry; +import com.onthegomap.planetiler.archive.TileCopy; import com.onthegomap.planetiler.benchmarks.LongLongMapBench; import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping; import com.onthegomap.planetiler.custommap.ConfiguredMapMain; @@ -65,7 +66,9 @@ public class Main { entry("verify-monaco", VerifyMonaco::main), entry("stats", TileSizeStats::main), entry("top-osm-tiles", TopOsmTiles::main), - entry("compare", CompareArchives::main) + entry("compare", CompareArchives::main), + + entry("tile-copy", TileCopy::main) ); private static EntryPoint bundledSchema(String path) { From 19c834fb5d94260cad7b2d1768a7fb94cc24a38e Mon Sep 17 00:00:00 2001 From: bbilger Date: Sat, 6 Jan 2024 22:16:59 +0100 Subject: [PATCH 2/3] properly close the getAllTiles iterator --- .../planetiler/stream/ReadableStreamArchive.java | 14 +++++++++++--- .../java/com/onthegomap/planetiler/TestUtils.java | 4 +++- .../planetiler/files/ReadableFilesArchiveTest.java | 8 ++++++-- .../stream/ReadableCsvStreamArchiveTest.java | 8 ++++++-- .../stream/ReadableJsonStreamArchiveTest.java | 8 ++++++-- .../stream/ReadableProtoStreamArchiveTest.java | 9 +++++++-- 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java index f6f9ecff..2ea88215 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java @@ -26,7 +26,9 @@ abstract class ReadableStreamArchive implements ReadableTileArchive { @Override public final byte[] getTile(TileCoord coord) { - return getAllTiles().stream().filter(c -> c.coord().equals(coord)).map(Tile::bytes).findFirst().orElse(null); + try (var tiles = getAllTiles(); var s = tiles.stream()) { + return s.filter(c -> c.coord().equals(coord)).map(Tile::bytes).findFirst().orElse(null); + } } @Override @@ -34,11 +36,17 @@ abstract class ReadableStreamArchive implements ReadableTileArchive { return getTile(TileCoord.ofXYZ(x, y, z)); } + /** + * Callers MUST make sure to close the iterator/derived stream! + */ @Override public final CloseableIterator getAllTileCoords() { return getAllTiles().map(Tile::coord); } + /** + * Callers MUST make sure to close the iterator/derived stream! + */ @Override public final CloseableIterator getAllTiles() { return createIterator() @@ -53,8 +61,8 @@ abstract class ReadableStreamArchive implements ReadableTileArchive { } private TileArchiveMetadata loadMetadata() { - try (var it = createIterator()) { - return it.stream().map(this::mapEntryToMetadata).flatMap(Optional::stream).findFirst().orElse(null); + try (var i = createIterator(); var s = i.stream()) { + return s.map(this::mapEntryToMetadata).flatMap(Optional::stream).findFirst().orElse(null); } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 1d11e550..8d847228 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -294,7 +294,9 @@ public class TestUtils { } public static Set getTiles(ReadableTileArchive db) { - return db.getAllTiles().stream().collect(Collectors.toSet()); + try (var t = db.getAllTiles(); var s = t.stream()) { + return s.collect(Collectors.toUnmodifiableSet()); + } } public static int getTilesDataCount(Mbtiles db) throws SQLException { diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java index ad9d0038..127030bf 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java @@ -49,8 +49,12 @@ class ReadableFilesArchiveTest { Files.write(files.get(i), new byte[]{(byte) i}); } - try (var reader = ReadableFilesArchive.newReader(tilesDir, Arguments.of())) { - final List tiles = reader.getAllTiles().stream().sorted().toList(); + try ( + var reader = ReadableFilesArchive.newReader(tilesDir, Arguments.of()); + var t = reader.getAllTiles(); + var s = t.stream() + ) { + final List tiles = s.sorted().toList(); assertEquals( List.of( new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}), diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java index 31258995..0f08420a 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java @@ -63,8 +63,12 @@ class ReadableCsvStreamArchiveTest { ); try (var reader = ReadableCsvArchive.newReader(TileArchiveConfig.Format.CSV, csvFile, config)) { - assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); - assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + try (var s = reader.getAllTiles().stream()) { + assertEquals(expectedTiles, s.toList()); + } + try (var s = reader.getAllTiles().stream()) { + assertEquals(expectedTiles, s.toList()); + } assertNull(reader.metadata()); assertNull(reader.metadata()); assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3))); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java index ab728f8e..f95145b9 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java @@ -36,8 +36,12 @@ class ReadableJsonStreamArchiveTest { new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) ); try (var reader = ReadableJsonStreamArchive.newReader(jsonFile, config)) { - assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); - assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + try (var s = reader.getAllTiles().stream()) { + assertEquals(expectedTiles, s.toList()); + } + try (var s = reader.getAllTiles().stream()) { + assertEquals(expectedTiles, s.toList()); + } assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata()); assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata()); assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3))); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java index 7681f98e..6ffd685c 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java @@ -52,8 +52,13 @@ class ReadableProtoStreamArchiveTest { new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}) ); try (var reader = ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()))) { - assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); - assertEquals(expectedTiles, reader.getAllTiles().stream().toList()); + try (var s = reader.getAllTiles().stream()) { + assertEquals(expectedTiles, s.toList()); + } + try (var s = reader.getAllTiles().stream()) { + assertEquals(expectedTiles, s.toList()); + } + assertEquals(metadataDeserialized, reader.metadata()); assertEquals(metadataDeserialized, reader.metadata()); } } From be4994a44a668009912afa79917f77609b0f0fb5 Mon Sep 17 00:00:00 2001 From: bbilger Date: Tue, 23 Jan 2024 19:18:10 +0100 Subject: [PATCH 3/3] refactoring --- .../planetiler/archive/TileArchiveConfig.java | 25 +- .../planetiler/archive/TileCopy.java | 236 ------------------ .../planetiler/config/Arguments.java | 28 ++- .../onthegomap/planetiler/copy/TileCopy.java | 131 ++++++++++ .../planetiler/copy/TileCopyConfig.java | 108 ++++++++ .../planetiler/copy/TileCopyContext.java | 123 +++++++++ .../planetiler/copy/TileCopyWorkItem.java | 51 ++++ .../copy/TileCopyWorkItemGenerators.java | 147 +++++++++++ .../planetiler/copy/TileDataReEncoders.java | 31 +++ .../onthegomap/planetiler/geo/TileCoord.java | 6 +- .../stream/ReadableProtoStreamArchive.java | 2 +- .../com/onthegomap/planetiler/TestUtils.java | 6 +- .../{archive => copy}/TileCopyTest.java | 62 ++++- .../java/com/onthegomap/planetiler/Main.java | 2 +- 14 files changed, 690 insertions(+), 268 deletions(-) delete mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java create mode 100644 planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java rename planetiler-core/src/test/java/com/onthegomap/planetiler/{archive => copy}/TileCopyTest.java (80%) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java index 0e9f8f39..1abdf00b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -267,11 +267,11 @@ public record TileArchiveConfig( public enum Format { MBTILES("mbtiles", false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */, - false, TileOrder.TMS), - PMTILES("pmtiles", false, false, TileOrder.HILBERT), + false, false, TileOrder.TMS), + PMTILES("pmtiles", false, false, false, TileOrder.HILBERT), // should be before PBF in order to avoid collisions - FILES("files", true, true, TileOrder.TMS) { + FILES("files", true, true, true, TileOrder.TMS) { @Override boolean isUriSupported(URI uri) { final String path = uri.getPath(); @@ -280,25 +280,28 @@ public record TileArchiveConfig( } }, - CSV("csv", true, true, TileOrder.TMS), + CSV("csv", true, true, false, TileOrder.TMS), /** identical to {@link Format#CSV} - except for the column separator */ - TSV("tsv", true, true, TileOrder.TMS), + TSV("tsv", true, true, false, TileOrder.TMS), - PROTO("proto", true, true, TileOrder.TMS), + PROTO("proto", true, true, false, TileOrder.TMS), /** identical to {@link Format#PROTO} */ - PBF("pbf", true, true, TileOrder.TMS), + PBF("pbf", true, true, false, TileOrder.TMS), - JSON("json", true, true, TileOrder.TMS); + JSON("json", true, true, false, TileOrder.TMS); private final String id; private final boolean supportsAppend; private final boolean supportsConcurrentWrites; + private final boolean supportsConcurrentReads; private final TileOrder order; - Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, TileOrder order) { + Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, boolean supportsConcurrentReads, + TileOrder order) { this.id = id; this.supportsAppend = supportsAppend; this.supportsConcurrentWrites = supportsConcurrentWrites; + this.supportsConcurrentReads = supportsConcurrentReads; this.order = order; } @@ -318,6 +321,10 @@ public record TileArchiveConfig( return supportsConcurrentWrites; } + public boolean supportsConcurrentReads() { + return supportsConcurrentReads; + } + boolean isUriSupported(URI uri) { final String path = uri.getPath(); return path != null && path.endsWith("." + id); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java deleted file mode 100644 index 03d8a139..00000000 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java +++ /dev/null @@ -1,236 +0,0 @@ -package com.onthegomap.planetiler.archive; - -import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.CommonConfigs; -import com.onthegomap.planetiler.stats.Counter; -import com.onthegomap.planetiler.stats.ProcessInfo; -import com.onthegomap.planetiler.stats.ProgressLoggers; -import com.onthegomap.planetiler.stats.Stats; -import com.onthegomap.planetiler.util.Gzip; -import com.onthegomap.planetiler.util.Hashing; -import com.onthegomap.planetiler.worker.WorkerPipeline; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.OptionalLong; -import java.util.function.Function; -import java.util.function.UnaryOperator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility to copy/convert tiles and metadata from one archive into another. - *

- * Example usages: - * - * 

- * --input=tiles.mbtiles --output=tiles.mbtiles
- * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
- * --input=tiles.pmtiles --output=tiles.mbtiles
- * --input=tiles.mbtiles --output=tiles/
- * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
- * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
- * --input=tiles.mbtiles --output=tiles.proto
- *
- */ -public class TileCopy { - - private static final Logger LOGGER = LoggerFactory.getLogger(TileCopy.class); - - private final TileCopyConfig config; - - private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter(); - - - TileCopy(TileCopyConfig config) { - this.config = config; - } - - public void run() throws IOException { - - if (!config.inArchive().exists()) { - throw new IllegalArgumentException("the input archive does not exist"); - } - - config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads()); - - final var loggers = ProgressLoggers.create() - .addRateCounter("tiles", tilesWrittenOverall::get); - - try ( - var reader = TileArchives.newReader(config.inArchive(), config.inArguments()); - var writer = TileArchives.newWriter(config.outArchive(), config.outArguments()) - ) { - - final TileArchiveMetadata inMetadata = getInMetadata(reader); - final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata); - - writer.initialize(); - try ( - var rawTiles = reader.getAllTiles(); - var it = config.skipEmpty() ? rawTiles.filter(t -> t.bytes() != null && t.bytes().length > 0) : rawTiles - ) { - - final var tileConverter = tileConverter(inMetadata.tileCompression(), outMetadata.tileCompression(), writer); - var pipeline = WorkerPipeline.start("archive", config.stats()) - .readFrom("tiles", () -> it) - .addBuffer("buffer", config.queueSize()) - .sinkTo("write", config.tileWriterThreads(), itt -> tileWriter(writer, itt, tileConverter)); - - final var f = pipeline.done().thenRun(() -> writer.finish(outMetadata)); - - loggers.awaitAndLog(f, config.logInterval()); - } - } - } - - private void tileWriter(WriteableTileArchive archive, Iterable itt, - Function tileConverter) { - - final Counter tilesWritten = tilesWrittenOverall.counterForThread(); - try (var tileWriter = archive.newTileWriter()) { - for (Tile t : itt) { - tileWriter.write(tileConverter.apply(t)); - tilesWritten.inc(); - } - } - } - - private static Function tileConverter(TileCompression inCompression, - TileCompression outCompression, WriteableTileArchive writer) { - - final UnaryOperator bytesReEncoder = bytesReEncoder(inCompression, outCompression); - final Function hasher = - writer.deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) : - b -> OptionalLong.empty(); - - return t -> new TileEncodingResult(t.coord(), bytesReEncoder.apply(t.bytes()), hasher.apply(t.bytes())); - } - - private static UnaryOperator bytesReEncoder(TileCompression inCompression, TileCompression outCompression) { - if (inCompression == outCompression) { - return UnaryOperator.identity(); - } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) { - return Gzip::gunzip; - } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) { - return Gzip::gzip; - } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.GZIP) { - return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b); - } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.NONE) { - return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b; - } else { - throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression); - } - } - - private TileArchiveMetadata getInMetadata(ReadableTileArchive reader) { - TileArchiveMetadata inMetadata = config.inMetadata(); - if (inMetadata == null) { - inMetadata = reader.metadata(); - if (inMetadata == null) { - LOGGER.atWarn() - .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata"); - inMetadata = fallbackMetadata(); - } - } - if (inMetadata.tileCompression() == null) { - inMetadata = inMetadata.withTileCompression(config.inCompression()); - } - - return inMetadata; - } - - private TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata) { - if (config.outCompression() == TileCompression.UNKNWON && inMetadata.tileCompression() == TileCompression.UNKNWON) { - return inMetadata.withTileCompression(TileCompression.GZIP); - } else if (config.outCompression() != TileCompression.UNKNWON) { - return inMetadata.withTileCompression(config.outCompression()); - } else { - return inMetadata; - } - } - - private static TileArchiveMetadata fallbackMetadata() { - return new TileArchiveMetadata( - "unknown", - null, - null, - null, - null, - TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf - null, - null, - null, - null, - new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers - Map.of(), - null - ); - } - - record TileCopyConfig( - TileArchiveConfig inArchive, - TileArchiveConfig outArchive, - Arguments inArguments, - Arguments outArguments, - TileCompression inCompression, - TileCompression outCompression, - int tileWriterThreads, - Duration logInterval, - Stats stats, - int queueSize, - boolean append, - boolean force, - TileArchiveMetadata inMetadata, - boolean skipEmpty - ) { - static TileCopyConfig fromArguments(Arguments baseArguments) { - - final Arguments inArguments = baseArguments.withPrefix("in"); - final Arguments outArguments = baseArguments.withPrefix("out"); - final Arguments baseOrOutArguments = outArguments.orElse(baseArguments); - - final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null); - final TileArchiveMetadata inMetadata; - if (inMetadataPath != null) { - try { - inMetadata = - TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } else { - inMetadata = null; - } - - return new TileCopyConfig( - TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")), - TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")), - inArguments, - outArguments, - getTileCompressionArg(inArguments, "the input tile compression"), - getTileCompressionArg(outArguments, "the output tile compression"), - CommonConfigs.tileWriterThreads(baseOrOutArguments), - CommonConfigs.logInterval(baseArguments), - baseArguments.getStats(), - Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)), - CommonConfigs.appendToArchive(baseOrOutArguments), - CommonConfigs.force(baseOrOutArguments), - inMetadata, - baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", false) - ); - } - } - - private static TileCompression getTileCompressionArg(Arguments args, String description) { - return args.getObject("tile_compression", description, TileCompression.UNKNWON, - v -> TileCompression.findById(v).orElseThrow()); - } - - public static void main(String[] args) throws IOException { - new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run(); - } -} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java index 879e20e3..70281ff6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java @@ -45,11 +45,17 @@ public class Arguments { private final UnaryOperator provider; private final Supplier> keys; + private final String logKeyPrefix; private boolean silent = false; - private Arguments(UnaryOperator provider, Supplier> keys) { + private Arguments(UnaryOperator provider, Supplier> keys, String logKeyPrefix) { this.provider = provider; this.keys = keys; + this.logKeyPrefix = logKeyPrefix; + } + + private Arguments(UnaryOperator provider, Supplier> keys) { + this(provider, keys, ""); } /** @@ -65,7 +71,7 @@ public class Arguments { } static Arguments fromJvmProperties(UnaryOperator getter, Supplier> keys) { - return fromPrefixed(getter, keys, "planetiler", ".", false); + return fromPrefixed(getter, keys, "planetiler", ".", false, ""); } /** @@ -81,7 +87,7 @@ public class Arguments { } static Arguments fromEnvironment(UnaryOperator getter, Supplier> keys) { - return fromPrefixed(getter, keys, "PLANETILER", "_", true); + return fromPrefixed(getter, keys, "PLANETILER", "_", true, ""); } /** @@ -213,21 +219,22 @@ public class Arguments { } private static Arguments from(UnaryOperator provider, Supplier> rawKeys, - UnaryOperator forward, UnaryOperator reverse) { + UnaryOperator forward, UnaryOperator reverse, String logKeyPrefix) { Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> { String reversed = reverse.apply(key); return normalize(key).equals(normalize(reversed)) ? Stream.empty() : Stream.of(reversed); }).toList(); - return new Arguments(key -> provider.apply(forward.apply(key)), keys); + return new Arguments(key -> provider.apply(forward.apply(key)), keys, logKeyPrefix); } private static Arguments fromPrefixed(UnaryOperator provider, Supplier> keys, - String prefix, String separator, boolean uppperCase) { + String prefix, String separator, boolean uppperCase, String logKeyPrefix) { var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)), Pattern.CASE_INSENSITIVE); return from(provider, keys, key -> normalize(prefix + separator + key, separator, uppperCase), - key -> normalize(prefixRegex.matcher(key).replaceFirst("")) + key -> normalize(prefixRegex.matcher(key).replaceFirst("")), + logKeyPrefix ); } @@ -262,7 +269,8 @@ public class Arguments { () -> Stream.concat( other.keys.get().stream(), keys.get().stream() - ).distinct().toList() + ).distinct().toList(), + other.logKeyPrefix ); if (silent) { result.silence(); @@ -307,7 +315,7 @@ public class Arguments { protected void logArgValue(String key, String description, Object result) { if (!silent && LOGGER.isDebugEnabled()) { - LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description); + LOGGER.debug("argument: {}{}={} ({})", logKeyPrefix, key.replaceFirst("\\|.*$", ""), result, description); } } @@ -532,7 +540,7 @@ public class Arguments { * Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}. */ public Arguments withPrefix(String prefix) { - return fromPrefixed(provider, keys, prefix, "_", false); + return fromPrefixed(provider, keys, prefix, "_", false, logKeyPrefix + prefix + "_"); } /** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java new file mode 100644 index 00000000..3155a5ee --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java @@ -0,0 +1,131 @@ +package com.onthegomap.planetiler.copy; + +import static com.onthegomap.planetiler.worker.Worker.joinFutures; + +import com.onthegomap.planetiler.archive.TileArchives; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.stats.Counter; +import com.onthegomap.planetiler.stats.ProgressLoggers; +import com.onthegomap.planetiler.worker.WorkQueue; +import com.onthegomap.planetiler.worker.WorkerPipeline; +import java.io.IOException; +import java.time.Duration; + +/** + * Utility to copy/convert tiles and metadata from one archive into another. + *

+ * Example usages: + * + * 

+ * --input=tiles.mbtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
+ * --input=tiles.pmtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles/
+ * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
+ * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
+ * --input=tiles.mbtiles --output=tiles.proto
+ *
+ */ +public class TileCopy { + + private final TileCopyConfig config; + + private final Counter.MultiThreadCounter tilesReadOverall = Counter.newMultiThreadCounter(); + private final Counter.MultiThreadCounter tilesProcessedOverall = Counter.newMultiThreadCounter(); + private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter(); + + + TileCopy(TileCopyConfig config) { + this.config = config; + } + + public void run() throws IOException { + + if (!config.inArchive().exists()) { + throw new IllegalArgumentException("the input archive does not exist"); + } + + config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads()); + + final var loggers = ProgressLoggers.create() + .addRateCounter("tiles_read", tilesReadOverall::get) + .addRateCounter("tiles_processed", tilesProcessedOverall::get) + .addRateCounter("tiles_written", tilesWrittenOverall::get); + + try ( + var reader = TileArchives.newReader(config.inArchive(), config.inArguments()); + var writer = TileArchives.newWriter(config.outArchive(), config.outArguments()) + ) { + + final TileCopyContext context = TileCopyContext.create(reader, writer, config); + + final var pipeline = WorkerPipeline.start("copy", config.stats()); + final WorkQueue resultQueue = + new WorkQueue<>("results", config.workQueueCapacity(), config.workQueueMaxBatch(), config.stats()); + + try ( + var it = TileCopyWorkItemGenerators.create(context) + ) { + + writer.initialize(); + + final var readerBranch = pipeline + .fromGenerator("iterator", next -> { + try (resultQueue) { + while (it.hasNext()) { + final TileCopyWorkItem t = it.next(); + resultQueue.accept(t); // put to queue immediately => retain order + next.accept(t); + } + } + }) + .addBuffer("to_read", config.workQueueCapacity(), config.workQueueMaxBatch()) + .addWorker("read", config.tileReaderThreads(), (prev, next) -> { + final Counter tilesRead = tilesReadOverall.counterForThread(); + for (var item : prev) { + item.loadOriginalTileData(); + tilesRead.inc(); + next.accept(item); + } + }) + .addBuffer("to_process", config.workQueueCapacity(), config.workQueueMaxBatch()) + .sinkTo("process", config.tileReaderThreads(), prev -> { + final Counter tilesProcessed = tilesProcessedOverall.counterForThread(); + for (var item : prev) { + item.process(); + tilesProcessed.inc(); + } + }); + + final var writerBranch = pipeline.readFromQueue(resultQueue) + .sinkTo("write", config.tileWriterThreads(), prev -> { + final Counter tilesWritten = tilesWrittenOverall.counterForThread(); + try (var tileWriter = writer.newTileWriter()) { + for (var item : prev) { + var result = item.toTileEncodingResult(); + if (result.tileData() == null && config.skipEmpty()) { + continue; + } + tileWriter.write(result); + tilesWritten.inc(); + } + } + }); + + final var writerDone = writerBranch.done().thenRun(() -> writer.finish(context.outMetadata())); + final var readerDone = readerBranch.done(); + + loggers + .newLine() + .addPipelineStats(readerBranch) + .addPipelineStats(writerBranch); + + loggers.awaitAndLog(joinFutures(writerDone, readerDone), Duration.ofSeconds(1)); + } + } + } + + public static void main(String[] args) throws IOException { + new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run(); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java new file mode 100644 index 00000000..c54ca229 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java @@ -0,0 +1,108 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer; +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.CommonConfigs; +import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.stats.ProcessInfo; +import com.onthegomap.planetiler.stats.Stats; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import org.locationtech.jts.geom.Envelope; + +record TileCopyConfig( + TileArchiveConfig inArchive, + TileArchiveConfig outArchive, + Arguments inArguments, + Arguments outArguments, + TileCompression inCompression, + TileCompression outCompression, + int tileWriterThreads, + int tileReaderThreads, + int tileProcessorThreads, + Duration logInterval, + Stats stats, + int queueSize, + boolean append, + boolean force, + TileArchiveMetadata inMetadata, + boolean skipEmpty, + Envelope filterBounds, + int filterMinzoom, + int filterMaxzoom, + boolean scanTilesInOrder, + TileOrder outputTileOrder, + int workQueueCapacity, + int workQueueMaxBatch +) { + + TileCopyConfig { + if (tileReaderThreads > 1 && !inArchive.format().supportsConcurrentReads()) { + throw new IllegalArgumentException(inArchive.format().id() + " does not support concurrent reads"); + } + if (tileWriterThreads > 1 && !outArchive.format().supportsConcurrentReads()) { + throw new IllegalArgumentException(outArchive.format().id() + " does not support concurrent writes"); + } + if (filterMinzoom > filterMaxzoom) { + throw new IllegalArgumentException("require minzoom <= maxzoom"); + } + } + + static TileCopyConfig fromArguments(Arguments baseArguments) { + + final Arguments inArguments = baseArguments.withPrefix("in"); + final Arguments outArguments = baseArguments.withPrefix("out"); + final Arguments baseOrOutArguments = outArguments.orElse(baseArguments); + + final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null); + final TileArchiveMetadata inMetadata; + if (inMetadataPath != null) { + try { + inMetadata = + TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + inMetadata = null; + } + + return new TileCopyConfig( + TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")), + TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")), + inArguments, + outArguments, + getTileCompressionArg(inArguments, "the input tile compression"), + getTileCompressionArg(outArguments, "the output tile compression"), + CommonConfigs.tileWriterThreads(baseOrOutArguments), + baseArguments.getInteger("tile_read_threads", "number of threads used to read tile data", 1), + baseArguments.getInteger("tile_process_threads", "number of threads used to process tile data", + Math.max(1, Runtime.getRuntime().availableProcessors() - 2)), + CommonConfigs.logInterval(baseArguments), + baseArguments.getStats(), + Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)), + CommonConfigs.appendToArchive(baseOrOutArguments), + CommonConfigs.force(baseOrOutArguments), + inMetadata, + baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", true), + baseArguments.bounds("filter_bounds", "the bounds to filter"), + baseArguments.getInteger("filter_minzoom", "the min zoom", 0), + baseArguments.getInteger("filter_maxzoom", "the max zoom", 14), + baseArguments.getBoolean("scan_tiles_in_order", "output the tiles in the same order they are in the", true), + baseArguments.getObject("output_tile_order", "the output tile order (if not scanned)", null, + s -> s == null ? null : TileOrder.valueOf(s.toUpperCase())), + 100_000, + 100 + ); + } + + private static TileCompression getTileCompressionArg(Arguments args, String description) { + return args.getObject("tile_compression", description, TileCompression.UNKNOWN, + v -> TileCompression.findById(v).orElseThrow()); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java new file mode 100644 index 00000000..da13dd81 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java @@ -0,0 +1,123 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.archive.WriteableTileArchive; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; +import org.locationtech.jts.geom.Envelope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +record TileCopyContext( + TileArchiveMetadata inMetadata, + TileArchiveMetadata outMetadata, + TileCopyConfig config, + ReadableTileArchive reader, + WriteableTileArchive writer +) { + + private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyContext.class); + + TileCompression inputCompression() { + return inMetadata.tileCompression(); + } + + TileCompression outputCompression() { + return outMetadata.tileCompression(); + } + + static TileCopyContext create(ReadableTileArchive reader, WriteableTileArchive writer, TileCopyConfig config) { + final TileArchiveMetadata inMetadata = getInMetadata(reader, config); + final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata, config); + return new TileCopyContext( + inMetadata, + outMetadata, + config, + reader, + writer + ); + } + + private static TileArchiveMetadata getInMetadata(ReadableTileArchive inArchive, TileCopyConfig config) { + TileArchiveMetadata inMetadata = config.inMetadata(); + if (inMetadata == null) { + inMetadata = inArchive.metadata(); + if (inMetadata == null) { + LOGGER.atWarn() + .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata"); + inMetadata = fallbackMetadata(); + } + } + if (inMetadata.tileCompression() == null) { + inMetadata = inMetadata.withTileCompression(config.inCompression()); + } + + return inMetadata; + } + + private static TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata, TileCopyConfig config) { + + final TileCompression tileCompression; + if (config.outCompression() == TileCompression.UNKNOWN && inMetadata.tileCompression() == TileCompression.UNKNOWN) { + tileCompression = TileCompression.GZIP; + } else if (config.outCompression() != TileCompression.UNKNOWN) { + tileCompression = config.outCompression(); + } else { + tileCompression = inMetadata.tileCompression(); + } + + final Envelope bounds; + if (config.filterBounds() != null) { + bounds = config.filterBounds(); + } else if (inMetadata.bounds() != null) { + bounds = inMetadata.bounds(); + } else { + bounds = null; + } + + final int minzoom = Stream.of(inMetadata.minzoom(), config.filterMinzoom()).filter(Objects::nonNull) + .mapToInt(Integer::intValue).max().orElse(0); + + final int maxzoom = Stream.of(inMetadata.maxzoom(), config.filterMaxzoom()).filter(Objects::nonNull) + .mapToInt(Integer::intValue).min().orElse(0); + + return new TileArchiveMetadata( + inMetadata.name(), + inMetadata.description(), + inMetadata.attribution(), + inMetadata.version(), + inMetadata.type(), + inMetadata.format(), + bounds, + inMetadata.center(), + minzoom, + maxzoom, + inMetadata.json(), + inMetadata.others(), + tileCompression + ); + } + + private static TileArchiveMetadata fallbackMetadata() { + return new TileArchiveMetadata( + "unknown", + null, + null, + null, + null, + TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf + null, + null, + 0, + 14, + new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers + Map.of(), + null + ); + } + +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java new file mode 100644 index 00000000..505ea557 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java @@ -0,0 +1,51 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.geo.TileCoord; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +public class TileCopyWorkItem { + private final TileCoord coord; + private final Supplier originalTileDataLoader; + private final UnaryOperator reEncoder; + private final Function hasher; + private final CompletableFuture originalData = new CompletableFuture<>(); + private final CompletableFuture reEncodedData = new CompletableFuture<>(); + private final CompletableFuture reEncodedDataHash = new CompletableFuture<>(); + + TileCopyWorkItem( + TileCoord coord, + Supplier originalTileDataLoader, + UnaryOperator reEncoder, + Function hasher + ) { + this.coord = coord; + this.originalTileDataLoader = originalTileDataLoader; + this.reEncoder = reEncoder; + this.hasher = hasher; + } + + public TileCoord getCoord() { + return coord; + } + + void loadOriginalTileData() { + originalData.complete(originalTileDataLoader.get()); + } + + void process() throws ExecutionException, InterruptedException { + final var reEncoded = reEncoder.apply(originalData.get()); + final var hash = hasher.apply(reEncoded); + reEncodedData.complete(reEncoded); + reEncodedDataHash.complete(hash); + } + + TileEncodingResult toTileEncodingResult() throws ExecutionException, InterruptedException { + return new TileEncodingResult(coord, reEncodedData.get(), reEncodedDataHash.get()); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java new file mode 100644 index 00000000..db89b4f8 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java @@ -0,0 +1,147 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.config.Bounds; +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.geo.TileExtents; +import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.util.CloseableIterator; +import com.onthegomap.planetiler.util.Hashing; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; +import org.locationtech.jts.geom.Envelope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class TileCopyWorkItemGenerators { + + private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyWorkItemGenerators.class); + + private TileCopyWorkItemGenerators() {} + + @SuppressWarnings("java:S2095") + static CloseableIterator create(TileCopyContext context) { + + final TileArchiveConfig.Format inFormat = context.config().inArchive().format(); + final TileArchiveConfig.Format outFormat = context.config().outArchive().format(); + + final boolean inOrder; + if (outFormat == TileArchiveConfig.Format.FILES) { + inOrder = true; // always use the input formats native order when outputting files + } else if (inFormat == TileArchiveConfig.Format.FILES) { + inOrder = false; // never use the inOrder looper when using files since there's no order guarantee + } else { + inOrder = context.config().scanTilesInOrder(); + } + + final int minzoom = context.outMetadata().minzoom(); + final int maxzoom = context.outMetadata().maxzoom(); + final Envelope filterBounds = + context.outMetadata().bounds() == null ? Bounds.WORLD.world() : + GeoUtils.toWorldBounds(context.outMetadata().bounds()); + final var boundsFilter = TileExtents.computeFromWorldBounds(maxzoom, filterBounds); + final Predicate zoomFilter = i -> i.getCoord().z() >= minzoom && i.getCoord().z() <= maxzoom; + + final ProcessorArgs processorArgs = new ProcessorArgs( + TileDataReEncoders.create(context), + context.writer().deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) : + b -> OptionalLong.empty() + ); + + if (inOrder) { + CloseableIterator it = new EagerInOrder(context.reader().getAllTiles(), processorArgs) + .filter(zoomFilter); + if (!Objects.equals(context.inMetadata().bounds(), context.outMetadata().bounds())) { + it = it.filter(i -> boundsFilter.test(i.getCoord())); + } + return it; + + } else { + + final boolean warnPoorlySupported = switch (inFormat) { + case CSV, TSV, PROTO, PBF, JSON -> true; + case PMTILES, MBTILES, FILES -> false; + }; + if (warnPoorlySupported) { + LOGGER.atWarn().log("{} random access is very slow", inFormat.id()); + } + + final TileOrder tileOrder = Optional.ofNullable(context.config().outputTileOrder()) + .orElse(context.writer().tileOrder()); + + return new TileOrderLoop(tileOrder, minzoom, maxzoom, context.reader()::getTile, processorArgs) + .filter(i -> boundsFilter.test(i.getCoord())); + } + } + + private record EagerInOrder( + CloseableIterator it, + ProcessorArgs processorArgs + ) implements CloseableIterator { + + @Override + public void close() { + it.close(); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public TileCopyWorkItem next() { + if (!it.hasNext()) { + throw new NoSuchElementException(); + } + final Tile t = it.next(); + return new TileCopyWorkItem(t.coord(), t::bytes, processorArgs.reEncoder(), processorArgs.hasher()); + } + } + + private static class TileOrderLoop implements CloseableIterator { + + private final TileOrder tileOrder; + private final int max; + private final Function dataLoader; + private final ProcessorArgs processorArgs; + private int current; + + TileOrderLoop(TileOrder tileOrder, int minZoom, int maxZoom, Function dataLoader, + ProcessorArgs processorArgs) { + this.tileOrder = tileOrder; + this.current = TileCoord.startIndexForZoom(minZoom); + this.max = TileCoord.endIndexForZoom(maxZoom); + this.dataLoader = dataLoader; + this.processorArgs = processorArgs; + } + + @Override + public boolean hasNext() { + return current <= max; + } + + @Override + public TileCopyWorkItem next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final TileCoord c = tileOrder.decode(current++); + return new TileCopyWorkItem(c, () -> dataLoader.apply(c), processorArgs.reEncoder(), processorArgs.hasher()); + } + + @Override + public void close() { + // nothing to close + } + } + + private record ProcessorArgs(UnaryOperator reEncoder, Function hasher) {} +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java new file mode 100644 index 00000000..9ffcd366 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java @@ -0,0 +1,31 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.util.Gzip; +import java.util.function.UnaryOperator; + +final class TileDataReEncoders { + + private TileDataReEncoders() {} + + static UnaryOperator create(TileCopyContext c) { + // for now just one - but compose multiple as needed in the future (decompress, do something, compress) + return reCompressor(c.inputCompression(), c.outputCompression()); + } + + private static UnaryOperator reCompressor(TileCompression inCompression, TileCompression outCompression) { + if (inCompression == outCompression) { + return b -> b; + } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) { + return Gzip::gunzip; + } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) { + return Gzip::gzip; + } else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.GZIP) { + return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b); + } else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.NONE) { + return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b; + } else { + throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java index 6d6370a7..8cc45144 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java @@ -42,10 +42,14 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable< } } - private static int startIndexForZoom(int z) { + public static int startIndexForZoom(int z) { return ZOOM_START_INDEX[z]; } + public static int endIndexForZoom(int z) { + return ZOOM_START_INDEX[z] + (1 << z) * (1 << z) - 1; + } + private static int zoomForIndex(int idx) { for (int z = MAX_MAXZOOM; z >= 0; z--) { if (ZOOM_START_INDEX[z] <= idx) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java index 98fb9681..ff35e5ba 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java @@ -125,7 +125,7 @@ public class ReadableProtoStreamArchive extends ReadableStreamArchive TileCompression.UNKNWON; + case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNOWN; case TILE_COMPRESSION_GZIP -> TileCompression.GZIP; case TILE_COMPRESSION_NONE -> TileCompression.NONE; }; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 8d847228..bc30a637 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -80,7 +80,7 @@ public class TestUtils { public static final TileArchiveMetadata MAX_METADATA_DESERIALIZED = new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3), - new Coordinate(1.3, 3.7, 1.0), 2, 3, + new Coordinate(1.3, 3.7, 1.0), 0, 8, TileArchiveMetadata.TileArchiveMetadataJson.create( List.of( new LayerAttrStats.VectorLayer("vl0", @@ -102,8 +102,8 @@ public class TestUtils { "version":"version", "type":"type", "format":"format", - "minzoom":"2", - "maxzoom":"3", + "minzoom":"0", + "maxzoom":"8", "compression":"gzip", "bounds":"0,2,1,3", "center":"1.3,3.7,1", diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java similarity index 80% rename from planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java rename to planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java index c3e5b70d..983361a7 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.archive; +package com.onthegomap.planetiler.copy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -55,7 +55,7 @@ class TileCopyTest { "output", archiveOutPath.toString() )).orElse(Arguments.of(arguments)); - new TileCopy(TileCopy.TileCopyConfig.fromArguments(args)).run(); + new TileCopy(TileCopyConfig.fromArguments(args)).run(); if (archiveDataOut.contains("{")) { final List expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList(); @@ -142,28 +142,30 @@ class TileCopyTest { "csv to json - use fallback metadata", ARCHIVE_0_CSV_COMPRESSION_NONE, ARCHIVE_0_JSON_BASE.formatted( - "{\"name\":\"unknown\",\"format\":\"pbf\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"), + "{\"name\":\"unknown\",\"format\":\"pbf\",\"minzoom\":\"0\",\"maxzoom\":\"14\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"), Map.of("out_tile_compression", "none") ), argsOf( "csv to json - use external metadata", ARCHIVE_0_CSV_COMPRESSION_NONE, - ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"none\"}"), + ARCHIVE_0_JSON_BASE + .formatted("{\"name\":\"blub\",\"compression\":\"none\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"), Map.of("out_tile_compression", "none", "in_metadata", "blub") ), argsOf( "csv to json - null handling", replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), - replaceBase64(ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"gzip\"}"), "null") + replaceBase64(ARCHIVE_0_JSON_BASE + .formatted("{\"name\":\"blub\",\"compression\":\"gzip\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"), "null") .replace(",\"encodedData\":\"null\"", ""), - Map.of("in_metadata", "blub") + Map.of("in_metadata", "blub", "skip_empty", "false") ), argsOf( "json to csv - null handling", replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null") .replace(",\"encodedData\":\"null\"", ""), replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), - Map.of() + Map.of("skip_empty", "false") ), argsOf( "csv to csv - empty skipping on", @@ -187,6 +189,52 @@ class TileCopyTest { 1,2,3,AQ== """, Map.of("skip_empty", "false", "out_tile_compression", "none") + ), + argsOf( + "tiles in order", + """ + 1,2,3,AQ== + 0,0,0,AA== + """, + """ + 1,2,3,AQ== + 0,0,0,AA== + """, + Map.of("out_tile_compression", "none") + ), + argsOf( + "tiles re-order", + """ + 0,0,1,AQ== + 0,0,0,AA== + """, + """ + 0,0,0,AA== + 0,0,1,AQ== + """, + Map.of("out_tile_compression", "none", "scan_tiles_in_order", "false", "filter_maxzoom", "1") + ), + argsOf( + "filter min zoom", + """ + 0,0,0,AA== + 0,0,1,AQ== + """, + """ + 0,0,1,AQ== + """, + Map.of("out_tile_compression", "none", "filter_minzoom", "1") + ), + argsOf( + "filter max zoom", + """ + 0,0,1,AQ== + 0,0,0,AA== + """, + """ + 0,0,0,AA== + """, + Map.of("out_tile_compression", "none", "filter_maxzoom", "0") ) ); } diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index ecdb3300..0387eb1e 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -2,9 +2,9 @@ package com.onthegomap.planetiler; import static java.util.Map.entry; -import com.onthegomap.planetiler.archive.TileCopy; import com.onthegomap.planetiler.benchmarks.LongLongMapBench; import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping; +import com.onthegomap.planetiler.copy.TileCopy; import com.onthegomap.planetiler.custommap.ConfiguredMapMain; import com.onthegomap.planetiler.custommap.validator.SchemaValidator; import com.onthegomap.planetiler.examples.BikeRouteOverlay;