diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java index 0e9f8f39..1abdf00b 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java @@ -267,11 +267,11 @@ public record TileArchiveConfig( public enum Format { MBTILES("mbtiles", false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */, - false, TileOrder.TMS), - PMTILES("pmtiles", false, false, TileOrder.HILBERT), + false, false, TileOrder.TMS), + PMTILES("pmtiles", false, false, false, TileOrder.HILBERT), // should be before PBF in order to avoid collisions - FILES("files", true, true, TileOrder.TMS) { + FILES("files", true, true, true, TileOrder.TMS) { @Override boolean isUriSupported(URI uri) { final String path = uri.getPath(); @@ -280,25 +280,28 @@ public record TileArchiveConfig( } }, - CSV("csv", true, true, TileOrder.TMS), + CSV("csv", true, true, false, TileOrder.TMS), /** identical to {@link Format#CSV} - except for the column separator */ - TSV("tsv", true, true, TileOrder.TMS), + TSV("tsv", true, true, false, TileOrder.TMS), - PROTO("proto", true, true, TileOrder.TMS), + PROTO("proto", true, true, false, TileOrder.TMS), /** identical to {@link Format#PROTO} */ - PBF("pbf", true, true, TileOrder.TMS), + PBF("pbf", true, true, false, TileOrder.TMS), - JSON("json", true, true, TileOrder.TMS); + JSON("json", true, true, false, TileOrder.TMS); private final String id; private final boolean supportsAppend; private final boolean supportsConcurrentWrites; + private final boolean supportsConcurrentReads; private final TileOrder order; - Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, TileOrder order) { + Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, boolean supportsConcurrentReads, + TileOrder order) { this.id = id; this.supportsAppend = supportsAppend; this.supportsConcurrentWrites = supportsConcurrentWrites; + this.supportsConcurrentReads = supportsConcurrentReads; this.order = order; } @@ -318,6 +321,10 @@ public record TileArchiveConfig( return supportsConcurrentWrites; } + public boolean supportsConcurrentReads() { + return supportsConcurrentReads; + } + boolean isUriSupported(URI uri) { final String path = uri.getPath(); return path != null && path.endsWith("." + id); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java deleted file mode 100644 index 03d8a139..00000000 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java +++ /dev/null @@ -1,236 +0,0 @@ -package com.onthegomap.planetiler.archive; - -import com.onthegomap.planetiler.config.Arguments; -import com.onthegomap.planetiler.config.CommonConfigs; -import com.onthegomap.planetiler.stats.Counter; -import com.onthegomap.planetiler.stats.ProcessInfo; -import com.onthegomap.planetiler.stats.ProgressLoggers; -import com.onthegomap.planetiler.stats.Stats; -import com.onthegomap.planetiler.util.Gzip; -import com.onthegomap.planetiler.util.Hashing; -import com.onthegomap.planetiler.worker.WorkerPipeline; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.OptionalLong; -import java.util.function.Function; -import java.util.function.UnaryOperator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility to copy/convert tiles and metadata from one archive into another. - *

- * Example usages: - * - * 

- * --input=tiles.mbtiles --output=tiles.mbtiles
- * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
- * --input=tiles.pmtiles --output=tiles.mbtiles
- * --input=tiles.mbtiles --output=tiles/
- * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
- * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
- * --input=tiles.mbtiles --output=tiles.proto
- *
- */ -public class TileCopy { - - private static final Logger LOGGER = LoggerFactory.getLogger(TileCopy.class); - - private final TileCopyConfig config; - - private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter(); - - - TileCopy(TileCopyConfig config) { - this.config = config; - } - - public void run() throws IOException { - - if (!config.inArchive().exists()) { - throw new IllegalArgumentException("the input archive does not exist"); - } - - config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads()); - - final var loggers = ProgressLoggers.create() - .addRateCounter("tiles", tilesWrittenOverall::get); - - try ( - var reader = TileArchives.newReader(config.inArchive(), config.inArguments()); - var writer = TileArchives.newWriter(config.outArchive(), config.outArguments()) - ) { - - final TileArchiveMetadata inMetadata = getInMetadata(reader); - final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata); - - writer.initialize(); - try ( - var rawTiles = reader.getAllTiles(); - var it = config.skipEmpty() ? rawTiles.filter(t -> t.bytes() != null && t.bytes().length > 0) : rawTiles - ) { - - final var tileConverter = tileConverter(inMetadata.tileCompression(), outMetadata.tileCompression(), writer); - var pipeline = WorkerPipeline.start("archive", config.stats()) - .readFrom("tiles", () -> it) - .addBuffer("buffer", config.queueSize()) - .sinkTo("write", config.tileWriterThreads(), itt -> tileWriter(writer, itt, tileConverter)); - - final var f = pipeline.done().thenRun(() -> writer.finish(outMetadata)); - - loggers.awaitAndLog(f, config.logInterval()); - } - } - } - - private void tileWriter(WriteableTileArchive archive, Iterable itt, - Function tileConverter) { - - final Counter tilesWritten = tilesWrittenOverall.counterForThread(); - try (var tileWriter = archive.newTileWriter()) { - for (Tile t : itt) { - tileWriter.write(tileConverter.apply(t)); - tilesWritten.inc(); - } - } - } - - private static Function tileConverter(TileCompression inCompression, - TileCompression outCompression, WriteableTileArchive writer) { - - final UnaryOperator bytesReEncoder = bytesReEncoder(inCompression, outCompression); - final Function hasher = - writer.deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) : - b -> OptionalLong.empty(); - - return t -> new TileEncodingResult(t.coord(), bytesReEncoder.apply(t.bytes()), hasher.apply(t.bytes())); - } - - private static UnaryOperator bytesReEncoder(TileCompression inCompression, TileCompression outCompression) { - if (inCompression == outCompression) { - return UnaryOperator.identity(); - } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) { - return Gzip::gunzip; - } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) { - return Gzip::gzip; - } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.GZIP) { - return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b); - } else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.NONE) { - return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b; - } else { - throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression); - } - } - - private TileArchiveMetadata getInMetadata(ReadableTileArchive reader) { - TileArchiveMetadata inMetadata = config.inMetadata(); - if (inMetadata == null) { - inMetadata = reader.metadata(); - if (inMetadata == null) { - LOGGER.atWarn() - .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata"); - inMetadata = fallbackMetadata(); - } - } - if (inMetadata.tileCompression() == null) { - inMetadata = inMetadata.withTileCompression(config.inCompression()); - } - - return inMetadata; - } - - private TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata) { - if (config.outCompression() == TileCompression.UNKNWON && inMetadata.tileCompression() == TileCompression.UNKNWON) { - return inMetadata.withTileCompression(TileCompression.GZIP); - } else if (config.outCompression() != TileCompression.UNKNWON) { - return inMetadata.withTileCompression(config.outCompression()); - } else { - return inMetadata; - } - } - - private static TileArchiveMetadata fallbackMetadata() { - return new TileArchiveMetadata( - "unknown", - null, - null, - null, - null, - TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf - null, - null, - null, - null, - new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers - Map.of(), - null - ); - } - - record TileCopyConfig( - TileArchiveConfig inArchive, - TileArchiveConfig outArchive, - Arguments inArguments, - Arguments outArguments, - TileCompression inCompression, - TileCompression outCompression, - int tileWriterThreads, - Duration logInterval, - Stats stats, - int queueSize, - boolean append, - boolean force, - TileArchiveMetadata inMetadata, - boolean skipEmpty - ) { - static TileCopyConfig fromArguments(Arguments baseArguments) { - - final Arguments inArguments = baseArguments.withPrefix("in"); - final Arguments outArguments = baseArguments.withPrefix("out"); - final Arguments baseOrOutArguments = outArguments.orElse(baseArguments); - - final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null); - final TileArchiveMetadata inMetadata; - if (inMetadataPath != null) { - try { - inMetadata = - TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } else { - inMetadata = null; - } - - return new TileCopyConfig( - TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")), - TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")), - inArguments, - outArguments, - getTileCompressionArg(inArguments, "the input tile compression"), - getTileCompressionArg(outArguments, "the output tile compression"), - CommonConfigs.tileWriterThreads(baseOrOutArguments), - CommonConfigs.logInterval(baseArguments), - baseArguments.getStats(), - Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)), - CommonConfigs.appendToArchive(baseOrOutArguments), - CommonConfigs.force(baseOrOutArguments), - inMetadata, - baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", false) - ); - } - } - - private static TileCompression getTileCompressionArg(Arguments args, String description) { - return args.getObject("tile_compression", description, TileCompression.UNKNWON, - v -> TileCompression.findById(v).orElseThrow()); - } - - public static void main(String[] args) throws IOException { - new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run(); - } -} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java index 879e20e3..70281ff6 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java @@ -45,11 +45,17 @@ public class Arguments { private final UnaryOperator provider; private final Supplier> keys; + private final String logKeyPrefix; private boolean silent = false; - private Arguments(UnaryOperator provider, Supplier> keys) { + private Arguments(UnaryOperator provider, Supplier> keys, String logKeyPrefix) { this.provider = provider; this.keys = keys; + this.logKeyPrefix = logKeyPrefix; + } + + private Arguments(UnaryOperator provider, Supplier> keys) { + this(provider, keys, ""); } /** @@ -65,7 +71,7 @@ public class Arguments { } static Arguments fromJvmProperties(UnaryOperator getter, Supplier> keys) { - return fromPrefixed(getter, keys, "planetiler", ".", false); + return fromPrefixed(getter, keys, "planetiler", ".", false, ""); } /** @@ -81,7 +87,7 @@ public class Arguments { } static Arguments fromEnvironment(UnaryOperator getter, Supplier> keys) { - return fromPrefixed(getter, keys, "PLANETILER", "_", true); + return fromPrefixed(getter, keys, "PLANETILER", "_", true, ""); } /** @@ -213,21 +219,22 @@ public class Arguments { } private static Arguments from(UnaryOperator provider, Supplier> rawKeys, - UnaryOperator forward, UnaryOperator reverse) { + UnaryOperator forward, UnaryOperator reverse, String logKeyPrefix) { Supplier> keys = () -> rawKeys.get().stream().flatMap(key -> { String reversed = reverse.apply(key); return normalize(key).equals(normalize(reversed)) ? Stream.empty() : Stream.of(reversed); }).toList(); - return new Arguments(key -> provider.apply(forward.apply(key)), keys); + return new Arguments(key -> provider.apply(forward.apply(key)), keys, logKeyPrefix); } private static Arguments fromPrefixed(UnaryOperator provider, Supplier> keys, - String prefix, String separator, boolean uppperCase) { + String prefix, String separator, boolean uppperCase, String logKeyPrefix) { var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)), Pattern.CASE_INSENSITIVE); return from(provider, keys, key -> normalize(prefix + separator + key, separator, uppperCase), - key -> normalize(prefixRegex.matcher(key).replaceFirst("")) + key -> normalize(prefixRegex.matcher(key).replaceFirst("")), + logKeyPrefix ); } @@ -262,7 +269,8 @@ public class Arguments { () -> Stream.concat( other.keys.get().stream(), keys.get().stream() - ).distinct().toList() + ).distinct().toList(), + other.logKeyPrefix ); if (silent) { result.silence(); @@ -307,7 +315,7 @@ public class Arguments { protected void logArgValue(String key, String description, Object result) { if (!silent && LOGGER.isDebugEnabled()) { - LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description); + LOGGER.debug("argument: {}{}={} ({})", logKeyPrefix, key.replaceFirst("\\|.*$", ""), result, description); } } @@ -532,7 +540,7 @@ public class Arguments { * Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}. */ public Arguments withPrefix(String prefix) { - return fromPrefixed(provider, keys, prefix, "_", false); + return fromPrefixed(provider, keys, prefix, "_", false, logKeyPrefix + prefix + "_"); } /** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */ diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java new file mode 100644 index 00000000..3155a5ee --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java @@ -0,0 +1,131 @@ +package com.onthegomap.planetiler.copy; + +import static com.onthegomap.planetiler.worker.Worker.joinFutures; + +import com.onthegomap.planetiler.archive.TileArchives; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.stats.Counter; +import com.onthegomap.planetiler.stats.ProgressLoggers; +import com.onthegomap.planetiler.worker.WorkQueue; +import com.onthegomap.planetiler.worker.WorkerPipeline; +import java.io.IOException; +import java.time.Duration; + +/** + * Utility to copy/convert tiles and metadata from one archive into another. + *

+ * Example usages: + * + * 

+ * --input=tiles.mbtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
+ * --input=tiles.pmtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles/
+ * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
+ * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
+ * --input=tiles.mbtiles --output=tiles.proto
+ *
+ */ +public class TileCopy { + + private final TileCopyConfig config; + + private final Counter.MultiThreadCounter tilesReadOverall = Counter.newMultiThreadCounter(); + private final Counter.MultiThreadCounter tilesProcessedOverall = Counter.newMultiThreadCounter(); + private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter(); + + + TileCopy(TileCopyConfig config) { + this.config = config; + } + + public void run() throws IOException { + + if (!config.inArchive().exists()) { + throw new IllegalArgumentException("the input archive does not exist"); + } + + config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads()); + + final var loggers = ProgressLoggers.create() + .addRateCounter("tiles_read", tilesReadOverall::get) + .addRateCounter("tiles_processed", tilesProcessedOverall::get) + .addRateCounter("tiles_written", tilesWrittenOverall::get); + + try ( + var reader = TileArchives.newReader(config.inArchive(), config.inArguments()); + var writer = TileArchives.newWriter(config.outArchive(), config.outArguments()) + ) { + + final TileCopyContext context = TileCopyContext.create(reader, writer, config); + + final var pipeline = WorkerPipeline.start("copy", config.stats()); + final WorkQueue resultQueue = + new WorkQueue<>("results", config.workQueueCapacity(), config.workQueueMaxBatch(), config.stats()); + + try ( + var it = TileCopyWorkItemGenerators.create(context) + ) { + + writer.initialize(); + + final var readerBranch = pipeline + .fromGenerator("iterator", next -> { + try (resultQueue) { + while (it.hasNext()) { + final TileCopyWorkItem t = it.next(); + resultQueue.accept(t); // put to queue immediately => retain order + next.accept(t); + } + } + }) + .addBuffer("to_read", config.workQueueCapacity(), config.workQueueMaxBatch()) + .addWorker("read", config.tileReaderThreads(), (prev, next) -> { + final Counter tilesRead = tilesReadOverall.counterForThread(); + for (var item : prev) { + item.loadOriginalTileData(); + tilesRead.inc(); + next.accept(item); + } + }) + .addBuffer("to_process", config.workQueueCapacity(), config.workQueueMaxBatch()) + .sinkTo("process", config.tileReaderThreads(), prev -> { + final Counter tilesProcessed = tilesProcessedOverall.counterForThread(); + for (var item : prev) { + item.process(); + tilesProcessed.inc(); + } + }); + + final var writerBranch = pipeline.readFromQueue(resultQueue) + .sinkTo("write", config.tileWriterThreads(), prev -> { + final Counter tilesWritten = tilesWrittenOverall.counterForThread(); + try (var tileWriter = writer.newTileWriter()) { + for (var item : prev) { + var result = item.toTileEncodingResult(); + if (result.tileData() == null && config.skipEmpty()) { + continue; + } + tileWriter.write(result); + tilesWritten.inc(); + } + } + }); + + final var writerDone = writerBranch.done().thenRun(() -> writer.finish(context.outMetadata())); + final var readerDone = readerBranch.done(); + + loggers + .newLine() + .addPipelineStats(readerBranch) + .addPipelineStats(writerBranch); + + loggers.awaitAndLog(joinFutures(writerDone, readerDone), Duration.ofSeconds(1)); + } + } + } + + public static void main(String[] args) throws IOException { + new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run(); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java new file mode 100644 index 00000000..c54ca229 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java @@ -0,0 +1,108 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer; +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.config.Arguments; +import com.onthegomap.planetiler.config.CommonConfigs; +import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.stats.ProcessInfo; +import com.onthegomap.planetiler.stats.Stats; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import org.locationtech.jts.geom.Envelope; + +record TileCopyConfig( + TileArchiveConfig inArchive, + TileArchiveConfig outArchive, + Arguments inArguments, + Arguments outArguments, + TileCompression inCompression, + TileCompression outCompression, + int tileWriterThreads, + int tileReaderThreads, + int tileProcessorThreads, + Duration logInterval, + Stats stats, + int queueSize, + boolean append, + boolean force, + TileArchiveMetadata inMetadata, + boolean skipEmpty, + Envelope filterBounds, + int filterMinzoom, + int filterMaxzoom, + boolean scanTilesInOrder, + TileOrder outputTileOrder, + int workQueueCapacity, + int workQueueMaxBatch +) { + + TileCopyConfig { + if (tileReaderThreads > 1 && !inArchive.format().supportsConcurrentReads()) { + throw new IllegalArgumentException(inArchive.format().id() + " does not support concurrent reads"); + } + if (tileWriterThreads > 1 && !outArchive.format().supportsConcurrentReads()) { + throw new IllegalArgumentException(outArchive.format().id() + " does not support concurrent writes"); + } + if (filterMinzoom > filterMaxzoom) { + throw new IllegalArgumentException("require minzoom <= maxzoom"); + } + } + + static TileCopyConfig fromArguments(Arguments baseArguments) { + + final Arguments inArguments = baseArguments.withPrefix("in"); + final Arguments outArguments = baseArguments.withPrefix("out"); + final Arguments baseOrOutArguments = outArguments.orElse(baseArguments); + + final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null); + final TileArchiveMetadata inMetadata; + if (inMetadataPath != null) { + try { + inMetadata = + TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + inMetadata = null; + } + + return new TileCopyConfig( + TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")), + TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")), + inArguments, + outArguments, + getTileCompressionArg(inArguments, "the input tile compression"), + getTileCompressionArg(outArguments, "the output tile compression"), + CommonConfigs.tileWriterThreads(baseOrOutArguments), + baseArguments.getInteger("tile_read_threads", "number of threads used to read tile data", 1), + baseArguments.getInteger("tile_process_threads", "number of threads used to process tile data", + Math.max(1, Runtime.getRuntime().availableProcessors() - 2)), + CommonConfigs.logInterval(baseArguments), + baseArguments.getStats(), + Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)), + CommonConfigs.appendToArchive(baseOrOutArguments), + CommonConfigs.force(baseOrOutArguments), + inMetadata, + baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", true), + baseArguments.bounds("filter_bounds", "the bounds to filter"), + baseArguments.getInteger("filter_minzoom", "the min zoom", 0), + baseArguments.getInteger("filter_maxzoom", "the max zoom", 14), + baseArguments.getBoolean("scan_tiles_in_order", "output the tiles in the same order they are in the", true), + baseArguments.getObject("output_tile_order", "the output tile order (if not scanned)", null, + s -> s == null ? null : TileOrder.valueOf(s.toUpperCase())), + 100_000, + 100 + ); + } + + private static TileCompression getTileCompressionArg(Arguments args, String description) { + return args.getObject("tile_compression", description, TileCompression.UNKNOWN, + v -> TileCompression.findById(v).orElseThrow()); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java new file mode 100644 index 00000000..da13dd81 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java @@ -0,0 +1,123 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.ReadableTileArchive; +import com.onthegomap.planetiler.archive.TileArchiveMetadata; +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.archive.WriteableTileArchive; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; +import org.locationtech.jts.geom.Envelope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +record TileCopyContext( + TileArchiveMetadata inMetadata, + TileArchiveMetadata outMetadata, + TileCopyConfig config, + ReadableTileArchive reader, + WriteableTileArchive writer +) { + + private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyContext.class); + + TileCompression inputCompression() { + return inMetadata.tileCompression(); + } + + TileCompression outputCompression() { + return outMetadata.tileCompression(); + } + + static TileCopyContext create(ReadableTileArchive reader, WriteableTileArchive writer, TileCopyConfig config) { + final TileArchiveMetadata inMetadata = getInMetadata(reader, config); + final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata, config); + return new TileCopyContext( + inMetadata, + outMetadata, + config, + reader, + writer + ); + } + + private static TileArchiveMetadata getInMetadata(ReadableTileArchive inArchive, TileCopyConfig config) { + TileArchiveMetadata inMetadata = config.inMetadata(); + if (inMetadata == null) { + inMetadata = inArchive.metadata(); + if (inMetadata == null) { + LOGGER.atWarn() + .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata"); + inMetadata = fallbackMetadata(); + } + } + if (inMetadata.tileCompression() == null) { + inMetadata = inMetadata.withTileCompression(config.inCompression()); + } + + return inMetadata; + } + + private static TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata, TileCopyConfig config) { + + final TileCompression tileCompression; + if (config.outCompression() == TileCompression.UNKNOWN && inMetadata.tileCompression() == TileCompression.UNKNOWN) { + tileCompression = TileCompression.GZIP; + } else if (config.outCompression() != TileCompression.UNKNOWN) { + tileCompression = config.outCompression(); + } else { + tileCompression = inMetadata.tileCompression(); + } + + final Envelope bounds; + if (config.filterBounds() != null) { + bounds = config.filterBounds(); + } else if (inMetadata.bounds() != null) { + bounds = inMetadata.bounds(); + } else { + bounds = null; + } + + final int minzoom = Stream.of(inMetadata.minzoom(), config.filterMinzoom()).filter(Objects::nonNull) + .mapToInt(Integer::intValue).max().orElse(0); + + final int maxzoom = Stream.of(inMetadata.maxzoom(), config.filterMaxzoom()).filter(Objects::nonNull) + .mapToInt(Integer::intValue).min().orElse(0); + + return new TileArchiveMetadata( + inMetadata.name(), + inMetadata.description(), + inMetadata.attribution(), + inMetadata.version(), + inMetadata.type(), + inMetadata.format(), + bounds, + inMetadata.center(), + minzoom, + maxzoom, + inMetadata.json(), + inMetadata.others(), + tileCompression + ); + } + + private static TileArchiveMetadata fallbackMetadata() { + return new TileArchiveMetadata( + "unknown", + null, + null, + null, + null, + TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf + null, + null, + 0, + 14, + new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers + Map.of(), + null + ); + } + +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java new file mode 100644 index 00000000..505ea557 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java @@ -0,0 +1,51 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.TileEncodingResult; +import com.onthegomap.planetiler.geo.TileCoord; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +public class TileCopyWorkItem { + private final TileCoord coord; + private final Supplier originalTileDataLoader; + private final UnaryOperator reEncoder; + private final Function hasher; + private final CompletableFuture originalData = new CompletableFuture<>(); + private final CompletableFuture reEncodedData = new CompletableFuture<>(); + private final CompletableFuture reEncodedDataHash = new CompletableFuture<>(); + + TileCopyWorkItem( + TileCoord coord, + Supplier originalTileDataLoader, + UnaryOperator reEncoder, + Function hasher + ) { + this.coord = coord; + this.originalTileDataLoader = originalTileDataLoader; + this.reEncoder = reEncoder; + this.hasher = hasher; + } + + public TileCoord getCoord() { + return coord; + } + + void loadOriginalTileData() { + originalData.complete(originalTileDataLoader.get()); + } + + void process() throws ExecutionException, InterruptedException { + final var reEncoded = reEncoder.apply(originalData.get()); + final var hash = hasher.apply(reEncoded); + reEncodedData.complete(reEncoded); + reEncodedDataHash.complete(hash); + } + + TileEncodingResult toTileEncodingResult() throws ExecutionException, InterruptedException { + return new TileEncodingResult(coord, reEncodedData.get(), reEncodedDataHash.get()); + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java new file mode 100644 index 00000000..db89b4f8 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java @@ -0,0 +1,147 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.Tile; +import com.onthegomap.planetiler.archive.TileArchiveConfig; +import com.onthegomap.planetiler.config.Bounds; +import com.onthegomap.planetiler.geo.GeoUtils; +import com.onthegomap.planetiler.geo.TileCoord; +import com.onthegomap.planetiler.geo.TileExtents; +import com.onthegomap.planetiler.geo.TileOrder; +import com.onthegomap.planetiler.util.CloseableIterator; +import com.onthegomap.planetiler.util.Hashing; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; +import org.locationtech.jts.geom.Envelope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class TileCopyWorkItemGenerators { + + private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyWorkItemGenerators.class); + + private TileCopyWorkItemGenerators() {} + + @SuppressWarnings("java:S2095") + static CloseableIterator create(TileCopyContext context) { + + final TileArchiveConfig.Format inFormat = context.config().inArchive().format(); + final TileArchiveConfig.Format outFormat = context.config().outArchive().format(); + + final boolean inOrder; + if (outFormat == TileArchiveConfig.Format.FILES) { + inOrder = true; // always use the input formats native order when outputting files + } else if (inFormat == TileArchiveConfig.Format.FILES) { + inOrder = false; // never use the inOrder looper when using files since there's no order guarantee + } else { + inOrder = context.config().scanTilesInOrder(); + } + + final int minzoom = context.outMetadata().minzoom(); + final int maxzoom = context.outMetadata().maxzoom(); + final Envelope filterBounds = + context.outMetadata().bounds() == null ? Bounds.WORLD.world() : + GeoUtils.toWorldBounds(context.outMetadata().bounds()); + final var boundsFilter = TileExtents.computeFromWorldBounds(maxzoom, filterBounds); + final Predicate zoomFilter = i -> i.getCoord().z() >= minzoom && i.getCoord().z() <= maxzoom; + + final ProcessorArgs processorArgs = new ProcessorArgs( + TileDataReEncoders.create(context), + context.writer().deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) : + b -> OptionalLong.empty() + ); + + if (inOrder) { + CloseableIterator it = new EagerInOrder(context.reader().getAllTiles(), processorArgs) + .filter(zoomFilter); + if (!Objects.equals(context.inMetadata().bounds(), context.outMetadata().bounds())) { + it = it.filter(i -> boundsFilter.test(i.getCoord())); + } + return it; + + } else { + + final boolean warnPoorlySupported = switch (inFormat) { + case CSV, TSV, PROTO, PBF, JSON -> true; + case PMTILES, MBTILES, FILES -> false; + }; + if (warnPoorlySupported) { + LOGGER.atWarn().log("{} random access is very slow", inFormat.id()); + } + + final TileOrder tileOrder = Optional.ofNullable(context.config().outputTileOrder()) + .orElse(context.writer().tileOrder()); + + return new TileOrderLoop(tileOrder, minzoom, maxzoom, context.reader()::getTile, processorArgs) + .filter(i -> boundsFilter.test(i.getCoord())); + } + } + + private record EagerInOrder( + CloseableIterator it, + ProcessorArgs processorArgs + ) implements CloseableIterator { + + @Override + public void close() { + it.close(); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public TileCopyWorkItem next() { + if (!it.hasNext()) { + throw new NoSuchElementException(); + } + final Tile t = it.next(); + return new TileCopyWorkItem(t.coord(), t::bytes, processorArgs.reEncoder(), processorArgs.hasher()); + } + } + + private static class TileOrderLoop implements CloseableIterator { + + private final TileOrder tileOrder; + private final int max; + private final Function dataLoader; + private final ProcessorArgs processorArgs; + private int current; + + TileOrderLoop(TileOrder tileOrder, int minZoom, int maxZoom, Function dataLoader, + ProcessorArgs processorArgs) { + this.tileOrder = tileOrder; + this.current = TileCoord.startIndexForZoom(minZoom); + this.max = TileCoord.endIndexForZoom(maxZoom); + this.dataLoader = dataLoader; + this.processorArgs = processorArgs; + } + + @Override + public boolean hasNext() { + return current <= max; + } + + @Override + public TileCopyWorkItem next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + final TileCoord c = tileOrder.decode(current++); + return new TileCopyWorkItem(c, () -> dataLoader.apply(c), processorArgs.reEncoder(), processorArgs.hasher()); + } + + @Override + public void close() { + // nothing to close + } + } + + private record ProcessorArgs(UnaryOperator reEncoder, Function hasher) {} +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java new file mode 100644 index 00000000..9ffcd366 --- /dev/null +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java @@ -0,0 +1,31 @@ +package com.onthegomap.planetiler.copy; + +import com.onthegomap.planetiler.archive.TileCompression; +import com.onthegomap.planetiler.util.Gzip; +import java.util.function.UnaryOperator; + +final class TileDataReEncoders { + + private TileDataReEncoders() {} + + static UnaryOperator create(TileCopyContext c) { + // for now just one - but compose multiple as needed in the future (decompress, do something, compress) + return reCompressor(c.inputCompression(), c.outputCompression()); + } + + private static UnaryOperator reCompressor(TileCompression inCompression, TileCompression outCompression) { + if (inCompression == outCompression) { + return b -> b; + } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) { + return Gzip::gunzip; + } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) { + return Gzip::gzip; + } else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.GZIP) { + return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b); + } else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.NONE) { + return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b; + } else { + throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression); + } + } +} diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java index 6d6370a7..8cc45144 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java @@ -42,10 +42,14 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable< } } - private static int startIndexForZoom(int z) { + public static int startIndexForZoom(int z) { return ZOOM_START_INDEX[z]; } + public static int endIndexForZoom(int z) { + return ZOOM_START_INDEX[z] + (1 << z) * (1 << z) - 1; + } + private static int zoomForIndex(int idx) { for (int z = MAX_MAXZOOM; z >= 0; z--) { if (ZOOM_START_INDEX[z] <= idx) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java index 98fb9681..ff35e5ba 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java @@ -125,7 +125,7 @@ public class ReadableProtoStreamArchive extends ReadableStreamArchive TileCompression.UNKNWON; + case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNOWN; case TILE_COMPRESSION_GZIP -> TileCompression.GZIP; case TILE_COMPRESSION_NONE -> TileCompression.NONE; }; diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java index 8d847228..bc30a637 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java @@ -80,7 +80,7 @@ public class TestUtils { public static final TileArchiveMetadata MAX_METADATA_DESERIALIZED = new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3), - new Coordinate(1.3, 3.7, 1.0), 2, 3, + new Coordinate(1.3, 3.7, 1.0), 0, 8, TileArchiveMetadata.TileArchiveMetadataJson.create( List.of( new LayerAttrStats.VectorLayer("vl0", @@ -102,8 +102,8 @@ public class TestUtils { "version":"version", "type":"type", "format":"format", - "minzoom":"2", - "maxzoom":"3", + "minzoom":"0", + "maxzoom":"8", "compression":"gzip", "bounds":"0,2,1,3", "center":"1.3,3.7,1", diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java similarity index 80% rename from planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java rename to planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java index c3e5b70d..983361a7 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java @@ -1,4 +1,4 @@ -package com.onthegomap.planetiler.archive; +package com.onthegomap.planetiler.copy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -55,7 +55,7 @@ class TileCopyTest { "output", archiveOutPath.toString() )).orElse(Arguments.of(arguments)); - new TileCopy(TileCopy.TileCopyConfig.fromArguments(args)).run(); + new TileCopy(TileCopyConfig.fromArguments(args)).run(); if (archiveDataOut.contains("{")) { final List expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList(); @@ -142,28 +142,30 @@ class TileCopyTest { "csv to json - use fallback metadata", ARCHIVE_0_CSV_COMPRESSION_NONE, ARCHIVE_0_JSON_BASE.formatted( - "{\"name\":\"unknown\",\"format\":\"pbf\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"), + "{\"name\":\"unknown\",\"format\":\"pbf\",\"minzoom\":\"0\",\"maxzoom\":\"14\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"), Map.of("out_tile_compression", "none") ), argsOf( "csv to json - use external metadata", ARCHIVE_0_CSV_COMPRESSION_NONE, - ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"none\"}"), + ARCHIVE_0_JSON_BASE + .formatted("{\"name\":\"blub\",\"compression\":\"none\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"), Map.of("out_tile_compression", "none", "in_metadata", "blub") ), argsOf( "csv to json - null handling", replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), - replaceBase64(ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"gzip\"}"), "null") + replaceBase64(ARCHIVE_0_JSON_BASE + .formatted("{\"name\":\"blub\",\"compression\":\"gzip\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"), "null") .replace(",\"encodedData\":\"null\"", ""), - Map.of("in_metadata", "blub") + Map.of("in_metadata", "blub", "skip_empty", "false") ), argsOf( "json to csv - null handling", replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null") .replace(",\"encodedData\":\"null\"", ""), replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""), - Map.of() + Map.of("skip_empty", "false") ), argsOf( "csv to csv - empty skipping on", @@ -187,6 +189,52 @@ class TileCopyTest { 1,2,3,AQ== """, Map.of("skip_empty", "false", "out_tile_compression", "none") + ), + argsOf( + "tiles in order", + """ + 1,2,3,AQ== + 0,0,0,AA== + """, + """ + 1,2,3,AQ== + 0,0,0,AA== + """, + Map.of("out_tile_compression", "none") + ), + argsOf( + "tiles re-order", + """ + 0,0,1,AQ== + 0,0,0,AA== + """, + """ + 0,0,0,AA== + 0,0,1,AQ== + """, + Map.of("out_tile_compression", "none", "scan_tiles_in_order", "false", "filter_maxzoom", "1") + ), + argsOf( + "filter min zoom", + """ + 0,0,0,AA== + 0,0,1,AQ== + """, + """ + 0,0,1,AQ== + """, + Map.of("out_tile_compression", "none", "filter_minzoom", "1") + ), + argsOf( + "filter max zoom", + """ + 0,0,1,AQ== + 0,0,0,AA== + """, + """ + 0,0,0,AA== + """, + Map.of("out_tile_compression", "none", "filter_maxzoom", "0") ) ); } diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java index ecdb3300..0387eb1e 100644 --- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java +++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java @@ -2,9 +2,9 @@ package com.onthegomap.planetiler; import static java.util.Map.entry; -import com.onthegomap.planetiler.archive.TileCopy; import com.onthegomap.planetiler.benchmarks.LongLongMapBench; import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping; +import com.onthegomap.planetiler.copy.TileCopy; import com.onthegomap.planetiler.custommap.ConfiguredMapMain; import com.onthegomap.planetiler.custommap.validator.SchemaValidator; import com.onthegomap.planetiler.examples.BikeRouteOverlay;