> keys,
- String prefix, String separator, boolean uppperCase) {
+ String prefix, String separator, boolean uppperCase, String logKeyPrefix) {
var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)),
Pattern.CASE_INSENSITIVE);
return from(provider, keys,
key -> normalize(prefix + separator + key, separator, uppperCase),
- key -> normalize(prefixRegex.matcher(key).replaceFirst(""))
+ key -> normalize(prefixRegex.matcher(key).replaceFirst("")),
+ logKeyPrefix
);
}
@@ -262,7 +269,8 @@ public class Arguments {
() -> Stream.concat(
other.keys.get().stream(),
keys.get().stream()
- ).distinct().toList()
+ ).distinct().toList(),
+ other.logKeyPrefix
);
if (silent) {
result.silence();
@@ -307,7 +315,7 @@ public class Arguments {
protected void logArgValue(String key, String description, Object result) {
if (!silent && LOGGER.isDebugEnabled()) {
- LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description);
+ LOGGER.debug("argument: {}{}={} ({})", logKeyPrefix, key.replaceFirst("\\|.*$", ""), result, description);
}
}
@@ -532,7 +540,7 @@ public class Arguments {
* Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}.
*/
public Arguments withPrefix(String prefix) {
- return fromPrefixed(provider, keys, prefix, "_", false);
+ return fromPrefixed(provider, keys, prefix, "_", false, logKeyPrefix + prefix + "_");
}
/** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java
new file mode 100644
index 00000000..1e30db59
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java
@@ -0,0 +1,31 @@
+package com.onthegomap.planetiler.config;
+
+import com.onthegomap.planetiler.archive.TileArchiveConfig;
+import java.time.Duration;
+import java.util.stream.Stream;
+
+public final class CommonConfigs {
+ private CommonConfigs() {}
+
+ public static boolean force(Arguments arguments) {
+ return arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false);
+ }
+
+ public static boolean appendToArchive(Arguments arguments) {
+ return arguments.getBoolean("append",
+ "append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values())
+ .filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(),
+ false);
+ }
+
+ public static int tileWriterThreads(Arguments arguments) {
+ return arguments.getInteger("tile_write_threads",
+ "number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values())
+ .filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(),
+ 1);
+ }
+
+ public static Duration logInterval(Arguments arguments) {
+ return arguments.getDuration("loginterval", "time between logs", "10s");
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
index c505ce08..a843f2eb 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
@@ -1,6 +1,5 @@
package com.onthegomap.planetiler.config;
-import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileCompression;
import com.onthegomap.planetiler.collection.LongLongMap;
import com.onthegomap.planetiler.collection.Storage;
@@ -133,19 +132,13 @@ public record PlanetilerConfig(
featureProcessThreads,
arguments.getInteger("feature_read_threads", "number of threads to use when reading features at tile write time",
threads < 32 ? 1 : 2),
- arguments.getInteger("tile_write_threads",
- "number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values())
- .filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(),
- 1),
- arguments.getDuration("loginterval", "time between logs", "10s"),
+ CommonConfigs.tileWriterThreads(arguments),
+ CommonConfigs.logInterval(arguments),
minzoom,
maxzoom,
renderMaxzoom,
- arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
- arguments.getBoolean("append",
- "append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values())
- .filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(),
- false),
+ CommonConfigs.force(arguments),
+ CommonConfigs.appendToArchive(arguments),
arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),
arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true),
arguments.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks",
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java
new file mode 100644
index 00000000..3155a5ee
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopy.java
@@ -0,0 +1,131 @@
+package com.onthegomap.planetiler.copy;
+
+import static com.onthegomap.planetiler.worker.Worker.joinFutures;
+
+import com.onthegomap.planetiler.archive.TileArchives;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.stats.Counter;
+import com.onthegomap.planetiler.stats.ProgressLoggers;
+import com.onthegomap.planetiler.worker.WorkQueue;
+import com.onthegomap.planetiler.worker.WorkerPipeline;
+import java.io.IOException;
+import java.time.Duration;
+
+/**
+ * Utility to copy/convert tiles and metadata from one archive into another.
+ *
+ * Example usages:
+ *
+ *
+ * --input=tiles.mbtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
+ * --input=tiles.pmtiles --output=tiles.mbtiles
+ * --input=tiles.mbtiles --output=tiles/
+ * --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
+ * --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
+ * --input=tiles.mbtiles --output=tiles.proto
+ *
+ */
+public class TileCopy {
+
+ private final TileCopyConfig config;
+
+ private final Counter.MultiThreadCounter tilesReadOverall = Counter.newMultiThreadCounter();
+ private final Counter.MultiThreadCounter tilesProcessedOverall = Counter.newMultiThreadCounter();
+ private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter();
+
+
+ TileCopy(TileCopyConfig config) {
+ this.config = config;
+ }
+
+ public void run() throws IOException {
+
+ if (!config.inArchive().exists()) {
+ throw new IllegalArgumentException("the input archive does not exist");
+ }
+
+ config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads());
+
+ final var loggers = ProgressLoggers.create()
+ .addRateCounter("tiles_read", tilesReadOverall::get)
+ .addRateCounter("tiles_processed", tilesProcessedOverall::get)
+ .addRateCounter("tiles_written", tilesWrittenOverall::get);
+
+ try (
+ var reader = TileArchives.newReader(config.inArchive(), config.inArguments());
+ var writer = TileArchives.newWriter(config.outArchive(), config.outArguments())
+ ) {
+
+ final TileCopyContext context = TileCopyContext.create(reader, writer, config);
+
+ final var pipeline = WorkerPipeline.start("copy", config.stats());
+ final WorkQueue resultQueue =
+ new WorkQueue<>("results", config.workQueueCapacity(), config.workQueueMaxBatch(), config.stats());
+
+ try (
+ var it = TileCopyWorkItemGenerators.create(context)
+ ) {
+
+ writer.initialize();
+
+ final var readerBranch = pipeline
+ .fromGenerator("iterator", next -> {
+ try (resultQueue) {
+ while (it.hasNext()) {
+ final TileCopyWorkItem t = it.next();
+ resultQueue.accept(t); // put to queue immediately => retain order
+ next.accept(t);
+ }
+ }
+ })
+ .addBuffer("to_read", config.workQueueCapacity(), config.workQueueMaxBatch())
+ .addWorker("read", config.tileReaderThreads(), (prev, next) -> {
+ final Counter tilesRead = tilesReadOverall.counterForThread();
+ for (var item : prev) {
+ item.loadOriginalTileData();
+ tilesRead.inc();
+ next.accept(item);
+ }
+ })
+ .addBuffer("to_process", config.workQueueCapacity(), config.workQueueMaxBatch())
+ .sinkTo("process", config.tileReaderThreads(), prev -> {
+ final Counter tilesProcessed = tilesProcessedOverall.counterForThread();
+ for (var item : prev) {
+ item.process();
+ tilesProcessed.inc();
+ }
+ });
+
+ final var writerBranch = pipeline.readFromQueue(resultQueue)
+ .sinkTo("write", config.tileWriterThreads(), prev -> {
+ final Counter tilesWritten = tilesWrittenOverall.counterForThread();
+ try (var tileWriter = writer.newTileWriter()) {
+ for (var item : prev) {
+ var result = item.toTileEncodingResult();
+ if (result.tileData() == null && config.skipEmpty()) {
+ continue;
+ }
+ tileWriter.write(result);
+ tilesWritten.inc();
+ }
+ }
+ });
+
+ final var writerDone = writerBranch.done().thenRun(() -> writer.finish(context.outMetadata()));
+ final var readerDone = readerBranch.done();
+
+ loggers
+ .newLine()
+ .addPipelineStats(readerBranch)
+ .addPipelineStats(writerBranch);
+
+ loggers.awaitAndLog(joinFutures(writerDone, readerDone), Duration.ofSeconds(1));
+ }
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run();
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java
new file mode 100644
index 00000000..c54ca229
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyConfig.java
@@ -0,0 +1,108 @@
+package com.onthegomap.planetiler.copy;
+
+import com.onthegomap.planetiler.archive.TileArchiveConfig;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer;
+import com.onthegomap.planetiler.archive.TileCompression;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.config.CommonConfigs;
+import com.onthegomap.planetiler.geo.TileOrder;
+import com.onthegomap.planetiler.stats.ProcessInfo;
+import com.onthegomap.planetiler.stats.Stats;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import org.locationtech.jts.geom.Envelope;
+
+record TileCopyConfig(
+ TileArchiveConfig inArchive,
+ TileArchiveConfig outArchive,
+ Arguments inArguments,
+ Arguments outArguments,
+ TileCompression inCompression,
+ TileCompression outCompression,
+ int tileWriterThreads,
+ int tileReaderThreads,
+ int tileProcessorThreads,
+ Duration logInterval,
+ Stats stats,
+ int queueSize,
+ boolean append,
+ boolean force,
+ TileArchiveMetadata inMetadata,
+ boolean skipEmpty,
+ Envelope filterBounds,
+ int filterMinzoom,
+ int filterMaxzoom,
+ boolean scanTilesInOrder,
+ TileOrder outputTileOrder,
+ int workQueueCapacity,
+ int workQueueMaxBatch
+) {
+
+ TileCopyConfig {
+ if (tileReaderThreads > 1 && !inArchive.format().supportsConcurrentReads()) {
+ throw new IllegalArgumentException(inArchive.format().id() + " does not support concurrent reads");
+ }
+ if (tileWriterThreads > 1 && !outArchive.format().supportsConcurrentReads()) {
+ throw new IllegalArgumentException(outArchive.format().id() + " does not support concurrent writes");
+ }
+ if (filterMinzoom > filterMaxzoom) {
+ throw new IllegalArgumentException("require minzoom <= maxzoom");
+ }
+ }
+
+ static TileCopyConfig fromArguments(Arguments baseArguments) {
+
+ final Arguments inArguments = baseArguments.withPrefix("in");
+ final Arguments outArguments = baseArguments.withPrefix("out");
+ final Arguments baseOrOutArguments = outArguments.orElse(baseArguments);
+
+ final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null);
+ final TileArchiveMetadata inMetadata;
+ if (inMetadataPath != null) {
+ try {
+ inMetadata =
+ TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ } else {
+ inMetadata = null;
+ }
+
+ return new TileCopyConfig(
+ TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")),
+ TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")),
+ inArguments,
+ outArguments,
+ getTileCompressionArg(inArguments, "the input tile compression"),
+ getTileCompressionArg(outArguments, "the output tile compression"),
+ CommonConfigs.tileWriterThreads(baseOrOutArguments),
+ baseArguments.getInteger("tile_read_threads", "number of threads used to read tile data", 1),
+ baseArguments.getInteger("tile_process_threads", "number of threads used to process tile data",
+ Math.max(1, Runtime.getRuntime().availableProcessors() - 2)),
+ CommonConfigs.logInterval(baseArguments),
+ baseArguments.getStats(),
+ Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)),
+ CommonConfigs.appendToArchive(baseOrOutArguments),
+ CommonConfigs.force(baseOrOutArguments),
+ inMetadata,
+ baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", true),
+ baseArguments.bounds("filter_bounds", "the bounds to filter"),
+ baseArguments.getInteger("filter_minzoom", "the min zoom", 0),
+ baseArguments.getInteger("filter_maxzoom", "the max zoom", 14),
+ baseArguments.getBoolean("scan_tiles_in_order", "output the tiles in the same order they are in the", true),
+ baseArguments.getObject("output_tile_order", "the output tile order (if not scanned)", null,
+ s -> s == null ? null : TileOrder.valueOf(s.toUpperCase())),
+ 100_000,
+ 100
+ );
+ }
+
+ private static TileCompression getTileCompressionArg(Arguments args, String description) {
+ return args.getObject("tile_compression", description, TileCompression.UNKNOWN,
+ v -> TileCompression.findById(v).orElseThrow());
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java
new file mode 100644
index 00000000..da13dd81
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyContext.java
@@ -0,0 +1,123 @@
+package com.onthegomap.planetiler.copy;
+
+import com.onthegomap.planetiler.archive.ReadableTileArchive;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileCompression;
+import com.onthegomap.planetiler.archive.WriteableTileArchive;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
+import org.locationtech.jts.geom.Envelope;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+record TileCopyContext(
+ TileArchiveMetadata inMetadata,
+ TileArchiveMetadata outMetadata,
+ TileCopyConfig config,
+ ReadableTileArchive reader,
+ WriteableTileArchive writer
+) {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyContext.class);
+
+ TileCompression inputCompression() {
+ return inMetadata.tileCompression();
+ }
+
+ TileCompression outputCompression() {
+ return outMetadata.tileCompression();
+ }
+
+ static TileCopyContext create(ReadableTileArchive reader, WriteableTileArchive writer, TileCopyConfig config) {
+ final TileArchiveMetadata inMetadata = getInMetadata(reader, config);
+ final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata, config);
+ return new TileCopyContext(
+ inMetadata,
+ outMetadata,
+ config,
+ reader,
+ writer
+ );
+ }
+
+ private static TileArchiveMetadata getInMetadata(ReadableTileArchive inArchive, TileCopyConfig config) {
+ TileArchiveMetadata inMetadata = config.inMetadata();
+ if (inMetadata == null) {
+ inMetadata = inArchive.metadata();
+ if (inMetadata == null) {
+ LOGGER.atWarn()
+ .log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata");
+ inMetadata = fallbackMetadata();
+ }
+ }
+ if (inMetadata.tileCompression() == null) {
+ inMetadata = inMetadata.withTileCompression(config.inCompression());
+ }
+
+ return inMetadata;
+ }
+
+ private static TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata, TileCopyConfig config) {
+
+ final TileCompression tileCompression;
+ if (config.outCompression() == TileCompression.UNKNOWN && inMetadata.tileCompression() == TileCompression.UNKNOWN) {
+ tileCompression = TileCompression.GZIP;
+ } else if (config.outCompression() != TileCompression.UNKNOWN) {
+ tileCompression = config.outCompression();
+ } else {
+ tileCompression = inMetadata.tileCompression();
+ }
+
+ final Envelope bounds;
+ if (config.filterBounds() != null) {
+ bounds = config.filterBounds();
+ } else if (inMetadata.bounds() != null) {
+ bounds = inMetadata.bounds();
+ } else {
+ bounds = null;
+ }
+
+ final int minzoom = Stream.of(inMetadata.minzoom(), config.filterMinzoom()).filter(Objects::nonNull)
+ .mapToInt(Integer::intValue).max().orElse(0);
+
+ final int maxzoom = Stream.of(inMetadata.maxzoom(), config.filterMaxzoom()).filter(Objects::nonNull)
+ .mapToInt(Integer::intValue).min().orElse(0);
+
+ return new TileArchiveMetadata(
+ inMetadata.name(),
+ inMetadata.description(),
+ inMetadata.attribution(),
+ inMetadata.version(),
+ inMetadata.type(),
+ inMetadata.format(),
+ bounds,
+ inMetadata.center(),
+ minzoom,
+ maxzoom,
+ inMetadata.json(),
+ inMetadata.others(),
+ tileCompression
+ );
+ }
+
+ private static TileArchiveMetadata fallbackMetadata() {
+ return new TileArchiveMetadata(
+ "unknown",
+ null,
+ null,
+ null,
+ null,
+ TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf
+ null,
+ null,
+ 0,
+ 14,
+ new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers
+ Map.of(),
+ null
+ );
+ }
+
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java
new file mode 100644
index 00000000..505ea557
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItem.java
@@ -0,0 +1,51 @@
+package com.onthegomap.planetiler.copy;
+
+import com.onthegomap.planetiler.archive.TileEncodingResult;
+import com.onthegomap.planetiler.geo.TileCoord;
+import java.util.OptionalLong;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
+
+public class TileCopyWorkItem {
+ private final TileCoord coord;
+ private final Supplier originalTileDataLoader;
+ private final UnaryOperator reEncoder;
+ private final Function hasher;
+ private final CompletableFuture originalData = new CompletableFuture<>();
+ private final CompletableFuture reEncodedData = new CompletableFuture<>();
+ private final CompletableFuture reEncodedDataHash = new CompletableFuture<>();
+
+ TileCopyWorkItem(
+ TileCoord coord,
+ Supplier originalTileDataLoader,
+ UnaryOperator reEncoder,
+ Function hasher
+ ) {
+ this.coord = coord;
+ this.originalTileDataLoader = originalTileDataLoader;
+ this.reEncoder = reEncoder;
+ this.hasher = hasher;
+ }
+
+ public TileCoord getCoord() {
+ return coord;
+ }
+
+ void loadOriginalTileData() {
+ originalData.complete(originalTileDataLoader.get());
+ }
+
+ void process() throws ExecutionException, InterruptedException {
+ final var reEncoded = reEncoder.apply(originalData.get());
+ final var hash = hasher.apply(reEncoded);
+ reEncodedData.complete(reEncoded);
+ reEncodedDataHash.complete(hash);
+ }
+
+ TileEncodingResult toTileEncodingResult() throws ExecutionException, InterruptedException {
+ return new TileEncodingResult(coord, reEncodedData.get(), reEncodedDataHash.get());
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java
new file mode 100644
index 00000000..db89b4f8
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileCopyWorkItemGenerators.java
@@ -0,0 +1,147 @@
+package com.onthegomap.planetiler.copy;
+
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveConfig;
+import com.onthegomap.planetiler.config.Bounds;
+import com.onthegomap.planetiler.geo.GeoUtils;
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.geo.TileExtents;
+import com.onthegomap.planetiler.geo.TileOrder;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import com.onthegomap.planetiler.util.Hashing;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.UnaryOperator;
+import org.locationtech.jts.geom.Envelope;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+final class TileCopyWorkItemGenerators {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyWorkItemGenerators.class);
+
+ private TileCopyWorkItemGenerators() {}
+
+ @SuppressWarnings("java:S2095")
+ static CloseableIterator create(TileCopyContext context) {
+
+ final TileArchiveConfig.Format inFormat = context.config().inArchive().format();
+ final TileArchiveConfig.Format outFormat = context.config().outArchive().format();
+
+ final boolean inOrder;
+ if (outFormat == TileArchiveConfig.Format.FILES) {
+ inOrder = true; // always use the input formats native order when outputting files
+ } else if (inFormat == TileArchiveConfig.Format.FILES) {
+ inOrder = false; // never use the inOrder looper when using files since there's no order guarantee
+ } else {
+ inOrder = context.config().scanTilesInOrder();
+ }
+
+ final int minzoom = context.outMetadata().minzoom();
+ final int maxzoom = context.outMetadata().maxzoom();
+ final Envelope filterBounds =
+ context.outMetadata().bounds() == null ? Bounds.WORLD.world() :
+ GeoUtils.toWorldBounds(context.outMetadata().bounds());
+ final var boundsFilter = TileExtents.computeFromWorldBounds(maxzoom, filterBounds);
+ final Predicate zoomFilter = i -> i.getCoord().z() >= minzoom && i.getCoord().z() <= maxzoom;
+
+ final ProcessorArgs processorArgs = new ProcessorArgs(
+ TileDataReEncoders.create(context),
+ context.writer().deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) :
+ b -> OptionalLong.empty()
+ );
+
+ if (inOrder) {
+ CloseableIterator it = new EagerInOrder(context.reader().getAllTiles(), processorArgs)
+ .filter(zoomFilter);
+ if (!Objects.equals(context.inMetadata().bounds(), context.outMetadata().bounds())) {
+ it = it.filter(i -> boundsFilter.test(i.getCoord()));
+ }
+ return it;
+
+ } else {
+
+ final boolean warnPoorlySupported = switch (inFormat) {
+ case CSV, TSV, PROTO, PBF, JSON -> true;
+ case PMTILES, MBTILES, FILES -> false;
+ };
+ if (warnPoorlySupported) {
+ LOGGER.atWarn().log("{} random access is very slow", inFormat.id());
+ }
+
+ final TileOrder tileOrder = Optional.ofNullable(context.config().outputTileOrder())
+ .orElse(context.writer().tileOrder());
+
+ return new TileOrderLoop(tileOrder, minzoom, maxzoom, context.reader()::getTile, processorArgs)
+ .filter(i -> boundsFilter.test(i.getCoord()));
+ }
+ }
+
+ private record EagerInOrder(
+ CloseableIterator it,
+ ProcessorArgs processorArgs
+ ) implements CloseableIterator {
+
+ @Override
+ public void close() {
+ it.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return it.hasNext();
+ }
+
+ @Override
+ public TileCopyWorkItem next() {
+ if (!it.hasNext()) {
+ throw new NoSuchElementException();
+ }
+ final Tile t = it.next();
+ return new TileCopyWorkItem(t.coord(), t::bytes, processorArgs.reEncoder(), processorArgs.hasher());
+ }
+ }
+
+ private static class TileOrderLoop implements CloseableIterator {
+
+ private final TileOrder tileOrder;
+ private final int max;
+ private final Function dataLoader;
+ private final ProcessorArgs processorArgs;
+ private int current;
+
+ TileOrderLoop(TileOrder tileOrder, int minZoom, int maxZoom, Function dataLoader,
+ ProcessorArgs processorArgs) {
+ this.tileOrder = tileOrder;
+ this.current = TileCoord.startIndexForZoom(minZoom);
+ this.max = TileCoord.endIndexForZoom(maxZoom);
+ this.dataLoader = dataLoader;
+ this.processorArgs = processorArgs;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return current <= max;
+ }
+
+ @Override
+ public TileCopyWorkItem next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ final TileCoord c = tileOrder.decode(current++);
+ return new TileCopyWorkItem(c, () -> dataLoader.apply(c), processorArgs.reEncoder(), processorArgs.hasher());
+ }
+
+ @Override
+ public void close() {
+ // nothing to close
+ }
+ }
+
+ private record ProcessorArgs(UnaryOperator reEncoder, Function hasher) {}
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java
new file mode 100644
index 00000000..9ffcd366
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/copy/TileDataReEncoders.java
@@ -0,0 +1,31 @@
+package com.onthegomap.planetiler.copy;
+
+import com.onthegomap.planetiler.archive.TileCompression;
+import com.onthegomap.planetiler.util.Gzip;
+import java.util.function.UnaryOperator;
+
+final class TileDataReEncoders {
+
+ private TileDataReEncoders() {}
+
+ static UnaryOperator create(TileCopyContext c) {
+ // for now just one - but compose multiple as needed in the future (decompress, do something, compress)
+ return reCompressor(c.inputCompression(), c.outputCompression());
+ }
+
+ private static UnaryOperator reCompressor(TileCompression inCompression, TileCompression outCompression) {
+ if (inCompression == outCompression) {
+ return b -> b;
+ } else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) {
+ return Gzip::gunzip;
+ } else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) {
+ return Gzip::gzip;
+ } else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.GZIP) {
+ return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b);
+ } else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.NONE) {
+ return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b;
+ } else {
+ throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression);
+ }
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java
index ee4d70c0..b8b9c47b 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java
@@ -159,12 +159,17 @@ public class WriteableFilesArchive implements WriteableTileArchive {
}
lastCheckedFolder = folder;
try {
- Files.write(file, data);
+ if (data == null) {
+ Files.createFile(file);
+ } else {
+ Files.write(file, data);
+ }
} catch (IOException e) {
throw new UncheckedIOException(e);
}
-
- bytesWritten.incBy(data.length);
+ if (data != null) {
+ bytesWritten.incBy(data.length);
+ }
}
@Override
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java
index 6d6370a7..8cc45144 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/geo/TileCoord.java
@@ -42,10 +42,14 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
}
}
- private static int startIndexForZoom(int z) {
+ public static int startIndexForZoom(int z) {
return ZOOM_START_INDEX[z];
}
+ public static int endIndexForZoom(int z) {
+ return ZOOM_START_INDEX[z] + (1 << z) * (1 << z) - 1;
+ }
+
private static int zoomForIndex(int idx) {
for (int z = MAX_MAXZOOM; z >= 0; z--) {
if (ZOOM_START_INDEX[z] <= idx) {
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
index e68e1f7f..e1e9449a 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
@@ -6,6 +6,7 @@ import com.onthegomap.planetiler.archive.ReadableTileArchive;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer;
+import com.onthegomap.planetiler.archive.TileCompression;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.archive.WriteableTileArchive;
import com.onthegomap.planetiler.config.Arguments;
@@ -847,7 +848,11 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
*/
public TileArchiveMetadata get() {
Map map = new HashMap<>(getAll());
- return TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class);
+ var metadata = TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class);
+ if (metadata.tileCompression() == null) {
+ metadata = metadata.withTileCompression(TileCompression.GZIP);
+ }
+ return metadata;
}
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
index 99a43f15..d0a18ed7 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
@@ -6,7 +6,6 @@ import com.onthegomap.planetiler.VectorTile;
import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.geo.TileCoord;
import java.io.IOException;
-import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@@ -92,11 +91,7 @@ public class Verify {
}
private static List decode(byte[] zipped) {
- try {
- return VectorTile.decode(gunzip(zipped));
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
+ return VectorTile.decode(gunzip(zipped));
}
/**
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
index 795a50f0..e95b1d7d 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
@@ -57,8 +57,7 @@ public final class WriteablePmtiles implements WriteableTileArchive {
this.bytesWritten = bytesWritten;
}
- private static Directories makeDirectoriesWithLeaves(List subEntries, int leafSize, int attemptNum)
- throws IOException {
+ private static Directories makeDirectoriesWithLeaves(List subEntries, int leafSize, int attemptNum) {
LOGGER.info("Building directories with {} entries per leaf, attempt {}...", leafSize, attemptNum);
ArrayList rootEntries = new ArrayList<>();
ByteArrayList leavesOutputStream = new ByteArrayList();
@@ -91,9 +90,8 @@ public final class WriteablePmtiles implements WriteableTileArchive {
*
* @param entries a sorted ObjectArrayList of all entries in the tileset.
* @return byte arrays of the root and all leaf directories, and the # of leaves.
- * @throws IOException if compression fails
*/
- static Directories makeDirectories(List entries) throws IOException {
+ static Directories makeDirectories(List entries) {
int maxEntriesRootOnly = 16384;
int attemptNum = 1;
if (entries.size() < maxEntriesRootOnly) {
@@ -302,6 +300,9 @@ public final class WriteablePmtiles implements WriteableTileArchive {
long offset;
OptionalLong tileDataHashOpt = encodingResult.tileDataHash();
var data = encodingResult.tileData();
+ if (data == null) {
+ return;
+ }
TileCoord coord = encodingResult.coord();
long tileId = coord.hilbertEncoded();
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java
new file mode 100644
index 00000000..506aa02d
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java
@@ -0,0 +1,50 @@
+package com.onthegomap.planetiler.stream;
+
+import com.google.common.base.Suppliers;
+import java.util.Base64;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+enum CsvBinaryEncoding {
+
+ BASE64("base64", () -> Base64.getEncoder()::encodeToString, () -> Base64.getDecoder()::decode),
+ HEX("hex", () -> HexFormat.of()::formatHex, () -> HexFormat.of()::parseHex);
+
+ private final String id;
+ private final Supplier> encoder;
+ private final Supplier> decoder;
+
+ private CsvBinaryEncoding(String id, Supplier> encoder,
+ Supplier> decoder) {
+ this.id = id;
+ this.encoder = Suppliers.memoize(encoder::get);
+ this.decoder = Suppliers.memoize(decoder::get);
+ }
+
+ String encode(byte[] b) {
+ return encoder.get().apply(b);
+ }
+
+ byte[] decode(String s) {
+ return decoder.get().apply(s);
+ }
+
+ static List ids() {
+ return Stream.of(CsvBinaryEncoding.values()).map(CsvBinaryEncoding::id).toList();
+ }
+
+ static CsvBinaryEncoding fromId(String id) {
+ return Stream.of(CsvBinaryEncoding.values())
+ .filter(de -> de.id().equals(id))
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException(
+ "unexpected binary encoding - expected one of " + ids() + " but got " + id));
+ }
+
+ String id() {
+ return id;
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java
new file mode 100644
index 00000000..2f0808f7
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java
@@ -0,0 +1,46 @@
+package com.onthegomap.planetiler.stream;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import java.util.Arrays;
+import java.util.Objects;
+
+@JsonTypeInfo(
+ use = JsonTypeInfo.Id.NAME,
+ include = JsonTypeInfo.As.PROPERTY,
+ property = "type")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = JsonStreamArchiveEntry.TileEntry.class, name = "tile"),
+ @JsonSubTypes.Type(value = JsonStreamArchiveEntry.InitializationEntry.class, name = "initialization"),
+ @JsonSubTypes.Type(value = JsonStreamArchiveEntry.FinishEntry.class, name = "finish")
+})
+sealed interface JsonStreamArchiveEntry {
+ record TileEntry(int x, int y, int z, byte[] encodedData) implements JsonStreamArchiveEntry {
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(encodedData);
+ result = prime * result + Objects.hash(x, y, z);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return this == obj || (obj instanceof JsonStreamArchiveEntry.TileEntry tileEntry &&
+ Arrays.equals(encodedData, tileEntry.encodedData) && x == tileEntry.x && y == tileEntry.y && z == tileEntry.z);
+ }
+
+ @Override
+ public String toString() {
+ return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]";
+ }
+ }
+
+ record InitializationEntry() implements JsonStreamArchiveEntry {}
+
+
+ record FinishEntry(TileArchiveMetadata metadata) implements JsonStreamArchiveEntry {}
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java
new file mode 100644
index 00000000..f6df44e4
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java
@@ -0,0 +1,108 @@
+package com.onthegomap.planetiler.stream;
+
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveConfig;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.Scanner;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+
+/**
+ * Reads tiles from a CSV file. Counterpart to {@link WriteableCsvArchive}.
+ *
+ * Supported arguments:
+ *
+ * - column_separator
+ * - The column separator e.g. ",", ";", "\t"
+ * - line_separator
+ * - The line separator e.g. "\n", "\r", "\r\n"
+ *
+ *
+ * @see WriteableCsvArchive
+ */
+public class ReadableCsvArchive extends ReadableStreamArchive {
+
+ private final Pattern columnSeparatorPattern;
+ private final Pattern lineSeparatorPattern;
+ private final Function tileDataDecoder;
+
+ private ReadableCsvArchive(TileArchiveConfig.Format format, Path basePath, StreamArchiveConfig config) {
+ super(basePath, config);
+ this.columnSeparatorPattern =
+ Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format)));
+ this.lineSeparatorPattern =
+ Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format)));
+ final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions());
+ this.tileDataDecoder = binaryEncoding::decode;
+ }
+
+ public static ReadableCsvArchive newReader(TileArchiveConfig.Format format, Path basePath,
+ StreamArchiveConfig config) {
+ return new ReadableCsvArchive(format, basePath, config);
+ }
+
+ @Override
+ CloseableIterator createIterator() {
+ try {
+ @SuppressWarnings("java:S2095") final Scanner s =
+ new Scanner(basePath.toFile()).useDelimiter(lineSeparatorPattern);
+ return new CloseableIterator<>() {
+ @Override
+ public void close() {
+ s.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return s.hasNext();
+ }
+
+ @Override
+ public String next() {
+ return s.next();
+ }
+ };
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+
+ }
+
+ @Override
+ Optional mapEntryToTile(String entry) {
+ final String[] splits = columnSeparatorPattern.split(entry);
+ final byte[] bytes;
+ if (splits.length == 4) {
+ bytes = tileDataDecoder.apply(splits[3].strip());
+ } else if (splits.length == 3) {
+ bytes = null;
+ } else {
+ throw new InvalidCsvFormat(entry.length() > 20 ? entry.substring(0, 20) + "..." : entry);
+ }
+ return Optional.of(new Tile(
+ TileCoord.ofXYZ(
+ Integer.parseInt(splits[0].strip()),
+ Integer.parseInt(splits[1].strip()),
+ Integer.parseInt(splits[2].strip())
+ ),
+ bytes
+ ));
+ }
+
+ @Override
+ Optional mapEntryToMetadata(String entry) {
+ return Optional.empty();
+ }
+
+ static class InvalidCsvFormat extends RuntimeException {
+ InvalidCsvFormat(String message) {
+ super(message);
+ }
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java
new file mode 100644
index 00000000..640adf0b
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java
@@ -0,0 +1,82 @@
+package com.onthegomap.planetiler.stream;
+
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+
+/**
+ * Reads tiles and metadata from a delimited JSON file. Counterpart to {@link WriteableJsonStreamArchive}.
+ *
+ * @see WriteableJsonStreamArchive
+ */
+public class ReadableJsonStreamArchive extends ReadableStreamArchive {
+
+ private ReadableJsonStreamArchive(Path basePath, StreamArchiveConfig config) {
+ super(basePath, config);
+ }
+
+ public static ReadableJsonStreamArchive newReader(Path basePath, StreamArchiveConfig config) {
+ return new ReadableJsonStreamArchive(basePath, config);
+ }
+
+ @Override
+ CloseableIterator createIterator() {
+ BufferedReader reader = null;
+ try {
+ reader = Files.newBufferedReader(basePath);
+ final var readerFinal = reader;
+ final var it = StreamArchiveUtils.jsonMapperJsonStreamArchive
+ .readerFor(JsonStreamArchiveEntry.class)
+ .readValues(readerFinal);
+ return new CloseableIterator<>() {
+ @Override
+ public void close() {
+ try {
+ readerFinal.close();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return it.hasNext();
+ }
+
+ @Override
+ public JsonStreamArchiveEntry next() {
+ return it.next();
+ }
+ };
+ } catch (IOException e) {
+ closeSilentlyOnError(reader);
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ Optional mapEntryToTile(JsonStreamArchiveEntry entry) {
+ if (entry instanceof JsonStreamArchiveEntry.TileEntry tileEntry) {
+ return Optional.of(new Tile(
+ TileCoord.ofXYZ(tileEntry.x(), tileEntry.y(), tileEntry.z()),
+ tileEntry.encodedData()
+ ));
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ Optional mapEntryToMetadata(JsonStreamArchiveEntry entry) {
+ if (entry instanceof JsonStreamArchiveEntry.FinishEntry finishEntry) {
+ return Optional.ofNullable(finishEntry.metadata());
+ }
+ return Optional.empty();
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java
new file mode 100644
index 00000000..ff35e5ba
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java
@@ -0,0 +1,160 @@
+package com.onthegomap.planetiler.stream;
+
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.archive.TileCompression;
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.proto.StreamArchiveProto;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import com.onthegomap.planetiler.util.LayerAttrStats;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.CoordinateXY;
+import org.locationtech.jts.geom.Envelope;
+
+/**
+ * Reads tiles and metadata from a delimited protobuf file. Counterpart to {@link WriteableProtoStreamArchive}.
+ *
+ * @see WriteableProtoStreamArchive
+ */
+public class ReadableProtoStreamArchive extends ReadableStreamArchive {
+
+ private ReadableProtoStreamArchive(Path basePath, StreamArchiveConfig config) {
+ super(basePath, config);
+ }
+
+ public static ReadableProtoStreamArchive newReader(Path basePath, StreamArchiveConfig config) {
+ return new ReadableProtoStreamArchive(basePath, config);
+ }
+
+ @Override
+ CloseableIterator createIterator() {
+ try {
+ @SuppressWarnings("java:S2095") var in = new FileInputStream(basePath.toFile());
+ return new CloseableIterator<>() {
+ private StreamArchiveProto.Entry nextValue;
+
+ @Override
+ public void close() {
+ closeUnchecked(in);
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (nextValue != null) {
+ return true;
+ }
+ try {
+ nextValue = StreamArchiveProto.Entry.parseDelimitedFrom(in);
+ return nextValue != null;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public StreamArchiveProto.Entry next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ final StreamArchiveProto.Entry returnValue = nextValue;
+ nextValue = null;
+ return returnValue;
+ }
+ };
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ Optional mapEntryToTile(StreamArchiveProto.Entry entry) {
+ if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.TILE) {
+ return Optional.empty();
+ }
+ final StreamArchiveProto.TileEntry tileEntry = entry.getTile();
+ return Optional.of(new Tile(
+ TileCoord.ofXYZ(tileEntry.getX(), tileEntry.getY(), tileEntry.getZ()),
+ tileEntry.getEncodedData().toByteArray()
+ ));
+ }
+
+ @Override
+ Optional mapEntryToMetadata(StreamArchiveProto.Entry entry) {
+ if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.FINISH) {
+ return Optional.empty();
+ }
+ final StreamArchiveProto.Metadata metadata = entry.getFinish().getMetadata();
+ return Optional.of(new TileArchiveMetadata(
+ StringUtils.trimToNull(metadata.getName()),
+ StringUtils.trimToNull(metadata.getDescription()),
+ StringUtils.trimToNull(metadata.getAttribution()),
+ StringUtils.trimToNull(metadata.getVersion()),
+ StringUtils.trimToNull(metadata.getType()),
+ StringUtils.trimToNull(metadata.getFormat()),
+ deserializeEnvelope(metadata.hasBounds() ? metadata.getBounds() : null),
+ deserializeCoordinate(metadata.hasCenter() ? metadata.getCenter() : null),
+ metadata.hasMinZoom() ? metadata.getMinZoom() : null,
+ metadata.hasMaxZoom() ? metadata.getMaxZoom() : null,
+ extractMetadataJson(metadata),
+ metadata.getOthersMap(),
+ deserializeTileCompression(metadata.getTileCompression())
+ ));
+ }
+
+ private Envelope deserializeEnvelope(StreamArchiveProto.Envelope s) {
+ return s == null ? null : new Envelope(s.getMinX(), s.getMaxX(), s.getMinY(), s.getMaxY());
+ }
+
+ private Coordinate deserializeCoordinate(StreamArchiveProto.Coordinate s) {
+ if (s == null) {
+ return null;
+ }
+ return s.hasZ() ? new Coordinate(s.getX(), s.getY(), s.getZ()) : new CoordinateXY(s.getX(), s.getY());
+ }
+
+ private TileCompression deserializeTileCompression(StreamArchiveProto.TileCompression s) {
+ return switch (s) {
+ case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNOWN;
+ case TILE_COMPRESSION_GZIP -> TileCompression.GZIP;
+ case TILE_COMPRESSION_NONE -> TileCompression.NONE;
+ };
+ }
+
+ private TileArchiveMetadata.TileArchiveMetadataJson extractMetadataJson(StreamArchiveProto.Metadata s) {
+ final List vl = deserializeVectorLayers(s.getVectorLayersList());
+ return vl.isEmpty() ? null : new TileArchiveMetadata.TileArchiveMetadataJson(vl);
+ }
+
+ private List deserializeVectorLayers(List s) {
+ return s.stream()
+ .map(vl -> new LayerAttrStats.VectorLayer(
+ vl.getId(),
+ vl.getFieldsMap().entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> deserializeFieldType(e.getValue()))),
+ Optional.ofNullable(StringUtils.trimToNull(vl.getDescription())),
+ vl.hasMinZoom() ? OptionalInt.of(vl.getMinZoom()) : OptionalInt.empty(),
+ vl.hasMaxZoom() ? OptionalInt.of(vl.getMaxZoom()) : OptionalInt.empty()
+ ))
+ .toList();
+ }
+
+ private LayerAttrStats.FieldType deserializeFieldType(StreamArchiveProto.VectorLayer.FieldType s) {
+ return switch (s) {
+ case FIELD_TYPE_UNSPECIFIED, UNRECOGNIZED -> throw new IllegalArgumentException("unknown type");
+ case FIELD_TYPE_NUMBER -> LayerAttrStats.FieldType.NUMBER;
+ case FIELD_TYPE_BOOLEAN -> LayerAttrStats.FieldType.BOOLEAN;
+ case FIELD_TYPE_STRING -> LayerAttrStats.FieldType.STRING;
+ };
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java
new file mode 100644
index 00000000..2ea88215
--- /dev/null
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java
@@ -0,0 +1,102 @@
+package com.onthegomap.planetiler.stream;
+
+import com.google.common.base.Suppliers;
+import com.onthegomap.planetiler.archive.ReadableTileArchive;
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+abstract class ReadableStreamArchive implements ReadableTileArchive {
+
+ private final Supplier cachedMetadata = Suppliers.memoize(this::loadMetadata);
+
+ final Path basePath;
+ final StreamArchiveConfig config;
+
+ ReadableStreamArchive(Path basePath, StreamArchiveConfig config) {
+ this.basePath = basePath;
+ this.config = config;
+ }
+
+ @Override
+ public final byte[] getTile(TileCoord coord) {
+ try (var tiles = getAllTiles(); var s = tiles.stream()) {
+ return s.filter(c -> c.coord().equals(coord)).map(Tile::bytes).findFirst().orElse(null);
+ }
+ }
+
+ @Override
+ public final byte[] getTile(int x, int y, int z) {
+ return getTile(TileCoord.ofXYZ(x, y, z));
+ }
+
+ /**
+ * Callers MUST make sure to close the iterator/derived stream!
+ */
+ @Override
+ public final CloseableIterator getAllTileCoords() {
+ return getAllTiles().map(Tile::coord);
+ }
+
+ /**
+ * Callers MUST make sure to close the iterator/derived stream!
+ */
+ @Override
+ public final CloseableIterator getAllTiles() {
+ return createIterator()
+ .map(this::mapEntryToTile)
+ .filter(Optional::isPresent)
+ .map(Optional::get);
+ }
+
+ @Override
+ public final TileArchiveMetadata metadata() {
+ return cachedMetadata.get();
+ }
+
+ private TileArchiveMetadata loadMetadata() {
+ try (var i = createIterator(); var s = i.stream()) {
+ return s.map(this::mapEntryToMetadata).flatMap(Optional::stream).findFirst().orElse(null);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ // nothing to close
+ }
+
+ abstract CloseableIterator createIterator();
+
+ abstract Optional mapEntryToTile(E entry);
+
+ abstract Optional mapEntryToMetadata(E entry);
+
+ void closeSilentlyOnError(Closeable c) {
+ if (c == null) {
+ return;
+ }
+ try {
+ c.close();
+ } catch (Exception ignored) {
+ // ignore
+ }
+ }
+
+ @SuppressWarnings("java:S112")
+ void closeUnchecked(Closeable c) {
+ if (c == null) {
+ return;
+ }
+ try {
+ c.close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java
index 6f85ccb5..08b8967c 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java
@@ -1,10 +1,10 @@
package com.onthegomap.planetiler.stream;
import com.onthegomap.planetiler.config.Arguments;
-import com.onthegomap.planetiler.config.PlanetilerConfig;
+import com.onthegomap.planetiler.config.CommonConfigs;
-public record StreamArchiveConfig(boolean appendToFile, Arguments moreOptions) {
- public StreamArchiveConfig(PlanetilerConfig planetilerConfig, Arguments moreOptions) {
- this(planetilerConfig.append(), moreOptions);
+public record StreamArchiveConfig(boolean appendToFile, Arguments formatOptions) {
+ public StreamArchiveConfig(Arguments baseArguments, Arguments formatOptions) {
+ this(CommonConfigs.appendToArchive(baseArguments), formatOptions);
}
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java
index 8d23ef6f..b1ff2e20 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java
@@ -1,5 +1,8 @@
package com.onthegomap.planetiler.stream;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.google.common.net.UrlEscapers;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.config.Arguments;
@@ -12,8 +15,25 @@ import org.apache.commons.text.StringEscapeUtils;
public final class StreamArchiveUtils {
+ /**
+ * exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases
+ * => allow to output tiles, only
+ */
+ private static final String JSON_OPTION_WRITE_TILES_ONLY = "tiles_only";
+
+ private static final String JSON_OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator";
+
+ static final String CSV_OPTION_COLUMN_SEPARATOR = "column_separator";
+ static final String CSV_OPTION_LINE_SEPARATOR = "line_separator";
+ static final String CSV_OPTION_BINARY_ENCODING = "binary_encoding";
+
private static final Pattern quotedPattern = Pattern.compile("^'(.+?)'$");
+ static final JsonMapper jsonMapperJsonStreamArchive = JsonMapper.builder()
+ .serializationInclusion(JsonInclude.Include.NON_ABSENT)
+ .addModule(new Jdk8Module())
+ .build();
+
private StreamArchiveUtils() {}
public static Path constructIndexedPath(Path basePath, int index) {
@@ -39,6 +59,36 @@ public final class StreamArchiveUtils {
.translateEscapes();
}
+ static String jsonOptionRootValueSeparator(Arguments formatOptions) {
+ return getEscapedString(formatOptions, TileArchiveConfig.Format.JSON,
+ JSON_OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " "));
+ }
+
+ static boolean jsonOptionWriteTilesOnly(Arguments formatOptions) {
+ return formatOptions.getBoolean(JSON_OPTION_WRITE_TILES_ONLY, "write tiles, only", false);
+ }
+
+ static String csvOptionColumnSeparator(Arguments formatOptions, TileArchiveConfig.Format format) {
+ final String defaultColumnSeparator = switch (format) {
+ case CSV -> "','";
+ case TSV -> "'\\t'";
+ default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id());
+ };
+ return getEscapedString(formatOptions, format,
+ CSV_OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " "));
+ }
+
+ static String csvOptionLineSeparator(Arguments formatOptions, TileArchiveConfig.Format format) {
+ return StreamArchiveUtils.getEscapedString(formatOptions, format,
+ CSV_OPTION_LINE_SEPARATOR, "line separator", "'\\n'", List.of("\n", "\r\n"));
+ }
+
+ static CsvBinaryEncoding csvOptionBinaryEncoding(Arguments formatOptions) {
+ return CsvBinaryEncoding.fromId(formatOptions.getString(CSV_OPTION_BINARY_ENCODING,
+ "binary (tile) data encoding - one of " + CsvBinaryEncoding.ids(), "base64"));
+ }
+
+
private static String escapeJava(String s) {
if (!s.trim().equals(s)) {
s = "'" + s + "'";
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java
index 5f9d549e..be0bc513 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java
@@ -11,11 +11,7 @@ import java.io.UncheckedIOException;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
-import java.util.Base64;
-import java.util.HexFormat;
-import java.util.List;
import java.util.function.Function;
-import java.util.stream.Stream;
/**
* Writes tile data into a CSV file (or pipe).
@@ -67,31 +63,16 @@ import java.util.stream.Stream;
*/
public final class WriteableCsvArchive extends WriteableStreamArchive {
- static final String OPTION_COLUMN_SEPARATOR = "column_separator";
- static final String OPTION_LINE_SEPARTATOR = "line_separator";
- static final String OPTION_BINARY_ENCODING = "binary_encoding";
-
private final String columnSeparator;
private final String lineSeparator;
private final Function tileDataEncoder;
private WriteableCsvArchive(TileArchiveConfig.Format format, Path p, StreamArchiveConfig config) {
super(p, config);
- final String defaultColumnSeparator = switch (format) {
- case CSV -> "','";
- case TSV -> "'\\t'";
- default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id());
- };
- this.columnSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format,
- OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " "));
- this.lineSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format,
- OPTION_LINE_SEPARTATOR, "line separator", "'\\n'", List.of("\n", "\r\n"));
- final BinaryEncoding binaryEncoding = BinaryEncoding.fromId(config.moreOptions().getString(OPTION_BINARY_ENCODING,
- "binary (tile) data encoding - one of " + BinaryEncoding.ids(), "base64"));
- this.tileDataEncoder = switch (binaryEncoding) {
- case BASE64 -> Base64.getEncoder()::encodeToString;
- case HEX -> HexFormat.of()::formatHex;
- };
+ this.columnSeparator = StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format);
+ this.lineSeparator = StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format);
+ final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions());
+ this.tileDataEncoder = binaryEncoding::encode;
}
public static WriteableCsvArchive newWriteToFile(TileArchiveConfig.Format format, Path path,
@@ -131,7 +112,8 @@ public final class WriteableCsvArchive extends WriteableStreamArchive {
@Override
public void write(TileEncodingResult encodingResult) {
final TileCoord coord = encodingResult.coord();
- final String tileDataEncoded = tileDataEncoder.apply(encodingResult.tileData());
+ final byte[] data = encodingResult.tileData();
+ final String tileDataEncoded = data == null ? "" : tileDataEncoder.apply(encodingResult.tileData());
try {
// x | y | z | encoded data
writer.write("%d%s%d%s%d%s%s%s".formatted(coord.x(), columnSeparator, coord.y(), columnSeparator, coord.z(),
@@ -150,32 +132,4 @@ public final class WriteableCsvArchive extends WriteableStreamArchive {
}
}
}
-
- private enum BinaryEncoding {
-
- BASE64("base64"),
- HEX("hex");
-
- private final String id;
-
- private BinaryEncoding(String id) {
- this.id = id;
- }
-
- static List ids() {
- return Stream.of(BinaryEncoding.values()).map(BinaryEncoding::id).toList();
- }
-
- static BinaryEncoding fromId(String id) {
- return Stream.of(BinaryEncoding.values())
- .filter(de -> de.id().equals(id))
- .findFirst()
- .orElseThrow(() -> new IllegalArgumentException(
- "unexpected binary encoding - expected one of " + ids() + " but got " + id));
- }
-
- String id() {
- return id;
- }
- }
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java
index 1fad9f0b..671a8676 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java
@@ -1,14 +1,8 @@
package com.onthegomap.planetiler.stream;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SequenceWriter;
import com.fasterxml.jackson.databind.json.JsonMapper;
-import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
-import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.geo.TileCoord;
@@ -19,41 +13,26 @@ import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Writes JSON-serialized tile data as well as meta data into file(s). The entries are of type
- * {@link WriteableJsonStreamArchive.Entry} are separated by newline (by default).
+ * {@link JsonStreamArchiveEntry} are separated by newline (by default).
*/
public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
private static final Logger LOGGER = LoggerFactory.getLogger(WriteableJsonStreamArchive.class);
- /**
- * exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases
- * => allow to output tiles, only
- */
- private static final String OPTION_WRITE_TILES_ONLY = "tiles_only";
-
- private static final String OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator";
-
- static final JsonMapper jsonMapper = JsonMapper.builder()
- .serializationInclusion(Include.NON_ABSENT)
- .addModule(new Jdk8Module())
- .build();
+ private static final JsonMapper jsonMapper = StreamArchiveUtils.jsonMapperJsonStreamArchive;
private final boolean writeTilesOnly;
private final String rootValueSeparator;
private WriteableJsonStreamArchive(Path p, StreamArchiveConfig config) {
super(p, config);
- this.writeTilesOnly = config.moreOptions().getBoolean(OPTION_WRITE_TILES_ONLY, "write tiles, only", false);
- this.rootValueSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), TileArchiveConfig.Format.JSON,
- OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " "));
+ this.writeTilesOnly = StreamArchiveUtils.jsonOptionWriteTilesOnly(config.formatOptions());
+ this.rootValueSeparator = StreamArchiveUtils.jsonOptionRootValueSeparator(config.formatOptions());
}
public static WriteableJsonStreamArchive newWriteToFile(Path path, StreamArchiveConfig config) {
@@ -70,7 +49,7 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
if (writeTilesOnly) {
return;
}
- writeEntryFlush(new InitializationEntry());
+ writeEntryFlush(new JsonStreamArchiveEntry.InitializationEntry());
}
@Override
@@ -78,13 +57,13 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
if (writeTilesOnly) {
return;
}
- writeEntryFlush(new FinishEntry(metadata));
+ writeEntryFlush(new JsonStreamArchiveEntry.FinishEntry(metadata));
}
- private void writeEntryFlush(Entry entry) {
+ private void writeEntryFlush(JsonStreamArchiveEntry entry) {
try (var out = new OutputStreamWriter(getPrimaryOutputStream(), StandardCharsets.UTF_8.newEncoder())) {
jsonMapper
- .writerFor(Entry.class)
+ .writerFor(JsonStreamArchiveEntry.class)
.withoutFeatures(JsonGenerator.Feature.AUTO_CLOSE_TARGET)
.writeValue(out, entry);
out.write(rootValueSeparator);
@@ -104,7 +83,8 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
this.rootValueSeparator = rootValueSeparator;
try {
this.jsonWriter =
- jsonMapper.writerFor(Entry.class).withRootValueSeparator(rootValueSeparator).writeValues(outputStream);
+ jsonMapper.writerFor(JsonStreamArchiveEntry.class).withRootValueSeparator(rootValueSeparator)
+ .writeValues(outputStream);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -114,7 +94,8 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
public void write(TileEncodingResult encodingResult) {
final TileCoord coord = encodingResult.coord();
try {
- jsonWriter.write(new TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData()));
+ jsonWriter
+ .write(new JsonStreamArchiveEntry.TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData()));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -150,53 +131,4 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
}
}
-
- @JsonTypeInfo(
- use = JsonTypeInfo.Id.NAME,
- include = JsonTypeInfo.As.PROPERTY,
- property = "type")
- @JsonSubTypes({
- @Type(value = TileEntry.class, name = "tile"),
- @Type(value = InitializationEntry.class, name = "initialization"),
- @Type(value = FinishEntry.class, name = "finish")
- })
- sealed interface Entry {
-
- }
-
-
- record TileEntry(int x, int y, int z, byte[] encodedData) implements Entry {
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(encodedData);
- result = prime * result + Objects.hash(x, y, z);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof TileEntry)) {
- return false;
- }
- TileEntry other = (TileEntry) obj;
- return Arrays.equals(encodedData, other.encodedData) && x == other.x && y == other.y && z == other.z;
- }
-
- @Override
- public String toString() {
- return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]";
- }
- }
-
- record InitializationEntry() implements Entry {}
-
-
- record FinishEntry(TileArchiveMetadata metadata) implements Entry {}
-
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java
index 54566948..a618ffe2 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java
@@ -158,15 +158,17 @@ public final class WriteableProtoStreamArchive extends WriteableStreamArchive {
@Override
public void write(TileEncodingResult encodingResult) {
final TileCoord coord = encodingResult.coord();
- final StreamArchiveProto.TileEntry tile = StreamArchiveProto.TileEntry.newBuilder()
+ final byte[] data = encodingResult.tileData();
+ StreamArchiveProto.TileEntry.Builder tileBuilder = StreamArchiveProto.TileEntry.newBuilder()
.setZ(coord.z())
.setX(coord.x())
- .setY(coord.y())
- .setEncodedData(ByteString.copyFrom(encodingResult.tileData()))
- .build();
+ .setY(coord.y());
+ if (data != null) {
+ tileBuilder = tileBuilder.setEncodedData(ByteString.copyFrom(encodingResult.tileData()));
+ }
final StreamArchiveProto.Entry entry = StreamArchiveProto.Entry.newBuilder()
- .setTile(tile)
+ .setTile(tileBuilder.build())
.build();
try {
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java
index 4efe6257..6b44b21a 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java
@@ -2,8 +2,10 @@ package com.onthegomap.planetiler.util;
import java.io.Closeable;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import java.util.Spliterators;
import java.util.function.Function;
+import java.util.function.Predicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -56,4 +58,42 @@ public interface CloseableIterator extends Closeable, Iterator {
}
};
}
+
+ default CloseableIterator filter(Predicate predicate) {
+ final var parent = this;
+ return new CloseableIterator<>() {
+ private T nextValue;
+
+ @Override
+ public void close() {
+ parent.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (nextValue != null) {
+ return true;
+ }
+ while (parent.hasNext()) {
+ final T parentNext = parent.next();
+ if (predicate.test(parentNext)) {
+ nextValue = parentNext;
+ break;
+ }
+ }
+ return nextValue != null;
+ }
+
+ @Override
+ public T next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ final T returnValue = nextValue;
+ nextValue = null;
+ return returnValue;
+ }
+ };
+ }
+
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java
index 3e341208..56884ed9 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java
@@ -3,22 +3,42 @@ package com.onthegomap.planetiler.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
-public class Gzip {
+public final class Gzip {
- public static byte[] gzip(byte[] in) throws IOException {
+ private Gzip() {}
+
+ @SuppressWarnings("java:S1168") // null in, null out
+ public static byte[] gzip(byte[] in) {
+ if (in == null) {
+ return null;
+ }
var bos = new ByteArrayOutputStream(in.length);
try (var gzipOS = new GZIPOutputStream(bos)) {
gzipOS.write(in);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
}
return bos.toByteArray();
}
- public static byte[] gunzip(byte[] zipped) throws IOException {
+ @SuppressWarnings("java:S1168") // null in, null out
+ public static byte[] gunzip(byte[] zipped) {
+ if (zipped == null) {
+ return null;
+ }
try (var is = new GZIPInputStream(new ByteArrayInputStream(zipped))) {
return is.readAllBytes();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
}
}
+
+ public static boolean isZipped(byte[] in) {
+ return in != null && in.length > 2 && in[0] == (byte) GZIPInputStream.GZIP_MAGIC &&
+ in[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8);
+ }
}
diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java
index 45bc476e..a5b69663 100644
--- a/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java
+++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java
@@ -61,6 +61,9 @@ public final class Hashing {
*/
public static long fnv1a64(long initHash, byte... data) {
long hash = initHash;
+ if (data == null) {
+ return hash;
+ }
for (byte datum : data) {
hash ^= (datum & 0xff);
hash *= FNV1_PRIME_64;
diff --git a/planetiler-core/src/main/proto/stream_archive_proto.proto b/planetiler-core/src/main/proto/stream_archive_proto.proto
index 1df9f81f..374ab453 100644
--- a/planetiler-core/src/main/proto/stream_archive_proto.proto
+++ b/planetiler-core/src/main/proto/stream_archive_proto.proto
@@ -21,7 +21,7 @@ message InitializationEntry {
}
message FinishEntry {
- Metadata metadata = 1;
+ optional Metadata metadata = 1;
}
message Metadata {
@@ -32,10 +32,10 @@ message Metadata {
string version = 4;
string type = 5;
string format = 6;
- Envelope bounds = 7;
- Coordinate center = 8;
- int32 min_zoom = 9;
- int32 max_zoom = 10;
+ optional Envelope bounds = 7;
+ optional Coordinate center = 8;
+ optional int32 min_zoom = 9;
+ optional int32 max_zoom = 10;
repeated VectorLayer vector_layers = 11;
map others = 12;
TileCompression tile_compression = 13;
@@ -51,15 +51,15 @@ message Envelope {
message Coordinate {
double x = 1;
double y = 2;
- double z = 3;
+ optional double z = 3;
}
message VectorLayer {
string id = 1;
map fields = 2;
string description = 3;
- int32 min_zoom = 4;
- int32 max_zoom = 5;
+ optional int32 min_zoom = 4;
+ optional int32 max_zoom = 5;
enum FieldType {
FIELD_TYPE_UNSPECIFIED = 0;
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
index 610af5d9..6e9cdbe4 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
@@ -31,7 +31,10 @@ import com.onthegomap.planetiler.reader.osm.OsmElement;
import com.onthegomap.planetiler.reader.osm.OsmReader;
import com.onthegomap.planetiler.reader.osm.OsmRelationInfo;
import com.onthegomap.planetiler.stats.Stats;
-import com.onthegomap.planetiler.stream.InMemoryStreamArchive;
+import com.onthegomap.planetiler.stream.ReadableCsvArchive;
+import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive;
+import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive;
+import com.onthegomap.planetiler.stream.StreamArchiveConfig;
import com.onthegomap.planetiler.util.BuildInfo;
import com.onthegomap.planetiler.util.Gzip;
import com.onthegomap.planetiler.util.TileSizeStats;
@@ -2040,11 +2043,12 @@ class PlanetilerTests {
final ReadableTileArchiveFactory readableTileArchiveFactory = switch (format) {
case MBTILES -> Mbtiles::newReadOnlyDatabase;
- case CSV -> p -> InMemoryStreamArchive.fromCsv(p, ",");
- case TSV -> p -> InMemoryStreamArchive.fromCsv(p, "\t");
- case JSON -> InMemoryStreamArchive::fromJson;
+ case CSV, TSV ->
+ p -> ReadableCsvArchive.newReader(format, p, new StreamArchiveConfig(false, Arguments.of()));
+ case JSON -> p -> ReadableJsonStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()));
case PMTILES -> ReadablePmtiles::newReadFromFile;
- case PROTO, PBF -> InMemoryStreamArchive::fromProtobuf;
+ case PROTO, PBF ->
+ p -> ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()));
case FILES -> p -> ReadableFilesArchive.newReader(p, Arguments.of());
};
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
index 603d28e5..bc30a637 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
@@ -80,7 +80,7 @@ public class TestUtils {
public static final TileArchiveMetadata MAX_METADATA_DESERIALIZED =
new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3),
- new Coordinate(1.3, 3.7, 1.0), 2, 3,
+ new Coordinate(1.3, 3.7, 1.0), 0, 8,
TileArchiveMetadata.TileArchiveMetadataJson.create(
List.of(
new LayerAttrStats.VectorLayer("vl0",
@@ -102,8 +102,8 @@ public class TestUtils {
"version":"version",
"type":"type",
"format":"format",
- "minzoom":"2",
- "maxzoom":"3",
+ "minzoom":"0",
+ "maxzoom":"8",
"compression":"gzip",
"bounds":"0,2,1,3",
"center":"1.3,3.7,1",
@@ -294,7 +294,9 @@ public class TestUtils {
}
public static Set getTiles(ReadableTileArchive db) {
- return db.getAllTiles().stream().collect(Collectors.toSet());
+ try (var t = db.getAllTiles(); var s = t.stream()) {
+ return s.collect(Collectors.toUnmodifiableSet());
+ }
}
public static int getTilesDataCount(Mbtiles db) throws SQLException {
@@ -770,7 +772,7 @@ public class TestUtils {
if (!failures.isEmpty()) {
fail(String.join(System.lineSeparator(), failures));
}
- } catch (GeometryException | IOException e) {
+ } catch (GeometryException e) {
fail(e);
}
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java
new file mode 100644
index 00000000..983361a7
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/copy/TileCopyTest.java
@@ -0,0 +1,248 @@
+package com.onthegomap.planetiler.copy;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.onthegomap.planetiler.TestUtils;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.util.Gzip;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
+
+class TileCopyTest {
+
+ private static final String ARCHIVE_0_JSON_BASE = """
+ {"type":"initialization"}
+ {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
+ {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
+ {"type":"finish","metadata":%s}
+ """;
+
+ private static final String ARCHIVE_0_CSV_COMPRESSION_NONE = """
+ 0,0,0,AA==
+ 1,2,3,AQ==
+ """;
+
+ private static final String EXTERNAL_METADATA = "{\"name\": \"blub\"}";
+
+ @ParameterizedTest(name = "{index} - {0}")
+ @ArgumentsSource(TestArgs.class)
+ void testSimple(String testName, String archiveDataIn, String archiveDataOut, Map arguments,
+ @TempDir Path tempDir) throws Exception {
+
+ final Path archiveInPath = tempDir.resolve(archiveDataIn.contains("{") ? "in.json" : "in.csv");
+ final Path archiveOutPath = tempDir.resolve(archiveDataOut.contains("{") ? "out.json" : "out.csv");
+ final Path inMetadataPath = tempDir.resolve("metadata.json");
+
+ Files.writeString(archiveInPath, archiveDataIn);
+ Files.writeString(inMetadataPath, EXTERNAL_METADATA);
+
+ arguments = new LinkedHashMap<>(arguments);
+ arguments.replace("in_metadata", inMetadataPath.toString());
+
+ final Arguments args = Arguments.of(Map.of(
+ "input", archiveInPath.toString(),
+ "output", archiveOutPath.toString()
+ )).orElse(Arguments.of(arguments));
+
+ new TileCopy(TileCopyConfig.fromArguments(args)).run();
+
+ if (archiveDataOut.contains("{")) {
+ final List expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList();
+ final List actualLines = Files.readAllLines(archiveOutPath);
+ assertEquals(expectedLines.size(), actualLines.size());
+ for (int i = 0; i < expectedLines.size(); i++) {
+ TestUtils.assertSameJson(expectedLines.get(i), actualLines.get(i));
+ }
+ } else {
+ assertEquals(archiveDataOut, Files.readString(archiveOutPath));
+ }
+ }
+
+ private static String compressBase64(String archiveIn) {
+ final Base64.Encoder encoder = Base64.getEncoder();
+ for (int i = 0; i <= 1; i++) {
+ archiveIn = archiveIn.replace(
+ encoder.encodeToString(new byte[]{(byte) i}),
+ encoder.encodeToString(Gzip.gzip(new byte[]{(byte) i}))
+ );
+ }
+ return archiveIn;
+ }
+
+ private static String replaceBase64(String archiveIn, String replacement) {
+ final Base64.Encoder encoder = Base64.getEncoder();
+ for (int i = 0; i <= 1; i++) {
+ archiveIn = archiveIn.replace(
+ encoder.encodeToString(new byte[]{(byte) i}),
+ replacement
+ );
+ }
+ return archiveIn;
+ }
+
+ private static class TestArgs implements ArgumentsProvider {
+
+ @Override
+ public Stream extends org.junit.jupiter.params.provider.Arguments> provideArguments(ExtensionContext context) {
+ return Stream.of(
+ argsOf(
+ "json(w/o meta, compression:none) to csv(compression:none)",
+ ARCHIVE_0_JSON_BASE.formatted("null"),
+ ARCHIVE_0_CSV_COMPRESSION_NONE,
+ Map.of("out_tile_compression", "none")
+ ),
+ argsOf(
+ "json(w/o meta, compression:none) to csv(compression:gzip)",
+ ARCHIVE_0_JSON_BASE.formatted("null"),
+ compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE),
+ Map.of("out_tile_compression", "gzip")
+ ),
+ argsOf(
+ "json(w/o meta, compression:gzip) to csv(compression:none)",
+ compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")),
+ ARCHIVE_0_CSV_COMPRESSION_NONE,
+ Map.of("out_tile_compression", "none")
+ ),
+ argsOf(
+ "json(w/o meta, compression:gzip) to csv(compression:gzip)",
+ compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")),
+ compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE),
+ Map.of("out_tile_compression", "gzip")
+ ),
+ argsOf(
+ "json(w/o meta, compression:gzip) to csv(compression:gzip)",
+ compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")),
+ compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE),
+ Map.of("out_tile_compression", "gzip")
+ ),
+ argsOf(
+ "json(w/ meta, compression:gzip) to csv(compression:none)",
+ compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)),
+ ARCHIVE_0_CSV_COMPRESSION_NONE,
+ Map.of("out_tile_compression", "none")
+ ),
+ argsOf(
+ "json(w/ meta, compression:gzip) to json(w/ meta, compression:gzip)",
+ compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)),
+ compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)),
+ Map.of()
+ ),
+ argsOf(
+ "csv to json - use fallback metadata",
+ ARCHIVE_0_CSV_COMPRESSION_NONE,
+ ARCHIVE_0_JSON_BASE.formatted(
+ "{\"name\":\"unknown\",\"format\":\"pbf\",\"minzoom\":\"0\",\"maxzoom\":\"14\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"),
+ Map.of("out_tile_compression", "none")
+ ),
+ argsOf(
+ "csv to json - use external metadata",
+ ARCHIVE_0_CSV_COMPRESSION_NONE,
+ ARCHIVE_0_JSON_BASE
+ .formatted("{\"name\":\"blub\",\"compression\":\"none\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"),
+ Map.of("out_tile_compression", "none", "in_metadata", "blub")
+ ),
+ argsOf(
+ "csv to json - null handling",
+ replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""),
+ replaceBase64(ARCHIVE_0_JSON_BASE
+ .formatted("{\"name\":\"blub\",\"compression\":\"gzip\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"), "null")
+ .replace(",\"encodedData\":\"null\"", ""),
+ Map.of("in_metadata", "blub", "skip_empty", "false")
+ ),
+ argsOf(
+ "json to csv - null handling",
+ replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null")
+ .replace(",\"encodedData\":\"null\"", ""),
+ replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""),
+ Map.of("skip_empty", "false")
+ ),
+ argsOf(
+ "csv to csv - empty skipping on",
+ """
+ 0,0,0,
+ 1,2,3,AQ==
+ """,
+ """
+ 1,2,3,AQ==
+ """,
+ Map.of("skip_empty", "true", "out_tile_compression", "none")
+ ),
+ argsOf(
+ "csv to csv - empty skipping off",
+ """
+ 0,0,0,
+ 1,2,3,AQ==
+ """,
+ """
+ 0,0,0,
+ 1,2,3,AQ==
+ """,
+ Map.of("skip_empty", "false", "out_tile_compression", "none")
+ ),
+ argsOf(
+ "tiles in order",
+ """
+ 1,2,3,AQ==
+ 0,0,0,AA==
+ """,
+ """
+ 1,2,3,AQ==
+ 0,0,0,AA==
+ """,
+ Map.of("out_tile_compression", "none")
+ ),
+ argsOf(
+ "tiles re-order",
+ """
+ 0,0,1,AQ==
+ 0,0,0,AA==
+ """,
+ """
+ 0,0,0,AA==
+ 0,0,1,AQ==
+ """,
+ Map.of("out_tile_compression", "none", "scan_tiles_in_order", "false", "filter_maxzoom", "1")
+ ),
+ argsOf(
+ "filter min zoom",
+ """
+ 0,0,0,AA==
+ 0,0,1,AQ==
+ """,
+ """
+ 0,0,1,AQ==
+ """,
+ Map.of("out_tile_compression", "none", "filter_minzoom", "1")
+ ),
+ argsOf(
+ "filter max zoom",
+ """
+ 0,0,1,AQ==
+ 0,0,0,AA==
+ """,
+ """
+ 0,0,0,AA==
+ """,
+ Map.of("out_tile_compression", "none", "filter_maxzoom", "0")
+ )
+ );
+ }
+
+ private static org.junit.jupiter.params.provider.Arguments argsOf(String testName, String archiveDataIn,
+ String archiveDataOut, Map arguments) {
+ return org.junit.jupiter.params.provider.Arguments.of(testName, archiveDataIn, archiveDataOut, arguments);
+ }
+ }
+
+}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java
index ad9d0038..127030bf 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java
@@ -49,8 +49,12 @@ class ReadableFilesArchiveTest {
Files.write(files.get(i), new byte[]{(byte) i});
}
- try (var reader = ReadableFilesArchive.newReader(tilesDir, Arguments.of())) {
- final List tiles = reader.getAllTiles().stream().sorted().toList();
+ try (
+ var reader = ReadableFilesArchive.newReader(tilesDir, Arguments.of());
+ var t = reader.getAllTiles();
+ var s = t.stream()
+ ) {
+ final List tiles = s.sorted().toList();
assertEquals(
List.of(
new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java
deleted file mode 100644
index 1d30afb1..00000000
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java
+++ /dev/null
@@ -1,119 +0,0 @@
-package com.onthegomap.planetiler.stream;
-
-import com.onthegomap.planetiler.archive.ReadableTileArchive;
-import com.onthegomap.planetiler.archive.TileArchiveMetadata;
-import com.onthegomap.planetiler.archive.TileEncodingResult;
-import com.onthegomap.planetiler.geo.TileCoord;
-import com.onthegomap.planetiler.proto.StreamArchiveProto;
-import com.onthegomap.planetiler.util.CloseableIterator;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.OptionalLong;
-
-public class InMemoryStreamArchive implements ReadableTileArchive {
-
- private final List tileEncodings;
- private final TileArchiveMetadata metadata;
-
- private InMemoryStreamArchive(List tileEncodings, TileArchiveMetadata metadata) {
- this.tileEncodings = tileEncodings;
- this.metadata = metadata;
- }
-
- public static InMemoryStreamArchive fromCsv(Path p, String columnSepatator) throws IOException {
- var base64Decoder = Base64.getDecoder();
- final List tileEncodings = new ArrayList<>();
- try (var reader = Files.newBufferedReader(p)) {
- String line;
- while ((line = reader.readLine()) != null) {
- final String[] splits = line.split(columnSepatator);
- final TileCoord tileCoord = TileCoord.ofXYZ(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]),
- Integer.parseInt(splits[2]));
- tileEncodings.add(new TileEncodingResult(tileCoord, base64Decoder.decode(splits[3]), OptionalLong.empty()));
- }
- }
- return new InMemoryStreamArchive(tileEncodings, null);
- }
-
- public static InMemoryStreamArchive fromProtobuf(Path p) throws IOException {
- final List tileEncodings = new ArrayList<>();
- try (var in = Files.newInputStream(p)) {
- StreamArchiveProto.Entry entry;
- while ((entry = StreamArchiveProto.Entry.parseDelimitedFrom(in)) != null) {
- if (entry.getEntryCase() == StreamArchiveProto.Entry.EntryCase.TILE) {
- final StreamArchiveProto.TileEntry tileProto = entry.getTile();
- final TileCoord tileCoord = TileCoord.ofXYZ(tileProto.getX(), tileProto.getY(), tileProto.getZ());
- tileEncodings
- .add(new TileEncodingResult(tileCoord, tileProto.getEncodedData().toByteArray(), OptionalLong.empty()));
- }
- }
- }
- return new InMemoryStreamArchive(tileEncodings, null /* could add once the format is finalized*/);
- }
-
- public static InMemoryStreamArchive fromJson(Path p) throws IOException {
- final List tileEncodings = new ArrayList<>();
- final TileArchiveMetadata[] metadata = new TileArchiveMetadata[]{null};
- try (var reader = Files.newBufferedReader(p)) {
- WriteableJsonStreamArchive.jsonMapper
- .readerFor(WriteableJsonStreamArchive.Entry.class)
- .readValues(reader)
- .forEachRemaining(entry -> {
- if (entry instanceof WriteableJsonStreamArchive.TileEntry te) {
- final TileCoord tileCoord = TileCoord.ofXYZ(te.x(), te.y(), te.z());
- tileEncodings.add(new TileEncodingResult(tileCoord, te.encodedData(), OptionalLong.empty()));
- } else if (entry instanceof WriteableJsonStreamArchive.FinishEntry fe) {
- metadata[0] = fe.metadata();
- }
- });
- }
- return new InMemoryStreamArchive(tileEncodings, Objects.requireNonNull(metadata[0]));
- }
-
- @Override
- public void close() throws IOException {}
-
- @Override
- public byte[] getTile(int x, int y, int z) {
-
- final TileCoord coord = TileCoord.ofXYZ(x, y, z);
-
- return tileEncodings.stream()
- .filter(ter -> ter.coord().equals(coord)).findFirst()
- .map(TileEncodingResult::tileData)
- .orElse(null);
- }
-
- @Override
- public CloseableIterator getAllTileCoords() {
-
- final Iterator it = tileEncodings.iterator();
-
- return new CloseableIterator() {
- @Override
- public TileCoord next() {
- return it.next().coord();
- }
-
- @Override
- public boolean hasNext() {
- return it.hasNext();
- }
-
- @Override
- public void close() {}
- };
- }
-
- @Override
- public TileArchiveMetadata metadata() {
- return metadata;
- }
-
-}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java
new file mode 100644
index 00000000..0f08420a
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java
@@ -0,0 +1,79 @@
+package com.onthegomap.planetiler.stream;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveConfig;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.geo.TileCoord;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.text.StringEscapeUtils;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+class ReadableCsvStreamArchiveTest {
+
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '$', textBlock = """
+ ,$\\n$false$BASE64
+ ,$\\r$false$BASE64
+ ,$\\r\\n$false$BASE64
+ ,$x$false$BASE64
+ ;$\\n$false$BASE64
+ {$\\n$false$BASE64
+ ,${$false$BASE64
+ ,$\\n$false$HEX
+ ,$\\n$true$BASE64
+ """)
+ void testSimple(String columnSeparator, String lineSeparator, boolean pad, CsvBinaryEncoding encoding,
+ @TempDir Path tempDir) throws IOException {
+
+ final Path csvFile = tempDir.resolve("in.csv");
+ final String csv =
+ """
+ 0,0,0,AA==
+ 1,2,3,AQ==
+ """
+ .replace("\n", StringEscapeUtils.unescapeJava(lineSeparator))
+ .replace(",", columnSeparator + (pad ? " " : ""))
+ .replace("AA==", encoding == CsvBinaryEncoding.BASE64 ? "AA==" : "00")
+ .replace("AQ==", encoding == CsvBinaryEncoding.BASE64 ? "AQ==" : "01");
+
+ Files.writeString(csvFile, csv);
+ final StreamArchiveConfig config = new StreamArchiveConfig(
+ false,
+ Arguments.of(Map.of(
+ StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, columnSeparator,
+ StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, lineSeparator,
+ StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, encoding.id()
+ ))
+ );
+
+ final List expectedTiles = List.of(
+ new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
+ new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1})
+ );
+
+ try (var reader = ReadableCsvArchive.newReader(TileArchiveConfig.Format.CSV, csvFile, config)) {
+ try (var s = reader.getAllTiles().stream()) {
+ assertEquals(expectedTiles, s.toList());
+ }
+ try (var s = reader.getAllTiles().stream()) {
+ assertEquals(expectedTiles, s.toList());
+ }
+ assertNull(reader.metadata());
+ assertNull(reader.metadata());
+ assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3)));
+ assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0));
+ assertNull(reader.getTile(4, 5, 6));
+ }
+ }
+}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java
new file mode 100644
index 00000000..f95145b9
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java
@@ -0,0 +1,52 @@
+package com.onthegomap.planetiler.stream;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import com.onthegomap.planetiler.TestUtils;
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.geo.TileCoord;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class ReadableJsonStreamArchiveTest {
+
+ @Test
+ void testSimple(@TempDir Path tempDir) throws IOException {
+
+ final Path jsonFile = tempDir.resolve("in.json");
+ final String json = """
+ {"type":"initialization"}
+ {"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
+ {"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
+ {"type":"finish","metadata":%s}
+ """.formatted(TestUtils.MAX_METADATA_SERIALIZED);
+
+ Files.writeString(jsonFile, json);
+ final StreamArchiveConfig config = new StreamArchiveConfig(false, Arguments.of());
+
+ final List expectedTiles = List.of(
+ new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
+ new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1})
+ );
+ try (var reader = ReadableJsonStreamArchive.newReader(jsonFile, config)) {
+ try (var s = reader.getAllTiles().stream()) {
+ assertEquals(expectedTiles, s.toList());
+ }
+ try (var s = reader.getAllTiles().stream()) {
+ assertEquals(expectedTiles, s.toList());
+ }
+ assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata());
+ assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata());
+ assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3)));
+ assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0));
+ assertNull(reader.getTile(4, 5, 6));
+ }
+ }
+}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java
new file mode 100644
index 00000000..6ffd685c
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java
@@ -0,0 +1,65 @@
+package com.onthegomap.planetiler.stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.google.protobuf.ByteString;
+import com.onthegomap.planetiler.archive.Tile;
+import com.onthegomap.planetiler.archive.TileArchiveMetadata;
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.geo.TileCoord;
+import com.onthegomap.planetiler.proto.StreamArchiveProto;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class ReadableProtoStreamArchiveTest {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testSimple(boolean maxMetaData, @TempDir Path tempDir) throws IOException {
+
+ final StreamArchiveProto.Metadata metadataSerialized = maxMetaData ?
+ WriteableProtoStreamArchiveTest.maxMetadataSerialized : WriteableProtoStreamArchiveTest.minMetadataSerialized;
+
+ final TileArchiveMetadata metadataDeserialized = maxMetaData ?
+ WriteableProtoStreamArchiveTest.maxMetadataDeserialized : WriteableProtoStreamArchiveTest.minMetadataDeserialized;
+
+
+ final Path p = tempDir.resolve("out.proto");
+ try (var out = Files.newOutputStream(p)) {
+ StreamArchiveProto.Entry.newBuilder().setInitialization(
+ StreamArchiveProto.InitializationEntry.newBuilder()
+ ).build().writeDelimitedTo(out);
+ StreamArchiveProto.Entry.newBuilder().setTile(
+ StreamArchiveProto.TileEntry.newBuilder()
+ .setX(0).setY(0).setZ(0).setEncodedData(ByteString.copyFrom(new byte[]{0}))
+ ).build().writeDelimitedTo(out);
+ StreamArchiveProto.Entry.newBuilder().setTile(
+ StreamArchiveProto.TileEntry.newBuilder()
+ .setX(1).setY(2).setZ(3).setEncodedData(ByteString.copyFrom(new byte[]{1}))
+ ).build().writeDelimitedTo(out);
+ StreamArchiveProto.Entry.newBuilder().setFinish(
+ StreamArchiveProto.FinishEntry.newBuilder()
+ .setMetadata(metadataSerialized)
+ ).build().writeDelimitedTo(out);
+ }
+ final List expectedTiles = List.of(
+ new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
+ new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1})
+ );
+ try (var reader = ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()))) {
+ try (var s = reader.getAllTiles().stream()) {
+ assertEquals(expectedTiles, s.toList());
+ }
+ try (var s = reader.getAllTiles().stream()) {
+ assertEquals(expectedTiles, s.toList());
+ }
+ assertEquals(metadataDeserialized, reader.metadata());
+ assertEquals(metadataDeserialized, reader.metadata());
+ }
+ }
+}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
index 48d05956..e69d08e5 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
@@ -113,7 +113,7 @@ class WriteableCsvArchiveTest {
void testColumnSeparator(@TempDir Path tempDir) throws IOException {
final StreamArchiveConfig config =
- new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_COLUMN_SEPARATOR, "' '")));
+ new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, "' '")));
final String expectedCsv =
"""
@@ -128,7 +128,7 @@ class WriteableCsvArchiveTest {
void testLineSeparator(@TempDir Path tempDir) throws IOException {
final StreamArchiveConfig config =
- new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_LINE_SEPARTATOR, "'\\r'")));
+ new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, "'\\r'")));
final String expectedCsv =
"""
@@ -143,7 +143,7 @@ class WriteableCsvArchiveTest {
void testHexEncoding(@TempDir Path tempDir) throws IOException {
final StreamArchiveConfig config =
- new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_BINARY_ENCODING, "hex")));
+ new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, "hex")));
final String expectedCsv =
"""
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
index 934a62bf..846739ac 100644
--- a/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
@@ -28,8 +28,8 @@ import org.locationtech.jts.geom.Envelope;
class WriteableProtoStreamArchiveTest {
- private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null);
- private static final TileArchiveMetadata maxMetadataIn =
+ static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null);
+ static final TileArchiveMetadata maxMetadataDeserialized =
new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3),
new Coordinate(1.3, 3.7, 1.0), 2, 3,
TileArchiveMetadata.TileArchiveMetadataJson.create(
@@ -45,7 +45,7 @@ class WriteableProtoStreamArchiveTest {
),
Map.of("a", "b", "c", "d"),
TileCompression.GZIP);
- private static final StreamArchiveProto.Metadata maxMetadataOut = StreamArchiveProto.Metadata.newBuilder()
+ static final StreamArchiveProto.Metadata maxMetadataSerialized = StreamArchiveProto.Metadata.newBuilder()
.setName("name").setDescription("description").setAttribution("attribution").setVersion("version")
.setType("type").setFormat("format")
.setBounds(StreamArchiveProto.Envelope.newBuilder().setMinX(0).setMaxX(1).setMinY(2).setMaxY(3).build())
@@ -64,10 +64,10 @@ class WriteableProtoStreamArchiveTest {
.setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_GZIP)
.build();
- private static final TileArchiveMetadata minMetadataIn =
- new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null,
+ static final TileArchiveMetadata minMetadataDeserialized =
+ new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, Map.of(),
TileCompression.NONE);
- private static final StreamArchiveProto.Metadata minMetadataOut = StreamArchiveProto.Metadata.newBuilder()
+ static final StreamArchiveProto.Metadata minMetadataSerialized = StreamArchiveProto.Metadata.newBuilder()
.setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_NONE)
.build();
@@ -83,12 +83,12 @@ class WriteableProtoStreamArchiveTest {
tileWriter.write(tile0);
tileWriter.write(tile1);
}
- archive.finish(minMetadataIn);
+ archive.finish(minMetadataDeserialized);
}
try (InputStream in = Files.newInputStream(csvFile)) {
assertEquals(
- List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)),
+ List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataSerialized)),
readAllEntries(in)
);
}
@@ -119,12 +119,12 @@ class WriteableProtoStreamArchiveTest {
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(tile4);
}
- archive.finish(maxMetadataIn);
+ archive.finish(maxMetadataDeserialized);
}
try (InputStream in = Files.newInputStream(csvFilePrimary)) {
assertEquals(
- List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)),
+ List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataSerialized)),
readAllEntries(in)
);
}
diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java
new file mode 100644
index 00000000..33f16c90
--- /dev/null
+++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java
@@ -0,0 +1,36 @@
+package com.onthegomap.planetiler.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.List;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Test;
+
+class CloseableIteratorTest {
+
+ @Test
+ void testFilter() {
+ assertEquals(
+ List.of(2, 4),
+ CloseableIterator.of(Stream.of(1, 2, 3, 4, 5, 6))
+ .filter(i -> i == 2 || i == 4)
+ .stream()
+ .toList()
+ );
+ assertEquals(
+ List.of(),
+ CloseableIterator.of(Stream.of(100, 99, 98))
+ .filter(i -> i == 2 || i == 4)
+ .stream()
+ .toList()
+ );
+ assertEquals(
+ List.of(),
+ CloseableIterator.of(Stream.of())
+ .filter(i -> i == 2 || i == 4)
+ .stream()
+ .toList()
+ );
+ }
+
+}
diff --git a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
index becc425b..0387eb1e 100644
--- a/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
+++ b/planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
@@ -4,6 +4,7 @@ import static java.util.Map.entry;
import com.onthegomap.planetiler.benchmarks.LongLongMapBench;
import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping;
+import com.onthegomap.planetiler.copy.TileCopy;
import com.onthegomap.planetiler.custommap.ConfiguredMapMain;
import com.onthegomap.planetiler.custommap.validator.SchemaValidator;
import com.onthegomap.planetiler.examples.BikeRouteOverlay;
@@ -65,7 +66,9 @@ public class Main {
entry("verify-monaco", VerifyMonaco::main),
entry("stats", TileSizeStats::main),
entry("top-osm-tiles", TopOsmTiles::main),
- entry("compare", CompareArchives::main)
+ entry("compare", CompareArchives::main),
+
+ entry("tile-copy", TileCopy::main)
);
private static EntryPoint bundledSchema(String path) {