kopia lustrzana https://github.com/onthegomap/planetiler
refactoring
bbilger
rodzic
19c834fb5d
commit
be4994a44a
|
|
@ -267,11 +267,11 @@ public record TileArchiveConfig(
|
|||
public enum Format {
|
||||
MBTILES("mbtiles",
|
||||
false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */,
|
||||
false, TileOrder.TMS),
|
||||
PMTILES("pmtiles", false, false, TileOrder.HILBERT),
|
||||
false, false, TileOrder.TMS),
|
||||
PMTILES("pmtiles", false, false, false, TileOrder.HILBERT),
|
||||
|
||||
// should be before PBF in order to avoid collisions
|
||||
FILES("files", true, true, TileOrder.TMS) {
|
||||
FILES("files", true, true, true, TileOrder.TMS) {
|
||||
@Override
|
||||
boolean isUriSupported(URI uri) {
|
||||
final String path = uri.getPath();
|
||||
|
|
@ -280,25 +280,28 @@ public record TileArchiveConfig(
|
|||
}
|
||||
},
|
||||
|
||||
CSV("csv", true, true, TileOrder.TMS),
|
||||
CSV("csv", true, true, false, TileOrder.TMS),
|
||||
/** identical to {@link Format#CSV} - except for the column separator */
|
||||
TSV("tsv", true, true, TileOrder.TMS),
|
||||
TSV("tsv", true, true, false, TileOrder.TMS),
|
||||
|
||||
PROTO("proto", true, true, TileOrder.TMS),
|
||||
PROTO("proto", true, true, false, TileOrder.TMS),
|
||||
/** identical to {@link Format#PROTO} */
|
||||
PBF("pbf", true, true, TileOrder.TMS),
|
||||
PBF("pbf", true, true, false, TileOrder.TMS),
|
||||
|
||||
JSON("json", true, true, TileOrder.TMS);
|
||||
JSON("json", true, true, false, TileOrder.TMS);
|
||||
|
||||
private final String id;
|
||||
private final boolean supportsAppend;
|
||||
private final boolean supportsConcurrentWrites;
|
||||
private final boolean supportsConcurrentReads;
|
||||
private final TileOrder order;
|
||||
|
||||
Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, TileOrder order) {
|
||||
Format(String id, boolean supportsAppend, boolean supportsConcurrentWrites, boolean supportsConcurrentReads,
|
||||
TileOrder order) {
|
||||
this.id = id;
|
||||
this.supportsAppend = supportsAppend;
|
||||
this.supportsConcurrentWrites = supportsConcurrentWrites;
|
||||
this.supportsConcurrentReads = supportsConcurrentReads;
|
||||
this.order = order;
|
||||
}
|
||||
|
||||
|
|
@ -318,6 +321,10 @@ public record TileArchiveConfig(
|
|||
return supportsConcurrentWrites;
|
||||
}
|
||||
|
||||
public boolean supportsConcurrentReads() {
|
||||
return supportsConcurrentReads;
|
||||
}
|
||||
|
||||
boolean isUriSupported(URI uri) {
|
||||
final String path = uri.getPath();
|
||||
return path != null && path.endsWith("." + id);
|
||||
|
|
|
|||
|
|
@ -1,236 +0,0 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.CommonConfigs;
|
||||
import com.onthegomap.planetiler.stats.Counter;
|
||||
import com.onthegomap.planetiler.stats.ProcessInfo;
|
||||
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.util.Gzip;
|
||||
import com.onthegomap.planetiler.util.Hashing;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.UnaryOperator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Utility to copy/convert tiles and metadata from one archive into another.
|
||||
* <p>
|
||||
* Example usages:
|
||||
*
|
||||
* <pre>
|
||||
* --input=tiles.mbtiles --output=tiles.mbtiles
|
||||
* --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
|
||||
* --input=tiles.pmtiles --output=tiles.mbtiles
|
||||
* --input=tiles.mbtiles --output=tiles/
|
||||
* --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
|
||||
* --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
|
||||
* --input=tiles.mbtiles --output=tiles.proto
|
||||
* </pre>
|
||||
*/
|
||||
public class TileCopy {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TileCopy.class);
|
||||
|
||||
private final TileCopyConfig config;
|
||||
|
||||
private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter();
|
||||
|
||||
|
||||
TileCopy(TileCopyConfig config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public void run() throws IOException {
|
||||
|
||||
if (!config.inArchive().exists()) {
|
||||
throw new IllegalArgumentException("the input archive does not exist");
|
||||
}
|
||||
|
||||
config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads());
|
||||
|
||||
final var loggers = ProgressLoggers.create()
|
||||
.addRateCounter("tiles", tilesWrittenOverall::get);
|
||||
|
||||
try (
|
||||
var reader = TileArchives.newReader(config.inArchive(), config.inArguments());
|
||||
var writer = TileArchives.newWriter(config.outArchive(), config.outArguments())
|
||||
) {
|
||||
|
||||
final TileArchiveMetadata inMetadata = getInMetadata(reader);
|
||||
final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata);
|
||||
|
||||
writer.initialize();
|
||||
try (
|
||||
var rawTiles = reader.getAllTiles();
|
||||
var it = config.skipEmpty() ? rawTiles.filter(t -> t.bytes() != null && t.bytes().length > 0) : rawTiles
|
||||
) {
|
||||
|
||||
final var tileConverter = tileConverter(inMetadata.tileCompression(), outMetadata.tileCompression(), writer);
|
||||
var pipeline = WorkerPipeline.start("archive", config.stats())
|
||||
.readFrom("tiles", () -> it)
|
||||
.addBuffer("buffer", config.queueSize())
|
||||
.sinkTo("write", config.tileWriterThreads(), itt -> tileWriter(writer, itt, tileConverter));
|
||||
|
||||
final var f = pipeline.done().thenRun(() -> writer.finish(outMetadata));
|
||||
|
||||
loggers.awaitAndLog(f, config.logInterval());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void tileWriter(WriteableTileArchive archive, Iterable<Tile> itt,
|
||||
Function<Tile, TileEncodingResult> tileConverter) {
|
||||
|
||||
final Counter tilesWritten = tilesWrittenOverall.counterForThread();
|
||||
try (var tileWriter = archive.newTileWriter()) {
|
||||
for (Tile t : itt) {
|
||||
tileWriter.write(tileConverter.apply(t));
|
||||
tilesWritten.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static Function<Tile, TileEncodingResult> tileConverter(TileCompression inCompression,
|
||||
TileCompression outCompression, WriteableTileArchive writer) {
|
||||
|
||||
final UnaryOperator<byte[]> bytesReEncoder = bytesReEncoder(inCompression, outCompression);
|
||||
final Function<byte[], OptionalLong> hasher =
|
||||
writer.deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) :
|
||||
b -> OptionalLong.empty();
|
||||
|
||||
return t -> new TileEncodingResult(t.coord(), bytesReEncoder.apply(t.bytes()), hasher.apply(t.bytes()));
|
||||
}
|
||||
|
||||
private static UnaryOperator<byte[]> bytesReEncoder(TileCompression inCompression, TileCompression outCompression) {
|
||||
if (inCompression == outCompression) {
|
||||
return UnaryOperator.identity();
|
||||
} else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) {
|
||||
return Gzip::gunzip;
|
||||
} else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) {
|
||||
return Gzip::gzip;
|
||||
} else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.GZIP) {
|
||||
return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b);
|
||||
} else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.NONE) {
|
||||
return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b;
|
||||
} else {
|
||||
throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression);
|
||||
}
|
||||
}
|
||||
|
||||
private TileArchiveMetadata getInMetadata(ReadableTileArchive reader) {
|
||||
TileArchiveMetadata inMetadata = config.inMetadata();
|
||||
if (inMetadata == null) {
|
||||
inMetadata = reader.metadata();
|
||||
if (inMetadata == null) {
|
||||
LOGGER.atWarn()
|
||||
.log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata");
|
||||
inMetadata = fallbackMetadata();
|
||||
}
|
||||
}
|
||||
if (inMetadata.tileCompression() == null) {
|
||||
inMetadata = inMetadata.withTileCompression(config.inCompression());
|
||||
}
|
||||
|
||||
return inMetadata;
|
||||
}
|
||||
|
||||
private TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata) {
|
||||
if (config.outCompression() == TileCompression.UNKNWON && inMetadata.tileCompression() == TileCompression.UNKNWON) {
|
||||
return inMetadata.withTileCompression(TileCompression.GZIP);
|
||||
} else if (config.outCompression() != TileCompression.UNKNWON) {
|
||||
return inMetadata.withTileCompression(config.outCompression());
|
||||
} else {
|
||||
return inMetadata;
|
||||
}
|
||||
}
|
||||
|
||||
private static TileArchiveMetadata fallbackMetadata() {
|
||||
return new TileArchiveMetadata(
|
||||
"unknown",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers
|
||||
Map.of(),
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
record TileCopyConfig(
|
||||
TileArchiveConfig inArchive,
|
||||
TileArchiveConfig outArchive,
|
||||
Arguments inArguments,
|
||||
Arguments outArguments,
|
||||
TileCompression inCompression,
|
||||
TileCompression outCompression,
|
||||
int tileWriterThreads,
|
||||
Duration logInterval,
|
||||
Stats stats,
|
||||
int queueSize,
|
||||
boolean append,
|
||||
boolean force,
|
||||
TileArchiveMetadata inMetadata,
|
||||
boolean skipEmpty
|
||||
) {
|
||||
static TileCopyConfig fromArguments(Arguments baseArguments) {
|
||||
|
||||
final Arguments inArguments = baseArguments.withPrefix("in");
|
||||
final Arguments outArguments = baseArguments.withPrefix("out");
|
||||
final Arguments baseOrOutArguments = outArguments.orElse(baseArguments);
|
||||
|
||||
final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null);
|
||||
final TileArchiveMetadata inMetadata;
|
||||
if (inMetadataPath != null) {
|
||||
try {
|
||||
inMetadata =
|
||||
TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
} else {
|
||||
inMetadata = null;
|
||||
}
|
||||
|
||||
return new TileCopyConfig(
|
||||
TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")),
|
||||
TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")),
|
||||
inArguments,
|
||||
outArguments,
|
||||
getTileCompressionArg(inArguments, "the input tile compression"),
|
||||
getTileCompressionArg(outArguments, "the output tile compression"),
|
||||
CommonConfigs.tileWriterThreads(baseOrOutArguments),
|
||||
CommonConfigs.logInterval(baseArguments),
|
||||
baseArguments.getStats(),
|
||||
Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)),
|
||||
CommonConfigs.appendToArchive(baseOrOutArguments),
|
||||
CommonConfigs.force(baseOrOutArguments),
|
||||
inMetadata,
|
||||
baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", false)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static TileCompression getTileCompressionArg(Arguments args, String description) {
|
||||
return args.getObject("tile_compression", description, TileCompression.UNKNWON,
|
||||
v -> TileCompression.findById(v).orElseThrow());
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run();
|
||||
}
|
||||
}
|
||||
|
|
@ -45,11 +45,17 @@ public class Arguments {
|
|||
|
||||
private final UnaryOperator<String> provider;
|
||||
private final Supplier<? extends Collection<String>> keys;
|
||||
private final String logKeyPrefix;
|
||||
private boolean silent = false;
|
||||
|
||||
private Arguments(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys) {
|
||||
private Arguments(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys, String logKeyPrefix) {
|
||||
this.provider = provider;
|
||||
this.keys = keys;
|
||||
this.logKeyPrefix = logKeyPrefix;
|
||||
}
|
||||
|
||||
private Arguments(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys) {
|
||||
this(provider, keys, "");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -65,7 +71,7 @@ public class Arguments {
|
|||
}
|
||||
|
||||
static Arguments fromJvmProperties(UnaryOperator<String> getter, Supplier<? extends Collection<String>> keys) {
|
||||
return fromPrefixed(getter, keys, "planetiler", ".", false);
|
||||
return fromPrefixed(getter, keys, "planetiler", ".", false, "");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -81,7 +87,7 @@ public class Arguments {
|
|||
}
|
||||
|
||||
static Arguments fromEnvironment(UnaryOperator<String> getter, Supplier<Set<String>> keys) {
|
||||
return fromPrefixed(getter, keys, "PLANETILER", "_", true);
|
||||
return fromPrefixed(getter, keys, "PLANETILER", "_", true, "");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -213,21 +219,22 @@ public class Arguments {
|
|||
}
|
||||
|
||||
private static Arguments from(UnaryOperator<String> provider, Supplier<? extends Collection<String>> rawKeys,
|
||||
UnaryOperator<String> forward, UnaryOperator<String> reverse) {
|
||||
UnaryOperator<String> forward, UnaryOperator<String> reverse, String logKeyPrefix) {
|
||||
Supplier<List<String>> keys = () -> rawKeys.get().stream().flatMap(key -> {
|
||||
String reversed = reverse.apply(key);
|
||||
return normalize(key).equals(normalize(reversed)) ? Stream.empty() : Stream.of(reversed);
|
||||
}).toList();
|
||||
return new Arguments(key -> provider.apply(forward.apply(key)), keys);
|
||||
return new Arguments(key -> provider.apply(forward.apply(key)), keys, logKeyPrefix);
|
||||
}
|
||||
|
||||
private static Arguments fromPrefixed(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys,
|
||||
String prefix, String separator, boolean uppperCase) {
|
||||
String prefix, String separator, boolean uppperCase, String logKeyPrefix) {
|
||||
var prefixRegex = Pattern.compile("^" + Pattern.quote(normalize(prefix + separator, separator, uppperCase)),
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
return from(provider, keys,
|
||||
key -> normalize(prefix + separator + key, separator, uppperCase),
|
||||
key -> normalize(prefixRegex.matcher(key).replaceFirst(""))
|
||||
key -> normalize(prefixRegex.matcher(key).replaceFirst("")),
|
||||
logKeyPrefix
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -262,7 +269,8 @@ public class Arguments {
|
|||
() -> Stream.concat(
|
||||
other.keys.get().stream(),
|
||||
keys.get().stream()
|
||||
).distinct().toList()
|
||||
).distinct().toList(),
|
||||
other.logKeyPrefix
|
||||
);
|
||||
if (silent) {
|
||||
result.silence();
|
||||
|
|
@ -307,7 +315,7 @@ public class Arguments {
|
|||
|
||||
protected void logArgValue(String key, String description, Object result) {
|
||||
if (!silent && LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("argument: {}={} ({})", key.replaceFirst("\\|.*$", ""), result, description);
|
||||
LOGGER.debug("argument: {}{}={} ({})", logKeyPrefix, key.replaceFirst("\\|.*$", ""), result, description);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -532,7 +540,7 @@ public class Arguments {
|
|||
* Returns a new arguments instance that translates requests for a {@code "key"} to {@code "prefix_key"}.
|
||||
*/
|
||||
public Arguments withPrefix(String prefix) {
|
||||
return fromPrefixed(provider, keys, prefix, "_", false);
|
||||
return fromPrefixed(provider, keys, prefix, "_", false, logKeyPrefix + prefix + "_");
|
||||
}
|
||||
|
||||
/** Returns a view of this instance, that only supports requests for {@code allowedKeys}. */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import static com.onthegomap.planetiler.worker.Worker.joinFutures;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileArchives;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.stats.Counter;
|
||||
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
||||
import com.onthegomap.planetiler.worker.WorkQueue;
|
||||
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Utility to copy/convert tiles and metadata from one archive into another.
|
||||
* <p>
|
||||
* Example usages:
|
||||
*
|
||||
* <pre>
|
||||
* --input=tiles.mbtiles --output=tiles.mbtiles
|
||||
* --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
|
||||
* --input=tiles.pmtiles --output=tiles.mbtiles
|
||||
* --input=tiles.mbtiles --output=tiles/
|
||||
* --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
|
||||
* --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
|
||||
* --input=tiles.mbtiles --output=tiles.proto
|
||||
* </pre>
|
||||
*/
|
||||
public class TileCopy {
|
||||
|
||||
private final TileCopyConfig config;
|
||||
|
||||
private final Counter.MultiThreadCounter tilesReadOverall = Counter.newMultiThreadCounter();
|
||||
private final Counter.MultiThreadCounter tilesProcessedOverall = Counter.newMultiThreadCounter();
|
||||
private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter();
|
||||
|
||||
|
||||
TileCopy(TileCopyConfig config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public void run() throws IOException {
|
||||
|
||||
if (!config.inArchive().exists()) {
|
||||
throw new IllegalArgumentException("the input archive does not exist");
|
||||
}
|
||||
|
||||
config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads());
|
||||
|
||||
final var loggers = ProgressLoggers.create()
|
||||
.addRateCounter("tiles_read", tilesReadOverall::get)
|
||||
.addRateCounter("tiles_processed", tilesProcessedOverall::get)
|
||||
.addRateCounter("tiles_written", tilesWrittenOverall::get);
|
||||
|
||||
try (
|
||||
var reader = TileArchives.newReader(config.inArchive(), config.inArguments());
|
||||
var writer = TileArchives.newWriter(config.outArchive(), config.outArguments())
|
||||
) {
|
||||
|
||||
final TileCopyContext context = TileCopyContext.create(reader, writer, config);
|
||||
|
||||
final var pipeline = WorkerPipeline.start("copy", config.stats());
|
||||
final WorkQueue<TileCopyWorkItem> resultQueue =
|
||||
new WorkQueue<>("results", config.workQueueCapacity(), config.workQueueMaxBatch(), config.stats());
|
||||
|
||||
try (
|
||||
var it = TileCopyWorkItemGenerators.create(context)
|
||||
) {
|
||||
|
||||
writer.initialize();
|
||||
|
||||
final var readerBranch = pipeline
|
||||
.<TileCopyWorkItem>fromGenerator("iterator", next -> {
|
||||
try (resultQueue) {
|
||||
while (it.hasNext()) {
|
||||
final TileCopyWorkItem t = it.next();
|
||||
resultQueue.accept(t); // put to queue immediately => retain order
|
||||
next.accept(t);
|
||||
}
|
||||
}
|
||||
})
|
||||
.addBuffer("to_read", config.workQueueCapacity(), config.workQueueMaxBatch())
|
||||
.<TileCopyWorkItem>addWorker("read", config.tileReaderThreads(), (prev, next) -> {
|
||||
final Counter tilesRead = tilesReadOverall.counterForThread();
|
||||
for (var item : prev) {
|
||||
item.loadOriginalTileData();
|
||||
tilesRead.inc();
|
||||
next.accept(item);
|
||||
}
|
||||
})
|
||||
.addBuffer("to_process", config.workQueueCapacity(), config.workQueueMaxBatch())
|
||||
.sinkTo("process", config.tileReaderThreads(), prev -> {
|
||||
final Counter tilesProcessed = tilesProcessedOverall.counterForThread();
|
||||
for (var item : prev) {
|
||||
item.process();
|
||||
tilesProcessed.inc();
|
||||
}
|
||||
});
|
||||
|
||||
final var writerBranch = pipeline.readFromQueue(resultQueue)
|
||||
.sinkTo("write", config.tileWriterThreads(), prev -> {
|
||||
final Counter tilesWritten = tilesWrittenOverall.counterForThread();
|
||||
try (var tileWriter = writer.newTileWriter()) {
|
||||
for (var item : prev) {
|
||||
var result = item.toTileEncodingResult();
|
||||
if (result.tileData() == null && config.skipEmpty()) {
|
||||
continue;
|
||||
}
|
||||
tileWriter.write(result);
|
||||
tilesWritten.inc();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
final var writerDone = writerBranch.done().thenRun(() -> writer.finish(context.outMetadata()));
|
||||
final var readerDone = readerBranch.done();
|
||||
|
||||
loggers
|
||||
.newLine()
|
||||
.addPipelineStats(readerBranch)
|
||||
.addPipelineStats(writerBranch);
|
||||
|
||||
loggers.awaitAndLog(joinFutures(writerDone, readerDone), Duration.ofSeconds(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileArchiveConfig;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer;
|
||||
import com.onthegomap.planetiler.archive.TileCompression;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.CommonConfigs;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.stats.ProcessInfo;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
|
||||
record TileCopyConfig(
|
||||
TileArchiveConfig inArchive,
|
||||
TileArchiveConfig outArchive,
|
||||
Arguments inArguments,
|
||||
Arguments outArguments,
|
||||
TileCompression inCompression,
|
||||
TileCompression outCompression,
|
||||
int tileWriterThreads,
|
||||
int tileReaderThreads,
|
||||
int tileProcessorThreads,
|
||||
Duration logInterval,
|
||||
Stats stats,
|
||||
int queueSize,
|
||||
boolean append,
|
||||
boolean force,
|
||||
TileArchiveMetadata inMetadata,
|
||||
boolean skipEmpty,
|
||||
Envelope filterBounds,
|
||||
int filterMinzoom,
|
||||
int filterMaxzoom,
|
||||
boolean scanTilesInOrder,
|
||||
TileOrder outputTileOrder,
|
||||
int workQueueCapacity,
|
||||
int workQueueMaxBatch
|
||||
) {
|
||||
|
||||
TileCopyConfig {
|
||||
if (tileReaderThreads > 1 && !inArchive.format().supportsConcurrentReads()) {
|
||||
throw new IllegalArgumentException(inArchive.format().id() + " does not support concurrent reads");
|
||||
}
|
||||
if (tileWriterThreads > 1 && !outArchive.format().supportsConcurrentReads()) {
|
||||
throw new IllegalArgumentException(outArchive.format().id() + " does not support concurrent writes");
|
||||
}
|
||||
if (filterMinzoom > filterMaxzoom) {
|
||||
throw new IllegalArgumentException("require minzoom <= maxzoom");
|
||||
}
|
||||
}
|
||||
|
||||
static TileCopyConfig fromArguments(Arguments baseArguments) {
|
||||
|
||||
final Arguments inArguments = baseArguments.withPrefix("in");
|
||||
final Arguments outArguments = baseArguments.withPrefix("out");
|
||||
final Arguments baseOrOutArguments = outArguments.orElse(baseArguments);
|
||||
|
||||
final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null);
|
||||
final TileArchiveMetadata inMetadata;
|
||||
if (inMetadataPath != null) {
|
||||
try {
|
||||
inMetadata =
|
||||
TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
} else {
|
||||
inMetadata = null;
|
||||
}
|
||||
|
||||
return new TileCopyConfig(
|
||||
TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")),
|
||||
TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")),
|
||||
inArguments,
|
||||
outArguments,
|
||||
getTileCompressionArg(inArguments, "the input tile compression"),
|
||||
getTileCompressionArg(outArguments, "the output tile compression"),
|
||||
CommonConfigs.tileWriterThreads(baseOrOutArguments),
|
||||
baseArguments.getInteger("tile_read_threads", "number of threads used to read tile data", 1),
|
||||
baseArguments.getInteger("tile_process_threads", "number of threads used to process tile data",
|
||||
Math.max(1, Runtime.getRuntime().availableProcessors() - 2)),
|
||||
CommonConfigs.logInterval(baseArguments),
|
||||
baseArguments.getStats(),
|
||||
Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)),
|
||||
CommonConfigs.appendToArchive(baseOrOutArguments),
|
||||
CommonConfigs.force(baseOrOutArguments),
|
||||
inMetadata,
|
||||
baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", true),
|
||||
baseArguments.bounds("filter_bounds", "the bounds to filter"),
|
||||
baseArguments.getInteger("filter_minzoom", "the min zoom", 0),
|
||||
baseArguments.getInteger("filter_maxzoom", "the max zoom", 14),
|
||||
baseArguments.getBoolean("scan_tiles_in_order", "output the tiles in the same order they are in the", true),
|
||||
baseArguments.getObject("output_tile_order", "the output tile order (if not scanned)", null,
|
||||
s -> s == null ? null : TileOrder.valueOf(s.toUpperCase())),
|
||||
100_000,
|
||||
100
|
||||
);
|
||||
}
|
||||
|
||||
private static TileCompression getTileCompressionArg(Arguments args, String description) {
|
||||
return args.getObject("tile_compression", description, TileCompression.UNKNOWN,
|
||||
v -> TileCompression.findById(v).orElseThrow());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import com.onthegomap.planetiler.archive.ReadableTileArchive;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileCompression;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Stream;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
record TileCopyContext(
|
||||
TileArchiveMetadata inMetadata,
|
||||
TileArchiveMetadata outMetadata,
|
||||
TileCopyConfig config,
|
||||
ReadableTileArchive reader,
|
||||
WriteableTileArchive writer
|
||||
) {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyContext.class);
|
||||
|
||||
TileCompression inputCompression() {
|
||||
return inMetadata.tileCompression();
|
||||
}
|
||||
|
||||
TileCompression outputCompression() {
|
||||
return outMetadata.tileCompression();
|
||||
}
|
||||
|
||||
static TileCopyContext create(ReadableTileArchive reader, WriteableTileArchive writer, TileCopyConfig config) {
|
||||
final TileArchiveMetadata inMetadata = getInMetadata(reader, config);
|
||||
final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata, config);
|
||||
return new TileCopyContext(
|
||||
inMetadata,
|
||||
outMetadata,
|
||||
config,
|
||||
reader,
|
||||
writer
|
||||
);
|
||||
}
|
||||
|
||||
private static TileArchiveMetadata getInMetadata(ReadableTileArchive inArchive, TileCopyConfig config) {
|
||||
TileArchiveMetadata inMetadata = config.inMetadata();
|
||||
if (inMetadata == null) {
|
||||
inMetadata = inArchive.metadata();
|
||||
if (inMetadata == null) {
|
||||
LOGGER.atWarn()
|
||||
.log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata");
|
||||
inMetadata = fallbackMetadata();
|
||||
}
|
||||
}
|
||||
if (inMetadata.tileCompression() == null) {
|
||||
inMetadata = inMetadata.withTileCompression(config.inCompression());
|
||||
}
|
||||
|
||||
return inMetadata;
|
||||
}
|
||||
|
||||
private static TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata, TileCopyConfig config) {
|
||||
|
||||
final TileCompression tileCompression;
|
||||
if (config.outCompression() == TileCompression.UNKNOWN && inMetadata.tileCompression() == TileCompression.UNKNOWN) {
|
||||
tileCompression = TileCompression.GZIP;
|
||||
} else if (config.outCompression() != TileCompression.UNKNOWN) {
|
||||
tileCompression = config.outCompression();
|
||||
} else {
|
||||
tileCompression = inMetadata.tileCompression();
|
||||
}
|
||||
|
||||
final Envelope bounds;
|
||||
if (config.filterBounds() != null) {
|
||||
bounds = config.filterBounds();
|
||||
} else if (inMetadata.bounds() != null) {
|
||||
bounds = inMetadata.bounds();
|
||||
} else {
|
||||
bounds = null;
|
||||
}
|
||||
|
||||
final int minzoom = Stream.of(inMetadata.minzoom(), config.filterMinzoom()).filter(Objects::nonNull)
|
||||
.mapToInt(Integer::intValue).max().orElse(0);
|
||||
|
||||
final int maxzoom = Stream.of(inMetadata.maxzoom(), config.filterMaxzoom()).filter(Objects::nonNull)
|
||||
.mapToInt(Integer::intValue).min().orElse(0);
|
||||
|
||||
return new TileArchiveMetadata(
|
||||
inMetadata.name(),
|
||||
inMetadata.description(),
|
||||
inMetadata.attribution(),
|
||||
inMetadata.version(),
|
||||
inMetadata.type(),
|
||||
inMetadata.format(),
|
||||
bounds,
|
||||
inMetadata.center(),
|
||||
minzoom,
|
||||
maxzoom,
|
||||
inMetadata.json(),
|
||||
inMetadata.others(),
|
||||
tileCompression
|
||||
);
|
||||
}
|
||||
|
||||
private static TileArchiveMetadata fallbackMetadata() {
|
||||
return new TileArchiveMetadata(
|
||||
"unknown",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf
|
||||
null,
|
||||
null,
|
||||
0,
|
||||
14,
|
||||
new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers
|
||||
Map.of(),
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.function.UnaryOperator;
|
||||
|
||||
public class TileCopyWorkItem {
|
||||
private final TileCoord coord;
|
||||
private final Supplier<byte[]> originalTileDataLoader;
|
||||
private final UnaryOperator<byte[]> reEncoder;
|
||||
private final Function<byte[], OptionalLong> hasher;
|
||||
private final CompletableFuture<byte[]> originalData = new CompletableFuture<>();
|
||||
private final CompletableFuture<byte[]> reEncodedData = new CompletableFuture<>();
|
||||
private final CompletableFuture<OptionalLong> reEncodedDataHash = new CompletableFuture<>();
|
||||
|
||||
TileCopyWorkItem(
|
||||
TileCoord coord,
|
||||
Supplier<byte[]> originalTileDataLoader,
|
||||
UnaryOperator<byte[]> reEncoder,
|
||||
Function<byte[], OptionalLong> hasher
|
||||
) {
|
||||
this.coord = coord;
|
||||
this.originalTileDataLoader = originalTileDataLoader;
|
||||
this.reEncoder = reEncoder;
|
||||
this.hasher = hasher;
|
||||
}
|
||||
|
||||
public TileCoord getCoord() {
|
||||
return coord;
|
||||
}
|
||||
|
||||
void loadOriginalTileData() {
|
||||
originalData.complete(originalTileDataLoader.get());
|
||||
}
|
||||
|
||||
void process() throws ExecutionException, InterruptedException {
|
||||
final var reEncoded = reEncoder.apply(originalData.get());
|
||||
final var hash = hasher.apply(reEncoded);
|
||||
reEncodedData.complete(reEncoded);
|
||||
reEncodedDataHash.complete(hash);
|
||||
}
|
||||
|
||||
TileEncodingResult toTileEncodingResult() throws ExecutionException, InterruptedException {
|
||||
return new TileEncodingResult(coord, reEncodedData.get(), reEncodedDataHash.get());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import com.onthegomap.planetiler.archive.Tile;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveConfig;
|
||||
import com.onthegomap.planetiler.config.Bounds;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.geo.TileExtents;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.util.CloseableIterator;
|
||||
import com.onthegomap.planetiler.util.Hashing;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.function.UnaryOperator;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
final class TileCopyWorkItemGenerators {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(TileCopyWorkItemGenerators.class);
|
||||
|
||||
private TileCopyWorkItemGenerators() {}
|
||||
|
||||
@SuppressWarnings("java:S2095")
|
||||
static CloseableIterator<TileCopyWorkItem> create(TileCopyContext context) {
|
||||
|
||||
final TileArchiveConfig.Format inFormat = context.config().inArchive().format();
|
||||
final TileArchiveConfig.Format outFormat = context.config().outArchive().format();
|
||||
|
||||
final boolean inOrder;
|
||||
if (outFormat == TileArchiveConfig.Format.FILES) {
|
||||
inOrder = true; // always use the input formats native order when outputting files
|
||||
} else if (inFormat == TileArchiveConfig.Format.FILES) {
|
||||
inOrder = false; // never use the inOrder looper when using files since there's no order guarantee
|
||||
} else {
|
||||
inOrder = context.config().scanTilesInOrder();
|
||||
}
|
||||
|
||||
final int minzoom = context.outMetadata().minzoom();
|
||||
final int maxzoom = context.outMetadata().maxzoom();
|
||||
final Envelope filterBounds =
|
||||
context.outMetadata().bounds() == null ? Bounds.WORLD.world() :
|
||||
GeoUtils.toWorldBounds(context.outMetadata().bounds());
|
||||
final var boundsFilter = TileExtents.computeFromWorldBounds(maxzoom, filterBounds);
|
||||
final Predicate<TileCopyWorkItem> zoomFilter = i -> i.getCoord().z() >= minzoom && i.getCoord().z() <= maxzoom;
|
||||
|
||||
final ProcessorArgs processorArgs = new ProcessorArgs(
|
||||
TileDataReEncoders.create(context),
|
||||
context.writer().deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) :
|
||||
b -> OptionalLong.empty()
|
||||
);
|
||||
|
||||
if (inOrder) {
|
||||
CloseableIterator<TileCopyWorkItem> it = new EagerInOrder(context.reader().getAllTiles(), processorArgs)
|
||||
.filter(zoomFilter);
|
||||
if (!Objects.equals(context.inMetadata().bounds(), context.outMetadata().bounds())) {
|
||||
it = it.filter(i -> boundsFilter.test(i.getCoord()));
|
||||
}
|
||||
return it;
|
||||
|
||||
} else {
|
||||
|
||||
final boolean warnPoorlySupported = switch (inFormat) {
|
||||
case CSV, TSV, PROTO, PBF, JSON -> true;
|
||||
case PMTILES, MBTILES, FILES -> false;
|
||||
};
|
||||
if (warnPoorlySupported) {
|
||||
LOGGER.atWarn().log("{} random access is very slow", inFormat.id());
|
||||
}
|
||||
|
||||
final TileOrder tileOrder = Optional.ofNullable(context.config().outputTileOrder())
|
||||
.orElse(context.writer().tileOrder());
|
||||
|
||||
return new TileOrderLoop(tileOrder, minzoom, maxzoom, context.reader()::getTile, processorArgs)
|
||||
.filter(i -> boundsFilter.test(i.getCoord()));
|
||||
}
|
||||
}
|
||||
|
||||
private record EagerInOrder(
|
||||
CloseableIterator<Tile> it,
|
||||
ProcessorArgs processorArgs
|
||||
) implements CloseableIterator<TileCopyWorkItem> {
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
it.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return it.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileCopyWorkItem next() {
|
||||
if (!it.hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
final Tile t = it.next();
|
||||
return new TileCopyWorkItem(t.coord(), t::bytes, processorArgs.reEncoder(), processorArgs.hasher());
|
||||
}
|
||||
}
|
||||
|
||||
private static class TileOrderLoop implements CloseableIterator<TileCopyWorkItem> {
|
||||
|
||||
private final TileOrder tileOrder;
|
||||
private final int max;
|
||||
private final Function<TileCoord, byte[]> dataLoader;
|
||||
private final ProcessorArgs processorArgs;
|
||||
private int current;
|
||||
|
||||
TileOrderLoop(TileOrder tileOrder, int minZoom, int maxZoom, Function<TileCoord, byte[]> dataLoader,
|
||||
ProcessorArgs processorArgs) {
|
||||
this.tileOrder = tileOrder;
|
||||
this.current = TileCoord.startIndexForZoom(minZoom);
|
||||
this.max = TileCoord.endIndexForZoom(maxZoom);
|
||||
this.dataLoader = dataLoader;
|
||||
this.processorArgs = processorArgs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return current <= max;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileCopyWorkItem next() {
|
||||
if (!hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
final TileCoord c = tileOrder.decode(current++);
|
||||
return new TileCopyWorkItem(c, () -> dataLoader.apply(c), processorArgs.reEncoder(), processorArgs.hasher());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// nothing to close
|
||||
}
|
||||
}
|
||||
|
||||
private record ProcessorArgs(UnaryOperator<byte[]> reEncoder, Function<byte[], OptionalLong> hasher) {}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileCompression;
|
||||
import com.onthegomap.planetiler.util.Gzip;
|
||||
import java.util.function.UnaryOperator;
|
||||
|
||||
final class TileDataReEncoders {
|
||||
|
||||
private TileDataReEncoders() {}
|
||||
|
||||
static UnaryOperator<byte[]> create(TileCopyContext c) {
|
||||
// for now just one - but compose multiple as needed in the future (decompress, do something, compress)
|
||||
return reCompressor(c.inputCompression(), c.outputCompression());
|
||||
}
|
||||
|
||||
private static UnaryOperator<byte[]> reCompressor(TileCompression inCompression, TileCompression outCompression) {
|
||||
if (inCompression == outCompression) {
|
||||
return b -> b;
|
||||
} else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) {
|
||||
return Gzip::gunzip;
|
||||
} else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) {
|
||||
return Gzip::gzip;
|
||||
} else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.GZIP) {
|
||||
return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b);
|
||||
} else if (inCompression == TileCompression.UNKNOWN && outCompression == TileCompression.NONE) {
|
||||
return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b;
|
||||
} else {
|
||||
throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -42,10 +42,14 @@ public record TileCoord(int encoded, int x, int y, int z) implements Comparable<
|
|||
}
|
||||
}
|
||||
|
||||
private static int startIndexForZoom(int z) {
|
||||
public static int startIndexForZoom(int z) {
|
||||
return ZOOM_START_INDEX[z];
|
||||
}
|
||||
|
||||
public static int endIndexForZoom(int z) {
|
||||
return ZOOM_START_INDEX[z] + (1 << z) * (1 << z) - 1;
|
||||
}
|
||||
|
||||
private static int zoomForIndex(int idx) {
|
||||
for (int z = MAX_MAXZOOM; z >= 0; z--) {
|
||||
if (ZOOM_START_INDEX[z] <= idx) {
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ public class ReadableProtoStreamArchive extends ReadableStreamArchive<StreamArch
|
|||
|
||||
private TileCompression deserializeTileCompression(StreamArchiveProto.TileCompression s) {
|
||||
return switch (s) {
|
||||
case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNWON;
|
||||
case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNOWN;
|
||||
case TILE_COMPRESSION_GZIP -> TileCompression.GZIP;
|
||||
case TILE_COMPRESSION_NONE -> TileCompression.NONE;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ public class TestUtils {
|
|||
|
||||
public static final TileArchiveMetadata MAX_METADATA_DESERIALIZED =
|
||||
new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3),
|
||||
new Coordinate(1.3, 3.7, 1.0), 2, 3,
|
||||
new Coordinate(1.3, 3.7, 1.0), 0, 8,
|
||||
TileArchiveMetadata.TileArchiveMetadataJson.create(
|
||||
List.of(
|
||||
new LayerAttrStats.VectorLayer("vl0",
|
||||
|
|
@ -102,8 +102,8 @@ public class TestUtils {
|
|||
"version":"version",
|
||||
"type":"type",
|
||||
"format":"format",
|
||||
"minzoom":"2",
|
||||
"maxzoom":"3",
|
||||
"minzoom":"0",
|
||||
"maxzoom":"8",
|
||||
"compression":"gzip",
|
||||
"bounds":"0,2,1,3",
|
||||
"center":"1.3,3.7,1",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
package com.onthegomap.planetiler.copy;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ class TileCopyTest {
|
|||
"output", archiveOutPath.toString()
|
||||
)).orElse(Arguments.of(arguments));
|
||||
|
||||
new TileCopy(TileCopy.TileCopyConfig.fromArguments(args)).run();
|
||||
new TileCopy(TileCopyConfig.fromArguments(args)).run();
|
||||
|
||||
if (archiveDataOut.contains("{")) {
|
||||
final List<String> expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList();
|
||||
|
|
@ -142,28 +142,30 @@ class TileCopyTest {
|
|||
"csv to json - use fallback metadata",
|
||||
ARCHIVE_0_CSV_COMPRESSION_NONE,
|
||||
ARCHIVE_0_JSON_BASE.formatted(
|
||||
"{\"name\":\"unknown\",\"format\":\"pbf\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"),
|
||||
"{\"name\":\"unknown\",\"format\":\"pbf\",\"minzoom\":\"0\",\"maxzoom\":\"14\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"),
|
||||
Map.of("out_tile_compression", "none")
|
||||
),
|
||||
argsOf(
|
||||
"csv to json - use external metadata",
|
||||
ARCHIVE_0_CSV_COMPRESSION_NONE,
|
||||
ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"none\"}"),
|
||||
ARCHIVE_0_JSON_BASE
|
||||
.formatted("{\"name\":\"blub\",\"compression\":\"none\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"),
|
||||
Map.of("out_tile_compression", "none", "in_metadata", "blub")
|
||||
),
|
||||
argsOf(
|
||||
"csv to json - null handling",
|
||||
replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""),
|
||||
replaceBase64(ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"gzip\"}"), "null")
|
||||
replaceBase64(ARCHIVE_0_JSON_BASE
|
||||
.formatted("{\"name\":\"blub\",\"compression\":\"gzip\",\"minzoom\":\"0\",\"maxzoom\":\"14\"}"), "null")
|
||||
.replace(",\"encodedData\":\"null\"", ""),
|
||||
Map.of("in_metadata", "blub")
|
||||
Map.of("in_metadata", "blub", "skip_empty", "false")
|
||||
),
|
||||
argsOf(
|
||||
"json to csv - null handling",
|
||||
replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null")
|
||||
.replace(",\"encodedData\":\"null\"", ""),
|
||||
replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""),
|
||||
Map.of()
|
||||
Map.of("skip_empty", "false")
|
||||
),
|
||||
argsOf(
|
||||
"csv to csv - empty skipping on",
|
||||
|
|
@ -187,6 +189,52 @@ class TileCopyTest {
|
|||
1,2,3,AQ==
|
||||
""",
|
||||
Map.of("skip_empty", "false", "out_tile_compression", "none")
|
||||
),
|
||||
argsOf(
|
||||
"tiles in order",
|
||||
"""
|
||||
1,2,3,AQ==
|
||||
0,0,0,AA==
|
||||
""",
|
||||
"""
|
||||
1,2,3,AQ==
|
||||
0,0,0,AA==
|
||||
""",
|
||||
Map.of("out_tile_compression", "none")
|
||||
),
|
||||
argsOf(
|
||||
"tiles re-order",
|
||||
"""
|
||||
0,0,1,AQ==
|
||||
0,0,0,AA==
|
||||
""",
|
||||
"""
|
||||
0,0,0,AA==
|
||||
0,0,1,AQ==
|
||||
""",
|
||||
Map.of("out_tile_compression", "none", "scan_tiles_in_order", "false", "filter_maxzoom", "1")
|
||||
),
|
||||
argsOf(
|
||||
"filter min zoom",
|
||||
"""
|
||||
0,0,0,AA==
|
||||
0,0,1,AQ==
|
||||
""",
|
||||
"""
|
||||
0,0,1,AQ==
|
||||
""",
|
||||
Map.of("out_tile_compression", "none", "filter_minzoom", "1")
|
||||
),
|
||||
argsOf(
|
||||
"filter max zoom",
|
||||
"""
|
||||
0,0,1,AQ==
|
||||
0,0,0,AA==
|
||||
""",
|
||||
"""
|
||||
0,0,0,AA==
|
||||
""",
|
||||
Map.of("out_tile_compression", "none", "filter_maxzoom", "0")
|
||||
)
|
||||
);
|
||||
}
|
||||
|
|
@ -2,9 +2,9 @@ package com.onthegomap.planetiler;
|
|||
|
||||
import static java.util.Map.entry;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileCopy;
|
||||
import com.onthegomap.planetiler.benchmarks.LongLongMapBench;
|
||||
import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping;
|
||||
import com.onthegomap.planetiler.copy.TileCopy;
|
||||
import com.onthegomap.planetiler.custommap.ConfiguredMapMain;
|
||||
import com.onthegomap.planetiler.custommap.validator.SchemaValidator;
|
||||
import com.onthegomap.planetiler.examples.BikeRouteOverlay;
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue