mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

add tile-copy utility 

to copy tiles from one archive into another
e.g. mbtiles to files, pmtiles to mbtiles, ...
pull/772/head
bbilger 2024-01-06 20:58:49 +01:00
rodzic fa7bffb04f
commit 264b3515c4
40 zmienionych plików z 1512 dodań i 378 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

34
planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java
Wyświetl plik

@ -41,7 +41,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.IntStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -656,18 +655,8 @@ public class Planetiler {
System.exit(0);
} else if (onlyDownloadSources) {
// don't check files if not generating map
} else if (config.append()) {
if (!output.format().supportsAppend()) {
throw new IllegalArgumentException("cannot append to " + output.format().id());
}
if (!output.exists()) {
throw new IllegalArgumentException(output.uri() + " must exist when appending");
}
} else if (overwrite || config.force()) {
output.delete();
} else if (output.exists()) {
throw new IllegalArgumentException(
output.uri() + " already exists, use the --force argument to overwrite or --append.");
} else {
output.setup(config.force() || overwrite, config.append(), config.tileWriteThreads());
}
Path layerStatsPath = arguments.file("layer_stats", "layer stats output path",
@ -677,23 +666,6 @@ public class Planetiler {
if (config.tileWriteThreads() < 1) {
throw new IllegalArgumentException("require tile_write_threads >= 1");
}
if (config.tileWriteThreads() > 1) {
if (!output.format().supportsConcurrentWrites()) {
throw new IllegalArgumentException(output.format() + " doesn't support concurrent writes");
}
IntStream.range(1, config.tileWriteThreads())
.mapToObj(output::getPathForMultiThreadedWriter)
.forEach(p -> {
if (!config.append() && (overwrite || config.force())) {
FileUtils.delete(p);
}
if (config.append() && !output.exists(p)) {
throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending");
} else if (!config.append() && output.exists(p)) {
throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending");
}
});
}
LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri());
@ -718,7 +690,7 @@ public class Planetiler {
// in case any temp files are left from a previous run...
FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath);
Files.createDirectories(tmpDir);
FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath());
FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath);
if (!toDownload.isEmpty()) {
download();

41
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java
Wyświetl plik

@ -16,6 +16,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.IntStream;
import java.util.stream.Stream;
/**
@ -158,7 +159,7 @@ public record TileArchiveConfig(
/**
* Returns the local <b>base</b> path for this archive, for which directories should be pre-created for.
*/
public Path getLocalBasePath() {
Path getLocalBasePath() {
Path p = getLocalPath();
if (format() == Format.FILES) {
p = FilesArchiveUtils.cleanBasePath(p);
@ -166,7 +167,6 @@ public record TileArchiveConfig(
return p;
}
/**
* Deletes the archive if possible.
*/
@ -187,7 +187,7 @@ public record TileArchiveConfig(
* @param p path to the archive
* @return {@code true} if the archive already exists, {@code false} otherwise.
*/
public boolean exists(Path p) {
private boolean exists(Path p) {
if (p == null) {
return false;
}
@ -229,6 +229,41 @@ public record TileArchiveConfig(
};
}
public void setup(boolean force, boolean append, int tileWriteThreads) {
if (append) {
if (!format().supportsAppend()) {
throw new IllegalArgumentException("cannot append to " + format().id());
}
if (!exists()) {
throw new IllegalArgumentException(uri() + " must exist when appending");
}
} else if (force) {
delete();
} else if (exists()) {
throw new IllegalArgumentException(uri() + " already exists, use the --force argument to overwrite or --append.");
}
if (tileWriteThreads > 1) {
if (!format().supportsConcurrentWrites()) {
throw new IllegalArgumentException(format() + " doesn't support concurrent writes");
}
IntStream.range(1, tileWriteThreads)
.mapToObj(this::getPathForMultiThreadedWriter)
.forEach(p -> {
if (!append && force) {
FileUtils.delete(p);
}
if (append && !exists(p)) {
throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending");
} else if (!append && exists(p)) {
throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending");
}
});
}
FileUtils.createParentDirectories(getLocalBasePath());
}
public enum Format {
MBTILES("mbtiles",
false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */,

5
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java
Wyświetl plik

@ -170,6 +170,11 @@ public record TileArchiveMetadata(
maxzoom, json, others, tileCompression);
}
public TileArchiveMetadata withTileCompression(TileCompression tileCompression) {
return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, minzoom,
maxzoom, json, others, tileCompression);
}
/*
* few workarounds to make collect unknown fields to others work,
* because @JsonAnySetter does not yet work on constructor/creator arguments

10
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java
Wyświetl plik

@ -261,7 +261,7 @@ public class TileArchiveWriter {
* To optimize emitting many identical consecutive tiles (like large ocean areas), memoize output to avoid
* recomputing if the input hasn't changed.
*/
byte[] lastBytes = null, lastEncoded = null;
byte[] lastBytes = null;
Long lastTileDataHash = null;
boolean lastIsFill = false;
List<TileSizeStats.LayerStats> lastLayerStats = null;
@ -276,24 +276,22 @@ public class TileArchiveWriter {
for (int i = 0; i < batch.in.size(); i++) {
FeatureGroup.TileFeatures tileFeatures = batch.in.get(i);
featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed());
byte[] bytes, encoded;
byte[] bytes;
List<TileSizeStats.LayerStats> layerStats;
Long tileDataHash;
if (tileFeatures.hasSameContents(last)) {
bytes = lastBytes;
encoded = lastEncoded;
tileDataHash = lastTileDataHash;
layerStats = lastLayerStats;
memoizedTiles.inc();
} else {
VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater);
if (skipFilled && (lastIsFill = tile.containsOnlyFills())) {
encoded = null;
layerStats = null;
bytes = null;
} else {
var proto = tile.toProto();
encoded = proto.toByteArray();
var encoded = proto.toByteArray();
bytes = switch (config.tileCompression()) {
case GZIP -> gzip(encoded);
case NONE -> encoded;
@ -307,7 +305,6 @@ public class TileArchiveWriter {
}
}
lastLayerStats = layerStats;
lastEncoded = encoded;
lastBytes = bytes;
last = tileFeatures;
if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) {
@ -326,7 +323,6 @@ public class TileArchiveWriter {
new TileEncodingResult(
tileFeatures.tileCoord(),
bytes,
encoded.length,
tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash),
layerStatsRows
)

43
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java
Wyświetl plik

@ -1,17 +1,23 @@
package com.onthegomap.planetiler.archive;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.CommonConfigs;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.files.ReadableFilesArchive;
import com.onthegomap.planetiler.files.WriteableFilesArchive;
import com.onthegomap.planetiler.mbtiles.Mbtiles;
import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
import com.onthegomap.planetiler.stream.ReadableCsvArchive;
import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive;
import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive;
import com.onthegomap.planetiler.stream.StreamArchiveConfig;
import com.onthegomap.planetiler.stream.WriteableCsvArchive;
import com.onthegomap.planetiler.stream.WriteableJsonStreamArchive;
import com.onthegomap.planetiler.stream.WriteableProtoStreamArchive;
import java.io.IOException;
import java.nio.file.Path;
import java.util.function.Supplier;
/** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */
public class TileArchives {
@ -37,45 +43,58 @@ public class TileArchives {
return newReader(TileArchiveConfig.from(archive), config);
}
public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
throws IOException {
return newWriter(archive, config.arguments());
}
/**
* Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}.
*
* @throws IOException if an error occurs creating the resource.
*/
public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
public static WriteableTileArchive newWriter(TileArchiveConfig archive, Arguments baseArguments)
throws IOException {
var options = archive.applyFallbacks(config.arguments());
var options = archive.applyFallbacks(baseArguments);
var format = archive.format();
return switch (format) {
case MBTILES ->
// pass-through legacy arguments for fallback
Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments()
Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(baseArguments
.subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION)));
case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath());
case CSV, TSV -> WriteableCsvArchive.newWriteToFile(format, archive.getLocalPath(),
new StreamArchiveConfig(config, options));
new StreamArchiveConfig(baseArguments, options));
case PROTO, PBF -> WriteableProtoStreamArchive.newWriteToFile(archive.getLocalPath(),
new StreamArchiveConfig(config, options));
new StreamArchiveConfig(baseArguments, options));
case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(),
new StreamArchiveConfig(config, options));
case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, config.force() || config.append());
new StreamArchiveConfig(baseArguments, options));
case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options,
CommonConfigs.appendToArchive(baseArguments) || CommonConfigs.force(baseArguments));
};
}
public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
throws IOException {
return newReader(archive, config.arguments());
}
/**
* Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}.
*
* @throws IOException if an error occurs opening the resource.
*/
public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
public static ReadableTileArchive newReader(TileArchiveConfig archive, Arguments baseArguments)
throws IOException {
var options = archive.applyFallbacks(config.arguments());
var options = archive.applyFallbacks(baseArguments);
Supplier<StreamArchiveConfig> streamArchiveConfig = () -> new StreamArchiveConfig(baseArguments, options);
return switch (archive.format()) {
case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options);
case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath());
case CSV, TSV -> throw new UnsupportedOperationException("reading CSV is not supported");
case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported");
case JSON -> throw new UnsupportedOperationException("reading JSON is not supported");
case CSV, TSV ->
ReadableCsvArchive.newReader(archive.format(), archive.getLocalPath(), streamArchiveConfig.get());
case PROTO, PBF -> ReadableProtoStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get());
case JSON -> ReadableJsonStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get());
case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath(), options);
};
}

5
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java
Wyświetl plik

@ -54,10 +54,5 @@ public enum TileCompression {
public TileCompression deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
return findById(p.getValueAsString()).orElse(TileCompression.UNKNOWN);
}
@Override
public TileCompression getNullValue(DeserializationContext ctxt) {
return TileCompression.GZIP;
}
}
}

236
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCopy.java 100644
Wyświetl plik

@ -0,0 +1,236 @@
package com.onthegomap.planetiler.archive;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.CommonConfigs;
import com.onthegomap.planetiler.stats.Counter;
import com.onthegomap.planetiler.stats.ProcessInfo;
import com.onthegomap.planetiler.stats.ProgressLoggers;
import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.util.Gzip;
import com.onthegomap.planetiler.util.Hashing;
import com.onthegomap.planetiler.worker.WorkerPipeline;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Utility to copy/convert tiles and metadata from one archive into another.
* <p>
* Example usages:
*
* <pre>
* --input=tiles.mbtiles --output=tiles.mbtiles
* --input=tiles.mbtiles --output=tiles.pmtiles --skip_empty=false
* --input=tiles.pmtiles --output=tiles.mbtiles
* --input=tiles.mbtiles --output=tiles/
* --input=tiles.mbtiles --output=tiles.json --out_tile_compression=gzip
* --input=tiles.mbtiles --output=tiles.csv --out_tile_compression=none
* --input=tiles.mbtiles --output=tiles.proto
* </pre>
*/
public class TileCopy {
private static final Logger LOGGER = LoggerFactory.getLogger(TileCopy.class);
private final TileCopyConfig config;
private final Counter.MultiThreadCounter tilesWrittenOverall = Counter.newMultiThreadCounter();
TileCopy(TileCopyConfig config) {
this.config = config;
}
public void run() throws IOException {
if (!config.inArchive().exists()) {
throw new IllegalArgumentException("the input archive does not exist");
}
config.outArchive().setup(config.force(), config.append(), config.tileWriterThreads());
final var loggers = ProgressLoggers.create()
.addRateCounter("tiles", tilesWrittenOverall::get);
try (
var reader = TileArchives.newReader(config.inArchive(), config.inArguments());
var writer = TileArchives.newWriter(config.outArchive(), config.outArguments())
) {
final TileArchiveMetadata inMetadata = getInMetadata(reader);
final TileArchiveMetadata outMetadata = getOutMetadata(inMetadata);
writer.initialize();
try (
var rawTiles = reader.getAllTiles();
var it = config.skipEmpty() ? rawTiles.filter(t -> t.bytes() != null && t.bytes().length > 0) : rawTiles
) {
final var tileConverter = tileConverter(inMetadata.tileCompression(), outMetadata.tileCompression(), writer);
var pipeline = WorkerPipeline.start("archive", config.stats())
.readFrom("tiles", () -> it)
.addBuffer("buffer", config.queueSize())
.sinkTo("write", config.tileWriterThreads(), itt -> tileWriter(writer, itt, tileConverter));
final var f = pipeline.done().thenRun(() -> writer.finish(outMetadata));
loggers.awaitAndLog(f, config.logInterval());
}
}
}
private void tileWriter(WriteableTileArchive archive, Iterable<Tile> itt,
Function<Tile, TileEncodingResult> tileConverter) {
final Counter tilesWritten = tilesWrittenOverall.counterForThread();
try (var tileWriter = archive.newTileWriter()) {
for (Tile t : itt) {
tileWriter.write(tileConverter.apply(t));
tilesWritten.inc();
}
}
}
private static Function<Tile, TileEncodingResult> tileConverter(TileCompression inCompression,
TileCompression outCompression, WriteableTileArchive writer) {
final UnaryOperator<byte[]> bytesReEncoder = bytesReEncoder(inCompression, outCompression);
final Function<byte[], OptionalLong> hasher =
writer.deduplicates() ? b -> OptionalLong.of(Hashing.fnv1a64(b)) :
b -> OptionalLong.empty();
return t -> new TileEncodingResult(t.coord(), bytesReEncoder.apply(t.bytes()), hasher.apply(t.bytes()));
}
private static UnaryOperator<byte[]> bytesReEncoder(TileCompression inCompression, TileCompression outCompression) {
if (inCompression == outCompression) {
return UnaryOperator.identity();
} else if (inCompression == TileCompression.GZIP && outCompression == TileCompression.NONE) {
return Gzip::gunzip;
} else if (inCompression == TileCompression.NONE && outCompression == TileCompression.GZIP) {
return Gzip::gzip;
} else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.GZIP) {
return b -> Gzip.isZipped(b) ? b : Gzip.gzip(b);
} else if (inCompression == TileCompression.UNKNWON && outCompression == TileCompression.NONE) {
return b -> Gzip.isZipped(b) ? Gzip.gunzip(b) : b;
} else {
throw new IllegalArgumentException("unhandled case: in=" + inCompression + " out=" + outCompression);
}
}
private TileArchiveMetadata getInMetadata(ReadableTileArchive reader) {
TileArchiveMetadata inMetadata = config.inMetadata();
if (inMetadata == null) {
inMetadata = reader.metadata();
if (inMetadata == null) {
LOGGER.atWarn()
.log("the input archive does not contain any metadata using fallback - consider passing one via in_metadata");
inMetadata = fallbackMetadata();
}
}
if (inMetadata.tileCompression() == null) {
inMetadata = inMetadata.withTileCompression(config.inCompression());
}
return inMetadata;
}
private TileArchiveMetadata getOutMetadata(TileArchiveMetadata inMetadata) {
if (config.outCompression() == TileCompression.UNKNWON && inMetadata.tileCompression() == TileCompression.UNKNWON) {
return inMetadata.withTileCompression(TileCompression.GZIP);
} else if (config.outCompression() != TileCompression.UNKNWON) {
return inMetadata.withTileCompression(config.outCompression());
} else {
return inMetadata;
}
}
private static TileArchiveMetadata fallbackMetadata() {
return new TileArchiveMetadata(
"unknown",
null,
null,
null,
null,
TileArchiveMetadata.MVT_FORMAT, // have to guess here that it's pbf
null,
null,
null,
null,
new TileArchiveMetadata.TileArchiveMetadataJson(List.of()), // cannot provide any vector layers
Map.of(),
null
);
}
record TileCopyConfig(
TileArchiveConfig inArchive,
TileArchiveConfig outArchive,
Arguments inArguments,
Arguments outArguments,
TileCompression inCompression,
TileCompression outCompression,
int tileWriterThreads,
Duration logInterval,
Stats stats,
int queueSize,
boolean append,
boolean force,
TileArchiveMetadata inMetadata,
boolean skipEmpty
) {
static TileCopyConfig fromArguments(Arguments baseArguments) {
final Arguments inArguments = baseArguments.withPrefix("in");
final Arguments outArguments = baseArguments.withPrefix("out");
final Arguments baseOrOutArguments = outArguments.orElse(baseArguments);
final Path inMetadataPath = inArguments.file("metadata", "path to metadata.json to use instead", null);
final TileArchiveMetadata inMetadata;
if (inMetadataPath != null) {
try {
inMetadata =
TileArchiveMetadataDeSer.mbtilesMapper().readValue(inMetadataPath.toFile(), TileArchiveMetadata.class);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
} else {
inMetadata = null;
}
return new TileCopyConfig(
TileArchiveConfig.from(baseArguments.getString("input", "input tile archive")),
TileArchiveConfig.from(baseArguments.getString("output", "output tile archive")),
inArguments,
outArguments,
getTileCompressionArg(inArguments, "the input tile compression"),
getTileCompressionArg(outArguments, "the output tile compression"),
CommonConfigs.tileWriterThreads(baseOrOutArguments),
CommonConfigs.logInterval(baseArguments),
baseArguments.getStats(),
Math.max(100, (int) (5_000d * ProcessInfo.getMaxMemoryBytes() / 100_000_000_000d)),
CommonConfigs.appendToArchive(baseOrOutArguments),
CommonConfigs.force(baseOrOutArguments),
inMetadata,
baseArguments.getBoolean("skip_empty", "skip empty (null/zero-bytes) tiles", false)
);
}
}
private static TileCompression getTileCompressionArg(Arguments args, String description) {
return args.getObject("tile_compression", description, TileCompression.UNKNWON,
v -> TileCompression.findById(v).orElseThrow());
}
public static void main(String[] args) throws IOException {
new TileCopy(TileCopyConfig.fromArguments(Arguments.fromEnvOrArgs(args))).run();
}
}

6
planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileEncodingResult.java
Wyświetl plik

@ -5,12 +5,10 @@ import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.OptionalLong;
import javax.annotation.Nonnull;
public record TileEncodingResult(
TileCoord coord,
@Nonnull byte[] tileData,
int rawTileSize,
byte[] tileData,
/* will always be empty in non-compact mode and might also be empty in compact mode */
OptionalLong tileDataHash,
List<String> layerStats
@ -20,7 +18,7 @@ public record TileEncodingResult(
byte[] tileData,
OptionalLong tileDataHash
) {
this(coord, tileData, tileData.length, tileDataHash, List.of());
this(coord, tileData, tileDataHash, List.of());
}
@Override

31
planetiler-core/src/main/java/com/onthegomap/planetiler/config/CommonConfigs.java 100644
Wyświetl plik

@ -0,0 +1,31 @@
package com.onthegomap.planetiler.config;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import java.time.Duration;
import java.util.stream.Stream;
public final class CommonConfigs {
private CommonConfigs() {}
public static boolean force(Arguments arguments) {
return arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false);
}
public static boolean appendToArchive(Arguments arguments) {
return arguments.getBoolean("append",
"append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values())
.filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(),
false);
}
public static int tileWriterThreads(Arguments arguments) {
return arguments.getInteger("tile_write_threads",
"number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values())
.filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(),
1);
}
public static Duration logInterval(Arguments arguments) {
return arguments.getDuration("loginterval", "time between logs", "10s");
}
}

15
planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java
Wyświetl plik

@ -1,6 +1,5 @@
package com.onthegomap.planetiler.config;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileCompression;
import com.onthegomap.planetiler.collection.LongLongMap;
import com.onthegomap.planetiler.collection.Storage;
@ -132,19 +131,13 @@ public record PlanetilerConfig(
featureProcessThreads,
arguments.getInteger("feature_read_threads", "number of threads to use when reading features at tile write time",
threads < 32 ? 1 : 2),
arguments.getInteger("tile_write_threads",
"number of threads used to write tiles - only supported by " + Stream.of(TileArchiveConfig.Format.values())
.filter(TileArchiveConfig.Format::supportsConcurrentWrites).map(TileArchiveConfig.Format::id).toList(),
1),
arguments.getDuration("loginterval", "time between logs", "10s"),
CommonConfigs.tileWriterThreads(arguments),
CommonConfigs.logInterval(arguments),
minzoom,
maxzoom,
renderMaxzoom,
arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
arguments.getBoolean("append",
"append to the output file - only supported by " + Stream.of(TileArchiveConfig.Format.values())
.filter(TileArchiveConfig.Format::supportsAppend).map(TileArchiveConfig.Format::id).toList(),
false),
CommonConfigs.force(arguments),
CommonConfigs.appendToArchive(arguments),
arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),
arguments.getBoolean("mmap_temp", "use memory-mapped IO for temp feature files", true),
arguments.getInteger("sort_max_readers", "maximum number of concurrent read threads to use when sorting chunks",

11
planetiler-core/src/main/java/com/onthegomap/planetiler/files/WriteableFilesArchive.java
Wyświetl plik

@ -159,12 +159,17 @@ public class WriteableFilesArchive implements WriteableTileArchive {
}
lastCheckedFolder = folder;
try {
Files.write(file, data);
if (data == null) {
Files.createFile(file);
} else {
Files.write(file, data);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
bytesWritten.incBy(data.length);
if (data != null) {
bytesWritten.incBy(data.length);
}
}
@Override

7
planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Mbtiles.java
Wyświetl plik

@ -6,6 +6,7 @@ import com.onthegomap.planetiler.archive.ReadableTileArchive;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileArchiveMetadataDeSer;
import com.onthegomap.planetiler.archive.TileCompression;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.archive.WriteableTileArchive;
import com.onthegomap.planetiler.config.Arguments;
@ -847,7 +848,11 @@ public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive
*/
public TileArchiveMetadata get() {
Map<String, String> map = new HashMap<>(getAll());
return TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class);
var metadata = TileArchiveMetadataDeSer.mbtilesMapper().convertValue(map, TileArchiveMetadata.class);
if (metadata.tileCompression() == null) {
metadata = metadata.withTileCompression(TileCompression.GZIP);
}
return metadata;
}
}
}

7
planetiler-core/src/main/java/com/onthegomap/planetiler/mbtiles/Verify.java
Wyświetl plik

@ -6,7 +6,6 @@ import com.onthegomap.planetiler.VectorTile;
import com.onthegomap.planetiler.geo.GeometryException;
import com.onthegomap.planetiler.geo.TileCoord;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@ -92,11 +91,7 @@ public class Verify {
}
private static List<VectorTile.Feature> decode(byte[] zipped) {
try {
return VectorTile.decode(gunzip(zipped));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return VectorTile.decode(gunzip(zipped));
}
/**

9
planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java
Wyświetl plik

@ -57,8 +57,7 @@ public final class WriteablePmtiles implements WriteableTileArchive {
this.bytesWritten = bytesWritten;
}
private static Directories makeDirectoriesWithLeaves(List<Pmtiles.Entry> subEntries, int leafSize, int attemptNum)
throws IOException {
private static Directories makeDirectoriesWithLeaves(List<Pmtiles.Entry> subEntries, int leafSize, int attemptNum) {
LOGGER.info("Building directories with {} entries per leaf, attempt {}...", leafSize, attemptNum);
ArrayList<Pmtiles.Entry> rootEntries = new ArrayList<>();
ByteArrayList leavesOutputStream = new ByteArrayList();
@ -91,9 +90,8 @@ public final class WriteablePmtiles implements WriteableTileArchive {
*
* @param entries a sorted ObjectArrayList of all entries in the tileset.
* @return byte arrays of the root and all leaf directories, and the # of leaves.
* @throws IOException if compression fails
*/
static Directories makeDirectories(List<Pmtiles.Entry> entries) throws IOException {
static Directories makeDirectories(List<Pmtiles.Entry> entries) {
int maxEntriesRootOnly = 16384;
int attemptNum = 1;
if (entries.size() < maxEntriesRootOnly) {
@ -302,6 +300,9 @@ public final class WriteablePmtiles implements WriteableTileArchive {
long offset;
OptionalLong tileDataHashOpt = encodingResult.tileDataHash();
var data = encodingResult.tileData();
if (data == null) {
return;
}
TileCoord coord = encodingResult.coord();
long tileId = coord.hilbertEncoded();

50
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/CsvBinaryEncoding.java 100644
Wyświetl plik

@ -0,0 +1,50 @@
package com.onthegomap.planetiler.stream;
import com.google.common.base.Suppliers;
import java.util.Base64;
import java.util.HexFormat;
import java.util.List;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;
enum CsvBinaryEncoding {
BASE64("base64", () -> Base64.getEncoder()::encodeToString, () -> Base64.getDecoder()::decode),
HEX("hex", () -> HexFormat.of()::formatHex, () -> HexFormat.of()::parseHex);
private final String id;
private final Supplier<Function<byte[], String>> encoder;
private final Supplier<Function<String, byte[]>> decoder;
private CsvBinaryEncoding(String id, Supplier<Function<byte[], String>> encoder,
Supplier<Function<String, byte[]>> decoder) {
this.id = id;
this.encoder = Suppliers.memoize(encoder::get);
this.decoder = Suppliers.memoize(decoder::get);
}
String encode(byte[] b) {
return encoder.get().apply(b);
}
byte[] decode(String s) {
return decoder.get().apply(s);
}
static List<String> ids() {
return Stream.of(CsvBinaryEncoding.values()).map(CsvBinaryEncoding::id).toList();
}
static CsvBinaryEncoding fromId(String id) {
return Stream.of(CsvBinaryEncoding.values())
.filter(de -> de.id().equals(id))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(
"unexpected binary encoding - expected one of " + ids() + " but got " + id));
}
String id() {
return id;
}
}

46
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/JsonStreamArchiveEntry.java 100644
Wyświetl plik

@ -0,0 +1,46 @@
package com.onthegomap.planetiler.stream;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import java.util.Arrays;
import java.util.Objects;
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type")
@JsonSubTypes({
@JsonSubTypes.Type(value = JsonStreamArchiveEntry.TileEntry.class, name = "tile"),
@JsonSubTypes.Type(value = JsonStreamArchiveEntry.InitializationEntry.class, name = "initialization"),
@JsonSubTypes.Type(value = JsonStreamArchiveEntry.FinishEntry.class, name = "finish")
})
sealed interface JsonStreamArchiveEntry {
record TileEntry(int x, int y, int z, byte[] encodedData) implements JsonStreamArchiveEntry {
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(encodedData);
result = prime * result + Objects.hash(x, y, z);
return result;
}
@Override
public boolean equals(Object obj) {
return this == obj || (obj instanceof JsonStreamArchiveEntry.TileEntry tileEntry &&
Arrays.equals(encodedData, tileEntry.encodedData) && x == tileEntry.x && y == tileEntry.y && z == tileEntry.z);
}
@Override
public String toString() {
return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]";
}
}
record InitializationEntry() implements JsonStreamArchiveEntry {}
record FinishEntry(TileArchiveMetadata metadata) implements JsonStreamArchiveEntry {}
}

108
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableCsvArchive.java 100644
Wyświetl plik

@ -0,0 +1,108 @@
package com.onthegomap.planetiler.stream;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.util.CloseableIterator;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.Optional;
import java.util.Scanner;
import java.util.function.Function;
import java.util.regex.Pattern;
/**
* Reads tiles from a CSV file. Counterpart to {@link WriteableCsvArchive}.
* <p>
* Supported arguments:
* <dl>
* <dt>column_separator</dt>
* <dd>The column separator e.g. ",", ";", "\t"</dd>
* <dt>line_separator</dt>
* <dd>The line separator e.g. "\n", "\r", "\r\n"</dd>
* </dl>
*
* @see WriteableCsvArchive
*/
public class ReadableCsvArchive extends ReadableStreamArchive<String> {
private final Pattern columnSeparatorPattern;
private final Pattern lineSeparatorPattern;
private final Function<String, byte[]> tileDataDecoder;
private ReadableCsvArchive(TileArchiveConfig.Format format, Path basePath, StreamArchiveConfig config) {
super(basePath, config);
this.columnSeparatorPattern =
Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format)));
this.lineSeparatorPattern =
Pattern.compile(Pattern.quote(StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format)));
final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions());
this.tileDataDecoder = binaryEncoding::decode;
}
public static ReadableCsvArchive newReader(TileArchiveConfig.Format format, Path basePath,
StreamArchiveConfig config) {
return new ReadableCsvArchive(format, basePath, config);
}
@Override
CloseableIterator<String> createIterator() {
try {
@SuppressWarnings("java:S2095") final Scanner s =
new Scanner(basePath.toFile()).useDelimiter(lineSeparatorPattern);
return new CloseableIterator<>() {
@Override
public void close() {
s.close();
}
@Override
public boolean hasNext() {
return s.hasNext();
}
@Override
public String next() {
return s.next();
}
};
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
@Override
Optional<Tile> mapEntryToTile(String entry) {
final String[] splits = columnSeparatorPattern.split(entry);
final byte[] bytes;
if (splits.length == 4) {
bytes = tileDataDecoder.apply(splits[3].strip());
} else if (splits.length == 3) {
bytes = null;
} else {
throw new InvalidCsvFormat(entry.length() > 20 ? entry.substring(0, 20) + "..." : entry);
}
return Optional.of(new Tile(
TileCoord.ofXYZ(
Integer.parseInt(splits[0].strip()),
Integer.parseInt(splits[1].strip()),
Integer.parseInt(splits[2].strip())
),
bytes
));
}
@Override
Optional<TileArchiveMetadata> mapEntryToMetadata(String entry) {
return Optional.empty();
}
static class InvalidCsvFormat extends RuntimeException {
InvalidCsvFormat(String message) {
super(message);
}
}
}

82
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchive.java 100644
Wyświetl plik

@ -0,0 +1,82 @@
package com.onthegomap.planetiler.stream;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.util.CloseableIterator;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
/**
* Reads tiles and metadata from a delimited JSON file. Counterpart to {@link WriteableJsonStreamArchive}.
*
* @see WriteableJsonStreamArchive
*/
public class ReadableJsonStreamArchive extends ReadableStreamArchive<JsonStreamArchiveEntry> {
private ReadableJsonStreamArchive(Path basePath, StreamArchiveConfig config) {
super(basePath, config);
}
public static ReadableJsonStreamArchive newReader(Path basePath, StreamArchiveConfig config) {
return new ReadableJsonStreamArchive(basePath, config);
}
@Override
CloseableIterator<JsonStreamArchiveEntry> createIterator() {
BufferedReader reader = null;
try {
reader = Files.newBufferedReader(basePath);
final var readerFinal = reader;
final var it = StreamArchiveUtils.jsonMapperJsonStreamArchive
.readerFor(JsonStreamArchiveEntry.class)
.<JsonStreamArchiveEntry>readValues(readerFinal);
return new CloseableIterator<>() {
@Override
public void close() {
try {
readerFinal.close();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
public JsonStreamArchiveEntry next() {
return it.next();
}
};
} catch (IOException e) {
closeSilentlyOnError(reader);
throw new UncheckedIOException(e);
}
}
@Override
Optional<Tile> mapEntryToTile(JsonStreamArchiveEntry entry) {
if (entry instanceof JsonStreamArchiveEntry.TileEntry tileEntry) {
return Optional.of(new Tile(
TileCoord.ofXYZ(tileEntry.x(), tileEntry.y(), tileEntry.z()),
tileEntry.encodedData()
));
}
return Optional.empty();
}
@Override
Optional<TileArchiveMetadata> mapEntryToMetadata(JsonStreamArchiveEntry entry) {
if (entry instanceof JsonStreamArchiveEntry.FinishEntry finishEntry) {
return Optional.ofNullable(finishEntry.metadata());
}
return Optional.empty();
}
}

160
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchive.java 100644
Wyświetl plik

@ -0,0 +1,160 @@
package com.onthegomap.planetiler.stream;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileCompression;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.proto.StreamArchiveProto;
import com.onthegomap.planetiler.util.CloseableIterator;
import com.onthegomap.planetiler.util.LayerAttrStats;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.CoordinateXY;
import org.locationtech.jts.geom.Envelope;
/**
* Reads tiles and metadata from a delimited protobuf file. Counterpart to {@link WriteableProtoStreamArchive}.
*
* @see WriteableProtoStreamArchive
*/
public class ReadableProtoStreamArchive extends ReadableStreamArchive<StreamArchiveProto.Entry> {
private ReadableProtoStreamArchive(Path basePath, StreamArchiveConfig config) {
super(basePath, config);
}
public static ReadableProtoStreamArchive newReader(Path basePath, StreamArchiveConfig config) {
return new ReadableProtoStreamArchive(basePath, config);
}
@Override
CloseableIterator<StreamArchiveProto.Entry> createIterator() {
try {
@SuppressWarnings("java:S2095") var in = new FileInputStream(basePath.toFile());
return new CloseableIterator<>() {
private StreamArchiveProto.Entry nextValue;
@Override
public void close() {
closeUnchecked(in);
}
@Override
public boolean hasNext() {
if (nextValue != null) {
return true;
}
try {
nextValue = StreamArchiveProto.Entry.parseDelimitedFrom(in);
return nextValue != null;
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
@Override
public StreamArchiveProto.Entry next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
final StreamArchiveProto.Entry returnValue = nextValue;
nextValue = null;
return returnValue;
}
};
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
@Override
Optional<Tile> mapEntryToTile(StreamArchiveProto.Entry entry) {
if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.TILE) {
return Optional.empty();
}
final StreamArchiveProto.TileEntry tileEntry = entry.getTile();
return Optional.of(new Tile(
TileCoord.ofXYZ(tileEntry.getX(), tileEntry.getY(), tileEntry.getZ()),
tileEntry.getEncodedData().toByteArray()
));
}
@Override
Optional<TileArchiveMetadata> mapEntryToMetadata(StreamArchiveProto.Entry entry) {
if (entry.getEntryCase() != StreamArchiveProto.Entry.EntryCase.FINISH) {
return Optional.empty();
}
final StreamArchiveProto.Metadata metadata = entry.getFinish().getMetadata();
return Optional.of(new TileArchiveMetadata(
StringUtils.trimToNull(metadata.getName()),
StringUtils.trimToNull(metadata.getDescription()),
StringUtils.trimToNull(metadata.getAttribution()),
StringUtils.trimToNull(metadata.getVersion()),
StringUtils.trimToNull(metadata.getType()),
StringUtils.trimToNull(metadata.getFormat()),
deserializeEnvelope(metadata.hasBounds() ? metadata.getBounds() : null),
deserializeCoordinate(metadata.hasCenter() ? metadata.getCenter() : null),
metadata.hasMinZoom() ? metadata.getMinZoom() : null,
metadata.hasMaxZoom() ? metadata.getMaxZoom() : null,
extractMetadataJson(metadata),
metadata.getOthersMap(),
deserializeTileCompression(metadata.getTileCompression())
));
}
private Envelope deserializeEnvelope(StreamArchiveProto.Envelope s) {
return s == null ? null : new Envelope(s.getMinX(), s.getMaxX(), s.getMinY(), s.getMaxY());
}
private Coordinate deserializeCoordinate(StreamArchiveProto.Coordinate s) {
if (s == null) {
return null;
}
return s.hasZ() ? new Coordinate(s.getX(), s.getY(), s.getZ()) : new CoordinateXY(s.getX(), s.getY());
}
private TileCompression deserializeTileCompression(StreamArchiveProto.TileCompression s) {
return switch (s) {
case TILE_COMPRESSION_UNSPECIFIED, UNRECOGNIZED -> TileCompression.UNKNWON;
case TILE_COMPRESSION_GZIP -> TileCompression.GZIP;
case TILE_COMPRESSION_NONE -> TileCompression.NONE;
};
}
private TileArchiveMetadata.TileArchiveMetadataJson extractMetadataJson(StreamArchiveProto.Metadata s) {
final List<LayerAttrStats.VectorLayer> vl = deserializeVectorLayers(s.getVectorLayersList());
return vl.isEmpty() ? null : new TileArchiveMetadata.TileArchiveMetadataJson(vl);
}
private List<LayerAttrStats.VectorLayer> deserializeVectorLayers(List<StreamArchiveProto.VectorLayer> s) {
return s.stream()
.map(vl -> new LayerAttrStats.VectorLayer(
vl.getId(),
vl.getFieldsMap().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> deserializeFieldType(e.getValue()))),
Optional.ofNullable(StringUtils.trimToNull(vl.getDescription())),
vl.hasMinZoom() ? OptionalInt.of(vl.getMinZoom()) : OptionalInt.empty(),
vl.hasMaxZoom() ? OptionalInt.of(vl.getMaxZoom()) : OptionalInt.empty()
))
.toList();
}
private LayerAttrStats.FieldType deserializeFieldType(StreamArchiveProto.VectorLayer.FieldType s) {
return switch (s) {
case FIELD_TYPE_UNSPECIFIED, UNRECOGNIZED -> throw new IllegalArgumentException("unknown type");
case FIELD_TYPE_NUMBER -> LayerAttrStats.FieldType.NUMBER;
case FIELD_TYPE_BOOLEAN -> LayerAttrStats.FieldType.BOOLEAN;
case FIELD_TYPE_STRING -> LayerAttrStats.FieldType.STRING;
};
}
}

94
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/ReadableStreamArchive.java 100644
Wyświetl plik

@ -0,0 +1,94 @@
package com.onthegomap.planetiler.stream;
import com.google.common.base.Suppliers;
import com.onthegomap.planetiler.archive.ReadableTileArchive;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.util.CloseableIterator;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Optional;
import java.util.function.Supplier;
abstract class ReadableStreamArchive<E> implements ReadableTileArchive {
private final Supplier<TileArchiveMetadata> cachedMetadata = Suppliers.memoize(this::loadMetadata);
final Path basePath;
final StreamArchiveConfig config;
ReadableStreamArchive(Path basePath, StreamArchiveConfig config) {
this.basePath = basePath;
this.config = config;
}
@Override
public final byte[] getTile(TileCoord coord) {
return getAllTiles().stream().filter(c -> c.coord().equals(coord)).map(Tile::bytes).findFirst().orElse(null);
}
@Override
public final byte[] getTile(int x, int y, int z) {
return getTile(TileCoord.ofXYZ(x, y, z));
}
@Override
public final CloseableIterator<TileCoord> getAllTileCoords() {
return getAllTiles().map(Tile::coord);
}
@Override
public final CloseableIterator<Tile> getAllTiles() {
return createIterator()
.map(this::mapEntryToTile)
.filter(Optional::isPresent)
.map(Optional::get);
}
@Override
public final TileArchiveMetadata metadata() {
return cachedMetadata.get();
}
private TileArchiveMetadata loadMetadata() {
try (var it = createIterator()) {
return it.stream().map(this::mapEntryToMetadata).flatMap(Optional::stream).findFirst().orElse(null);
}
}
@Override
public void close() throws IOException {
// nothing to close
}
abstract CloseableIterator<E> createIterator();
abstract Optional<Tile> mapEntryToTile(E entry);
abstract Optional<TileArchiveMetadata> mapEntryToMetadata(E entry);
void closeSilentlyOnError(Closeable c) {
if (c == null) {
return;
}
try {
c.close();
} catch (Exception ignored) {
// ignore
}
}
@SuppressWarnings("java:S112")
void closeUnchecked(Closeable c) {
if (c == null) {
return;
}
try {
c.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

8
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveConfig.java
Wyświetl plik

@ -1,10 +1,10 @@
package com.onthegomap.planetiler.stream;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.config.PlanetilerConfig;
import com.onthegomap.planetiler.config.CommonConfigs;
public record StreamArchiveConfig(boolean appendToFile, Arguments moreOptions) {
public StreamArchiveConfig(PlanetilerConfig planetilerConfig, Arguments moreOptions) {
this(planetilerConfig.append(), moreOptions);
public record StreamArchiveConfig(boolean appendToFile, Arguments formatOptions) {
public StreamArchiveConfig(Arguments baseArguments, Arguments formatOptions) {
this(CommonConfigs.appendToArchive(baseArguments), formatOptions);
}
}

50
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/StreamArchiveUtils.java
Wyświetl plik

@ -1,5 +1,8 @@
package com.onthegomap.planetiler.stream;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.google.common.net.UrlEscapers;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.config.Arguments;
@ -12,8 +15,25 @@ import org.apache.commons.text.StringEscapeUtils;
public final class StreamArchiveUtils {
/**
* exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases
* => allow to output tiles, only
*/
private static final String JSON_OPTION_WRITE_TILES_ONLY = "tiles_only";
private static final String JSON_OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator";
static final String CSV_OPTION_COLUMN_SEPARATOR = "column_separator";
static final String CSV_OPTION_LINE_SEPARATOR = "line_separator";
static final String CSV_OPTION_BINARY_ENCODING = "binary_encoding";
private static final Pattern quotedPattern = Pattern.compile("^'(.+?)'$");
static final JsonMapper jsonMapperJsonStreamArchive = JsonMapper.builder()
.serializationInclusion(JsonInclude.Include.NON_ABSENT)
.addModule(new Jdk8Module())
.build();
private StreamArchiveUtils() {}
public static Path constructIndexedPath(Path basePath, int index) {
@ -39,6 +59,36 @@ public final class StreamArchiveUtils {
.translateEscapes();
}
static String jsonOptionRootValueSeparator(Arguments formatOptions) {
return getEscapedString(formatOptions, TileArchiveConfig.Format.JSON,
JSON_OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " "));
}
static boolean jsonOptionWriteTilesOnly(Arguments formatOptions) {
return formatOptions.getBoolean(JSON_OPTION_WRITE_TILES_ONLY, "write tiles, only", false);
}
static String csvOptionColumnSeparator(Arguments formatOptions, TileArchiveConfig.Format format) {
final String defaultColumnSeparator = switch (format) {
case CSV -> "','";
case TSV -> "'\\t'";
default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id());
};
return getEscapedString(formatOptions, format,
CSV_OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " "));
}
static String csvOptionLineSeparator(Arguments formatOptions, TileArchiveConfig.Format format) {
return StreamArchiveUtils.getEscapedString(formatOptions, format,
CSV_OPTION_LINE_SEPARATOR, "line separator", "'\\n'", List.of("\n", "\r\n"));
}
static CsvBinaryEncoding csvOptionBinaryEncoding(Arguments formatOptions) {
return CsvBinaryEncoding.fromId(formatOptions.getString(CSV_OPTION_BINARY_ENCODING,
"binary (tile) data encoding - one of " + CsvBinaryEncoding.ids(), "base64"));
}
private static String escapeJava(String s) {
if (!s.trim().equals(s)) {
s = "'" + s + "'";

58
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableCsvArchive.java
Wyświetl plik

@ -11,11 +11,7 @@ import java.io.UncheckedIOException;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Base64;
import java.util.HexFormat;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Stream;
/**
* Writes tile data into a CSV file (or pipe).
@ -67,31 +63,16 @@ import java.util.stream.Stream;
*/
public final class WriteableCsvArchive extends WriteableStreamArchive {
static final String OPTION_COLUMN_SEPARATOR = "column_separator";
static final String OPTION_LINE_SEPARTATOR = "line_separator";
static final String OPTION_BINARY_ENCODING = "binary_encoding";
private final String columnSeparator;
private final String lineSeparator;
private final Function<byte[], String> tileDataEncoder;
private WriteableCsvArchive(TileArchiveConfig.Format format, Path p, StreamArchiveConfig config) {
super(p, config);
final String defaultColumnSeparator = switch (format) {
case CSV -> "','";
case TSV -> "'\\t'";
default -> throw new IllegalArgumentException("supported formats are csv and tsv but got " + format.id());
};
this.columnSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format,
OPTION_COLUMN_SEPARATOR, "column separator", defaultColumnSeparator, List.of(",", " "));
this.lineSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), format,
OPTION_LINE_SEPARTATOR, "line separator", "'\\n'", List.of("\n", "\r\n"));
final BinaryEncoding binaryEncoding = BinaryEncoding.fromId(config.moreOptions().getString(OPTION_BINARY_ENCODING,
"binary (tile) data encoding - one of " + BinaryEncoding.ids(), "base64"));
this.tileDataEncoder = switch (binaryEncoding) {
case BASE64 -> Base64.getEncoder()::encodeToString;
case HEX -> HexFormat.of()::formatHex;
};
this.columnSeparator = StreamArchiveUtils.csvOptionColumnSeparator(config.formatOptions(), format);
this.lineSeparator = StreamArchiveUtils.csvOptionLineSeparator(config.formatOptions(), format);
final CsvBinaryEncoding binaryEncoding = StreamArchiveUtils.csvOptionBinaryEncoding(config.formatOptions());
this.tileDataEncoder = binaryEncoding::encode;
}
public static WriteableCsvArchive newWriteToFile(TileArchiveConfig.Format format, Path path,
@ -131,7 +112,8 @@ public final class WriteableCsvArchive extends WriteableStreamArchive {
@Override
public void write(TileEncodingResult encodingResult) {
final TileCoord coord = encodingResult.coord();
final String tileDataEncoded = tileDataEncoder.apply(encodingResult.tileData());
final byte[] data = encodingResult.tileData();
final String tileDataEncoded = data == null ? "" : tileDataEncoder.apply(encodingResult.tileData());
try {
// x | y | z | encoded data
writer.write("%d%s%d%s%d%s%s%s".formatted(coord.x(), columnSeparator, coord.y(), columnSeparator, coord.z(),
@ -150,32 +132,4 @@ public final class WriteableCsvArchive extends WriteableStreamArchive {
}
}
}
private enum BinaryEncoding {
BASE64("base64"),
HEX("hex");
private final String id;
private BinaryEncoding(String id) {
this.id = id;
}
static List<String> ids() {
return Stream.of(BinaryEncoding.values()).map(BinaryEncoding::id).toList();
}
static BinaryEncoding fromId(String id) {
return Stream.of(BinaryEncoding.values())
.filter(de -> de.id().equals(id))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(
"unexpected binary encoding - expected one of " + ids() + " but got " + id));
}
String id() {
return id;
}
}
}

92
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableJsonStreamArchive.java
Wyświetl plik

@ -1,14 +1,8 @@
package com.onthegomap.planetiler.stream;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SequenceWriter;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.geo.TileCoord;
@ -19,41 +13,26 @@ import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Writes JSON-serialized tile data as well as meta data into file(s). The entries are of type
* {@link WriteableJsonStreamArchive.Entry} are separated by newline (by default).
* {@link JsonStreamArchiveEntry} are separated by newline (by default).
*/
public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
private static final Logger LOGGER = LoggerFactory.getLogger(WriteableJsonStreamArchive.class);
/**
* exposing meta data (non-tile data) might be useful for most use cases but complicates parsing for simple use cases
* => allow to output tiles, only
*/
private static final String OPTION_WRITE_TILES_ONLY = "tiles_only";
private static final String OPTION_ROOT_VALUE_SEPARATOR = "root_value_separator";
static final JsonMapper jsonMapper = JsonMapper.builder()
.serializationInclusion(Include.NON_ABSENT)
.addModule(new Jdk8Module())
.build();
private static final JsonMapper jsonMapper = StreamArchiveUtils.jsonMapperJsonStreamArchive;
private final boolean writeTilesOnly;
private final String rootValueSeparator;
private WriteableJsonStreamArchive(Path p, StreamArchiveConfig config) {
super(p, config);
this.writeTilesOnly = config.moreOptions().getBoolean(OPTION_WRITE_TILES_ONLY, "write tiles, only", false);
this.rootValueSeparator = StreamArchiveUtils.getEscapedString(config.moreOptions(), TileArchiveConfig.Format.JSON,
OPTION_ROOT_VALUE_SEPARATOR, "root value separator", "'\\n'", List.of("\n", " "));
this.writeTilesOnly = StreamArchiveUtils.jsonOptionWriteTilesOnly(config.formatOptions());
this.rootValueSeparator = StreamArchiveUtils.jsonOptionRootValueSeparator(config.formatOptions());
}
public static WriteableJsonStreamArchive newWriteToFile(Path path, StreamArchiveConfig config) {
@ -70,7 +49,7 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
if (writeTilesOnly) {
return;
}
writeEntryFlush(new InitializationEntry());
writeEntryFlush(new JsonStreamArchiveEntry.InitializationEntry());
}
@Override
@ -78,13 +57,13 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
if (writeTilesOnly) {
return;
}
writeEntryFlush(new FinishEntry(metadata));
writeEntryFlush(new JsonStreamArchiveEntry.FinishEntry(metadata));
}
private void writeEntryFlush(Entry entry) {
private void writeEntryFlush(JsonStreamArchiveEntry entry) {
try (var out = new OutputStreamWriter(getPrimaryOutputStream(), StandardCharsets.UTF_8.newEncoder())) {
jsonMapper
.writerFor(Entry.class)
.writerFor(JsonStreamArchiveEntry.class)
.withoutFeatures(JsonGenerator.Feature.AUTO_CLOSE_TARGET)
.writeValue(out, entry);
out.write(rootValueSeparator);
@ -104,7 +83,8 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
this.rootValueSeparator = rootValueSeparator;
try {
this.jsonWriter =
jsonMapper.writerFor(Entry.class).withRootValueSeparator(rootValueSeparator).writeValues(outputStream);
jsonMapper.writerFor(JsonStreamArchiveEntry.class).withRootValueSeparator(rootValueSeparator)
.writeValues(outputStream);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@ -114,7 +94,8 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
public void write(TileEncodingResult encodingResult) {
final TileCoord coord = encodingResult.coord();
try {
jsonWriter.write(new TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData()));
jsonWriter
.write(new JsonStreamArchiveEntry.TileEntry(coord.x(), coord.y(), coord.z(), encodingResult.tileData()));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@ -150,53 +131,4 @@ public final class WriteableJsonStreamArchive extends WriteableStreamArchive {
}
}
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type")
@JsonSubTypes({
@Type(value = TileEntry.class, name = "tile"),
@Type(value = InitializationEntry.class, name = "initialization"),
@Type(value = FinishEntry.class, name = "finish")
})
sealed interface Entry {
}
record TileEntry(int x, int y, int z, byte[] encodedData) implements Entry {
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(encodedData);
result = prime * result + Objects.hash(x, y, z);
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!(obj instanceof TileEntry)) {
return false;
}
TileEntry other = (TileEntry) obj;
return Arrays.equals(encodedData, other.encodedData) && x == other.x && y == other.y && z == other.z;
}
@Override
public String toString() {
return "TileEntry [x=" + x + ", y=" + y + ", z=" + z + ", encodedData=" + Arrays.toString(encodedData) + "]";
}
}
record InitializationEntry() implements Entry {}
record FinishEntry(TileArchiveMetadata metadata) implements Entry {}
}

12
planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchive.java
Wyświetl plik

@ -158,15 +158,17 @@ public final class WriteableProtoStreamArchive extends WriteableStreamArchive {
@Override
public void write(TileEncodingResult encodingResult) {
final TileCoord coord = encodingResult.coord();
final StreamArchiveProto.TileEntry tile = StreamArchiveProto.TileEntry.newBuilder()
final byte[] data = encodingResult.tileData();
StreamArchiveProto.TileEntry.Builder tileBuilder = StreamArchiveProto.TileEntry.newBuilder()
.setZ(coord.z())
.setX(coord.x())
.setY(coord.y())
.setEncodedData(ByteString.copyFrom(encodingResult.tileData()))
.build();
.setY(coord.y());
if (data != null) {
tileBuilder = tileBuilder.setEncodedData(ByteString.copyFrom(encodingResult.tileData()));
}
final StreamArchiveProto.Entry entry = StreamArchiveProto.Entry.newBuilder()
.setTile(tile)
.setTile(tileBuilder.build())
.build();
try {

40
planetiler-core/src/main/java/com/onthegomap/planetiler/util/CloseableIterator.java
Wyświetl plik

@ -2,8 +2,10 @@ package com.onthegomap.planetiler.util;
import java.io.Closeable;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Spliterators;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@ -56,4 +58,42 @@ public interface CloseableIterator<T> extends Closeable, Iterator<T> {
}
};
}
default CloseableIterator<T> filter(Predicate<T> predicate) {
final var parent = this;
return new CloseableIterator<>() {
private T nextValue;
@Override
public void close() {
parent.close();
}
@Override
public boolean hasNext() {
if (nextValue != null) {
return true;
}
while (parent.hasNext()) {
final T parentNext = parent.next();
if (predicate.test(parentNext)) {
nextValue = parentNext;
break;
}
}
return nextValue != null;
}
@Override
public T next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
final T returnValue = nextValue;
nextValue = null;
return returnValue;
}
};
}
}

26
planetiler-core/src/main/java/com/onthegomap/planetiler/util/Gzip.java
Wyświetl plik

@ -3,22 +3,42 @@ package com.onthegomap.planetiler.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
public class Gzip {
public final class Gzip {
public static byte[] gzip(byte[] in) throws IOException {
private Gzip() {}
@SuppressWarnings("java:S1168") // null in, null out
public static byte[] gzip(byte[] in) {
if (in == null) {
return null;
}
var bos = new ByteArrayOutputStream(in.length);
try (var gzipOS = new GZIPOutputStream(bos)) {
gzipOS.write(in);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return bos.toByteArray();
}
public static byte[] gunzip(byte[] zipped) throws IOException {
@SuppressWarnings("java:S1168") // null in, null out
public static byte[] gunzip(byte[] zipped) {
if (zipped == null) {
return null;
}
try (var is = new GZIPInputStream(new ByteArrayInputStream(zipped))) {
return is.readAllBytes();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
public static boolean isZipped(byte[] in) {
return in != null && in.length > 2 && in[0] == (byte) GZIPInputStream.GZIP_MAGIC &&
in[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8);
}
}

3
planetiler-core/src/main/java/com/onthegomap/planetiler/util/Hashing.java
Wyświetl plik

@ -61,6 +61,9 @@ public final class Hashing {
*/
public static long fnv1a64(long initHash, byte... data) {
long hash = initHash;
if (data == null) {
return hash;
}
for (byte datum : data) {
hash ^= (datum & 0xff);
hash *= FNV1_PRIME_64;

16
planetiler-core/src/main/proto/stream_archive_proto.proto
Wyświetl plik

@ -21,7 +21,7 @@ message InitializationEntry {
}
message FinishEntry {
Metadata metadata = 1;
optional Metadata metadata = 1;
}
message Metadata {
@ -32,10 +32,10 @@ message Metadata {
string version = 4;
string type = 5;
string format = 6;
Envelope bounds = 7;
Coordinate center = 8;
int32 min_zoom = 9;
int32 max_zoom = 10;
optional Envelope bounds = 7;
optional Coordinate center = 8;
optional int32 min_zoom = 9;
optional int32 max_zoom = 10;
repeated VectorLayer vector_layers = 11;
map<string, string> others = 12;
TileCompression tile_compression = 13;
@ -51,15 +51,15 @@ message Envelope {
message Coordinate {
double x = 1;
double y = 2;
double z = 3;
optional double z = 3;
}
message VectorLayer {
string id = 1;
map<string, FieldType> fields = 2;
string description = 3;
int32 min_zoom = 4;
int32 max_zoom = 5;
optional int32 min_zoom = 4;
optional int32 max_zoom = 5;
enum FieldType {
FIELD_TYPE_UNSPECIFIED = 0;

14
planetiler-core/src/test/java/com/onthegomap/planetiler/PlanetilerTests.java
Wyświetl plik

@ -31,7 +31,10 @@ import com.onthegomap.planetiler.reader.osm.OsmElement;
import com.onthegomap.planetiler.reader.osm.OsmReader;
import com.onthegomap.planetiler.reader.osm.OsmRelationInfo;
import com.onthegomap.planetiler.stats.Stats;
import com.onthegomap.planetiler.stream.InMemoryStreamArchive;
import com.onthegomap.planetiler.stream.ReadableCsvArchive;
import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive;
import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive;
import com.onthegomap.planetiler.stream.StreamArchiveConfig;
import com.onthegomap.planetiler.util.BuildInfo;
import com.onthegomap.planetiler.util.Gzip;
import com.onthegomap.planetiler.util.TileSizeStats;
@ -2040,11 +2043,12 @@ class PlanetilerTests {
final ReadableTileArchiveFactory readableTileArchiveFactory = switch (format) {
case MBTILES -> Mbtiles::newReadOnlyDatabase;
case CSV -> p -> InMemoryStreamArchive.fromCsv(p, ",");
case TSV -> p -> InMemoryStreamArchive.fromCsv(p, "\t");
case JSON -> InMemoryStreamArchive::fromJson;
case CSV, TSV ->
p -> ReadableCsvArchive.newReader(format, p, new StreamArchiveConfig(false, Arguments.of()));
case JSON -> p -> ReadableJsonStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()));
case PMTILES -> ReadablePmtiles::newReadFromFile;
case PROTO, PBF -> InMemoryStreamArchive::fromProtobuf;
case PROTO, PBF ->
p -> ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()));
case FILES -> p -> ReadableFilesArchive.newReader(p, Arguments.of());
};

2
planetiler-core/src/test/java/com/onthegomap/planetiler/TestUtils.java
Wyświetl plik

@ -770,7 +770,7 @@ public class TestUtils {
if (!failures.isEmpty()) {
fail(String.join(System.lineSeparator(), failures));
}
} catch (GeometryException | IOException e) {
} catch (GeometryException e) {
fail(e);
}
}

200
planetiler-core/src/test/java/com/onthegomap/planetiler/archive/TileCopyTest.java 100644
Wyświetl plik

@ -0,0 +1,200 @@
package com.onthegomap.planetiler.archive;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.onthegomap.planetiler.TestUtils;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.util.Gzip;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Base64;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ArgumentsProvider;
import org.junit.jupiter.params.provider.ArgumentsSource;
class TileCopyTest {
private static final String ARCHIVE_0_JSON_BASE = """
{"type":"initialization"}
{"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
{"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
{"type":"finish","metadata":%s}
""";
private static final String ARCHIVE_0_CSV_COMPRESSION_NONE = """
0,0,0,AA==
1,2,3,AQ==
""";
private static final String EXTERNAL_METADATA = "{\"name\": \"blub\"}";
@ParameterizedTest(name = "{index} - {0}")
@ArgumentsSource(TestArgs.class)
void testSimple(String testName, String archiveDataIn, String archiveDataOut, Map<String, String> arguments,
@TempDir Path tempDir) throws Exception {
final Path archiveInPath = tempDir.resolve(archiveDataIn.contains("{") ? "in.json" : "in.csv");
final Path archiveOutPath = tempDir.resolve(archiveDataOut.contains("{") ? "out.json" : "out.csv");
final Path inMetadataPath = tempDir.resolve("metadata.json");
Files.writeString(archiveInPath, archiveDataIn);
Files.writeString(inMetadataPath, EXTERNAL_METADATA);
arguments = new LinkedHashMap<>(arguments);
arguments.replace("in_metadata", inMetadataPath.toString());
final Arguments args = Arguments.of(Map.of(
"input", archiveInPath.toString(),
"output", archiveOutPath.toString()
)).orElse(Arguments.of(arguments));
new TileCopy(TileCopy.TileCopyConfig.fromArguments(args)).run();
if (archiveDataOut.contains("{")) {
final List<String> expectedLines = Arrays.stream(archiveDataOut.split("\n")).toList();
final List<String> actualLines = Files.readAllLines(archiveOutPath);
assertEquals(expectedLines.size(), actualLines.size());
for (int i = 0; i < expectedLines.size(); i++) {
TestUtils.assertSameJson(expectedLines.get(i), actualLines.get(i));
}
} else {
assertEquals(archiveDataOut, Files.readString(archiveOutPath));
}
}
private static String compressBase64(String archiveIn) {
final Base64.Encoder encoder = Base64.getEncoder();
for (int i = 0; i <= 1; i++) {
archiveIn = archiveIn.replace(
encoder.encodeToString(new byte[]{(byte) i}),
encoder.encodeToString(Gzip.gzip(new byte[]{(byte) i}))
);
}
return archiveIn;
}
private static String replaceBase64(String archiveIn, String replacement) {
final Base64.Encoder encoder = Base64.getEncoder();
for (int i = 0; i <= 1; i++) {
archiveIn = archiveIn.replace(
encoder.encodeToString(new byte[]{(byte) i}),
replacement
);
}
return archiveIn;
}
private static class TestArgs implements ArgumentsProvider {
@Override
public Stream<? extends org.junit.jupiter.params.provider.Arguments> provideArguments(ExtensionContext context) {
return Stream.of(
argsOf(
"json(w/o meta, compression:none) to csv(compression:none)",
ARCHIVE_0_JSON_BASE.formatted("null"),
ARCHIVE_0_CSV_COMPRESSION_NONE,
Map.of("out_tile_compression", "none")
),
argsOf(
"json(w/o meta, compression:none) to csv(compression:gzip)",
ARCHIVE_0_JSON_BASE.formatted("null"),
compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE),
Map.of("out_tile_compression", "gzip")
),
argsOf(
"json(w/o meta, compression:gzip) to csv(compression:none)",
compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")),
ARCHIVE_0_CSV_COMPRESSION_NONE,
Map.of("out_tile_compression", "none")
),
argsOf(
"json(w/o meta, compression:gzip) to csv(compression:gzip)",
compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")),
compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE),
Map.of("out_tile_compression", "gzip")
),
argsOf(
"json(w/o meta, compression:gzip) to csv(compression:gzip)",
compressBase64(ARCHIVE_0_JSON_BASE.formatted("null")),
compressBase64(ARCHIVE_0_CSV_COMPRESSION_NONE),
Map.of("out_tile_compression", "gzip")
),
argsOf(
"json(w/ meta, compression:gzip) to csv(compression:none)",
compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)),
ARCHIVE_0_CSV_COMPRESSION_NONE,
Map.of("out_tile_compression", "none")
),
argsOf(
"json(w/ meta, compression:gzip) to json(w/ meta, compression:gzip)",
compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)),
compressBase64(ARCHIVE_0_JSON_BASE.formatted(TestUtils.MAX_METADATA_SERIALIZED)),
Map.of()
),
argsOf(
"csv to json - use fallback metadata",
ARCHIVE_0_CSV_COMPRESSION_NONE,
ARCHIVE_0_JSON_BASE.formatted(
"{\"name\":\"unknown\",\"format\":\"pbf\",\"json\":\"{\\\"vector_layers\\\":[]}\",\"compression\":\"none\"}"),
Map.of("out_tile_compression", "none")
),
argsOf(
"csv to json - use external metadata",
ARCHIVE_0_CSV_COMPRESSION_NONE,
ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"none\"}"),
Map.of("out_tile_compression", "none", "in_metadata", "blub")
),
argsOf(
"csv to json - null handling",
replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""),
replaceBase64(ARCHIVE_0_JSON_BASE.formatted("{\"name\":\"blub\",\"compression\":\"gzip\"}"), "null")
.replace(",\"encodedData\":\"null\"", ""),
Map.of("in_metadata", "blub")
),
argsOf(
"json to csv - null handling",
replaceBase64(ARCHIVE_0_JSON_BASE.formatted("null"), "null")
.replace(",\"encodedData\":\"null\"", ""),
replaceBase64(ARCHIVE_0_CSV_COMPRESSION_NONE, ""),
Map.of()
),
argsOf(
"csv to csv - empty skipping on",
"""
0,0,0,
1,2,3,AQ==
""",
"""
1,2,3,AQ==
""",
Map.of("skip_empty", "true", "out_tile_compression", "none")
),
argsOf(
"csv to csv - empty skipping off",
"""
0,0,0,
1,2,3,AQ==
""",
"""
0,0,0,
1,2,3,AQ==
""",
Map.of("skip_empty", "false", "out_tile_compression", "none")
)
);
}
private static org.junit.jupiter.params.provider.Arguments argsOf(String testName, String archiveDataIn,
String archiveDataOut, Map<String, String> arguments) {
return org.junit.jupiter.params.provider.Arguments.of(testName, archiveDataIn, archiveDataOut, arguments);
}
}
}

119
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/InMemoryStreamArchive.java
Wyświetl plik

@ -1,119 +0,0 @@
package com.onthegomap.planetiler.stream;
import com.onthegomap.planetiler.archive.ReadableTileArchive;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.archive.TileEncodingResult;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.proto.StreamArchiveProto;
import com.onthegomap.planetiler.util.CloseableIterator;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.OptionalLong;
public class InMemoryStreamArchive implements ReadableTileArchive {
private final List<TileEncodingResult> tileEncodings;
private final TileArchiveMetadata metadata;
private InMemoryStreamArchive(List<TileEncodingResult> tileEncodings, TileArchiveMetadata metadata) {
this.tileEncodings = tileEncodings;
this.metadata = metadata;
}
public static InMemoryStreamArchive fromCsv(Path p, String columnSepatator) throws IOException {
var base64Decoder = Base64.getDecoder();
final List<TileEncodingResult> tileEncodings = new ArrayList<>();
try (var reader = Files.newBufferedReader(p)) {
String line;
while ((line = reader.readLine()) != null) {
final String[] splits = line.split(columnSepatator);
final TileCoord tileCoord = TileCoord.ofXYZ(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]),
Integer.parseInt(splits[2]));
tileEncodings.add(new TileEncodingResult(tileCoord, base64Decoder.decode(splits[3]), OptionalLong.empty()));
}
}
return new InMemoryStreamArchive(tileEncodings, null);
}
public static InMemoryStreamArchive fromProtobuf(Path p) throws IOException {
final List<TileEncodingResult> tileEncodings = new ArrayList<>();
try (var in = Files.newInputStream(p)) {
StreamArchiveProto.Entry entry;
while ((entry = StreamArchiveProto.Entry.parseDelimitedFrom(in)) != null) {
if (entry.getEntryCase() == StreamArchiveProto.Entry.EntryCase.TILE) {
final StreamArchiveProto.TileEntry tileProto = entry.getTile();
final TileCoord tileCoord = TileCoord.ofXYZ(tileProto.getX(), tileProto.getY(), tileProto.getZ());
tileEncodings
.add(new TileEncodingResult(tileCoord, tileProto.getEncodedData().toByteArray(), OptionalLong.empty()));
}
}
}
return new InMemoryStreamArchive(tileEncodings, null /* could add once the format is finalized*/);
}
public static InMemoryStreamArchive fromJson(Path p) throws IOException {
final List<TileEncodingResult> tileEncodings = new ArrayList<>();
final TileArchiveMetadata[] metadata = new TileArchiveMetadata[]{null};
try (var reader = Files.newBufferedReader(p)) {
WriteableJsonStreamArchive.jsonMapper
.readerFor(WriteableJsonStreamArchive.Entry.class)
.readValues(reader)
.forEachRemaining(entry -> {
if (entry instanceof WriteableJsonStreamArchive.TileEntry te) {
final TileCoord tileCoord = TileCoord.ofXYZ(te.x(), te.y(), te.z());
tileEncodings.add(new TileEncodingResult(tileCoord, te.encodedData(), OptionalLong.empty()));
} else if (entry instanceof WriteableJsonStreamArchive.FinishEntry fe) {
metadata[0] = fe.metadata();
}
});
}
return new InMemoryStreamArchive(tileEncodings, Objects.requireNonNull(metadata[0]));
}
@Override
public void close() throws IOException {}
@Override
public byte[] getTile(int x, int y, int z) {
final TileCoord coord = TileCoord.ofXYZ(x, y, z);
return tileEncodings.stream()
.filter(ter -> ter.coord().equals(coord)).findFirst()
.map(TileEncodingResult::tileData)
.orElse(null);
}
@Override
public CloseableIterator<TileCoord> getAllTileCoords() {
final Iterator<TileEncodingResult> it = tileEncodings.iterator();
return new CloseableIterator<TileCoord>() {
@Override
public TileCoord next() {
return it.next().coord();
}
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
public void close() {}
};
}
@Override
public TileArchiveMetadata metadata() {
return metadata;
}
}

75
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableCsvStreamArchiveTest.java 100644
Wyświetl plik

@ -0,0 +1,75 @@
package com.onthegomap.planetiler.stream;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveConfig;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.geo.TileCoord;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.apache.commons.text.StringEscapeUtils;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
class ReadableCsvStreamArchiveTest {
@ParameterizedTest
@CsvSource(delimiter = '$', textBlock = """
,$\\n$false$BASE64
,$\\r$false$BASE64
,$\\r\\n$false$BASE64
,$x$false$BASE64
;$\\n$false$BASE64
{$\\n$false$BASE64
,${$false$BASE64
,$\\n$false$HEX
,$\\n$true$BASE64
""")
void testSimple(String columnSeparator, String lineSeparator, boolean pad, CsvBinaryEncoding encoding,
@TempDir Path tempDir) throws IOException {
final Path csvFile = tempDir.resolve("in.csv");
final String csv =
"""
0,0,0,AA==
1,2,3,AQ==
"""
.replace("\n", StringEscapeUtils.unescapeJava(lineSeparator))
.replace(",", columnSeparator + (pad ? " " : ""))
.replace("AA==", encoding == CsvBinaryEncoding.BASE64 ? "AA==" : "00")
.replace("AQ==", encoding == CsvBinaryEncoding.BASE64 ? "AQ==" : "01");
Files.writeString(csvFile, csv);
final StreamArchiveConfig config = new StreamArchiveConfig(
false,
Arguments.of(Map.of(
StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, columnSeparator,
StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, lineSeparator,
StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, encoding.id()
))
);
final List<Tile> expectedTiles = List.of(
new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1})
);
try (var reader = ReadableCsvArchive.newReader(TileArchiveConfig.Format.CSV, csvFile, config)) {
assertEquals(expectedTiles, reader.getAllTiles().stream().toList());
assertEquals(expectedTiles, reader.getAllTiles().stream().toList());
assertNull(reader.metadata());
assertNull(reader.metadata());
assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3)));
assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0));
assertNull(reader.getTile(4, 5, 6));
}
}
}

48
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableJsonStreamArchiveTest.java 100644
Wyświetl plik

@ -0,0 +1,48 @@
package com.onthegomap.planetiler.stream;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import com.onthegomap.planetiler.TestUtils;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.geo.TileCoord;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class ReadableJsonStreamArchiveTest {
@Test
void testSimple(@TempDir Path tempDir) throws IOException {
final Path jsonFile = tempDir.resolve("in.json");
final String json = """
{"type":"initialization"}
{"type":"tile","x":0,"y":0,"z":0,"encodedData":"AA=="}
{"type":"tile","x":1,"y":2,"z":3,"encodedData":"AQ=="}
{"type":"finish","metadata":%s}
""".formatted(TestUtils.MAX_METADATA_SERIALIZED);
Files.writeString(jsonFile, json);
final StreamArchiveConfig config = new StreamArchiveConfig(false, Arguments.of());
final List<Tile> expectedTiles = List.of(
new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1})
);
try (var reader = ReadableJsonStreamArchive.newReader(jsonFile, config)) {
assertEquals(expectedTiles, reader.getAllTiles().stream().toList());
assertEquals(expectedTiles, reader.getAllTiles().stream().toList());
assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata());
assertEquals(TestUtils.MAX_METADATA_DESERIALIZED, reader.metadata());
assertArrayEquals(expectedTiles.get(1).bytes(), reader.getTile(TileCoord.ofXYZ(1, 2, 3)));
assertArrayEquals(expectedTiles.get(0).bytes(), reader.getTile(0, 0, 0));
assertNull(reader.getTile(4, 5, 6));
}
}
}

60
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/ReadableProtoStreamArchiveTest.java 100644
Wyświetl plik

@ -0,0 +1,60 @@
package com.onthegomap.planetiler.stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.google.protobuf.ByteString;
import com.onthegomap.planetiler.archive.Tile;
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
import com.onthegomap.planetiler.config.Arguments;
import com.onthegomap.planetiler.geo.TileCoord;
import com.onthegomap.planetiler.proto.StreamArchiveProto;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
class ReadableProtoStreamArchiveTest {
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testSimple(boolean maxMetaData, @TempDir Path tempDir) throws IOException {
final StreamArchiveProto.Metadata metadataSerialized = maxMetaData ?
WriteableProtoStreamArchiveTest.maxMetadataSerialized : WriteableProtoStreamArchiveTest.minMetadataSerialized;
final TileArchiveMetadata metadataDeserialized = maxMetaData ?
WriteableProtoStreamArchiveTest.maxMetadataDeserialized : WriteableProtoStreamArchiveTest.minMetadataDeserialized;
final Path p = tempDir.resolve("out.proto");
try (var out = Files.newOutputStream(p)) {
StreamArchiveProto.Entry.newBuilder().setInitialization(
StreamArchiveProto.InitializationEntry.newBuilder()
).build().writeDelimitedTo(out);
StreamArchiveProto.Entry.newBuilder().setTile(
StreamArchiveProto.TileEntry.newBuilder()
.setX(0).setY(0).setZ(0).setEncodedData(ByteString.copyFrom(new byte[]{0}))
).build().writeDelimitedTo(out);
StreamArchiveProto.Entry.newBuilder().setTile(
StreamArchiveProto.TileEntry.newBuilder()
.setX(1).setY(2).setZ(3).setEncodedData(ByteString.copyFrom(new byte[]{1}))
).build().writeDelimitedTo(out);
StreamArchiveProto.Entry.newBuilder().setFinish(
StreamArchiveProto.FinishEntry.newBuilder()
.setMetadata(metadataSerialized)
).build().writeDelimitedTo(out);
}
final List<Tile> expectedTiles = List.of(
new Tile(TileCoord.ofXYZ(0, 0, 0), new byte[]{0}),
new Tile(TileCoord.ofXYZ(1, 2, 3), new byte[]{1})
);
try (var reader = ReadableProtoStreamArchive.newReader(p, new StreamArchiveConfig(false, Arguments.of()))) {
assertEquals(expectedTiles, reader.getAllTiles().stream().toList());
assertEquals(expectedTiles, reader.getAllTiles().stream().toList());
assertEquals(metadataDeserialized, reader.metadata());
}
}
}

6
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableCsvArchiveTest.java
Wyświetl plik

@ -113,7 +113,7 @@ class WriteableCsvArchiveTest {
void testColumnSeparator(@TempDir Path tempDir) throws IOException {
final StreamArchiveConfig config =
new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_COLUMN_SEPARATOR, "' '")));
new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_COLUMN_SEPARATOR, "' '")));
final String expectedCsv =
"""
@ -128,7 +128,7 @@ class WriteableCsvArchiveTest {
void testLineSeparator(@TempDir Path tempDir) throws IOException {
final StreamArchiveConfig config =
new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_LINE_SEPARTATOR, "'\\r'")));
new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_LINE_SEPARATOR, "'\\r'")));
final String expectedCsv =
"""
@ -143,7 +143,7 @@ class WriteableCsvArchiveTest {
void testHexEncoding(@TempDir Path tempDir) throws IOException {
final StreamArchiveConfig config =
new StreamArchiveConfig(false, Arguments.of(Map.of(WriteableCsvArchive.OPTION_BINARY_ENCODING, "hex")));
new StreamArchiveConfig(false, Arguments.of(Map.of(StreamArchiveUtils.CSV_OPTION_BINARY_ENCODING, "hex")));
final String expectedCsv =
"""

20
planetiler-core/src/test/java/com/onthegomap/planetiler/stream/WriteableProtoStreamArchiveTest.java
Wyświetl plik

@ -28,8 +28,8 @@ import org.locationtech.jts.geom.Envelope;
class WriteableProtoStreamArchiveTest {
private static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null);
private static final TileArchiveMetadata maxMetadataIn =
static final StreamArchiveConfig defaultConfig = new StreamArchiveConfig(false, null);
static final TileArchiveMetadata maxMetadataDeserialized =
new TileArchiveMetadata("name", "description", "attribution", "version", "type", "format", new Envelope(0, 1, 2, 3),
new Coordinate(1.3, 3.7, 1.0), 2, 3,
TileArchiveMetadata.TileArchiveMetadataJson.create(
@ -45,7 +45,7 @@ class WriteableProtoStreamArchiveTest {
),
Map.of("a", "b", "c", "d"),
TileCompression.GZIP);
private static final StreamArchiveProto.Metadata maxMetadataOut = StreamArchiveProto.Metadata.newBuilder()
static final StreamArchiveProto.Metadata maxMetadataSerialized = StreamArchiveProto.Metadata.newBuilder()
.setName("name").setDescription("description").setAttribution("attribution").setVersion("version")
.setType("type").setFormat("format")
.setBounds(StreamArchiveProto.Envelope.newBuilder().setMinX(0).setMaxX(1).setMinY(2).setMaxY(3).build())
@ -64,10 +64,10 @@ class WriteableProtoStreamArchiveTest {
.setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_GZIP)
.build();
private static final TileArchiveMetadata minMetadataIn =
new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, null,
static final TileArchiveMetadata minMetadataDeserialized =
new TileArchiveMetadata(null, null, null, null, null, null, null, null, null, null, null, Map.of(),
TileCompression.NONE);
private static final StreamArchiveProto.Metadata minMetadataOut = StreamArchiveProto.Metadata.newBuilder()
static final StreamArchiveProto.Metadata minMetadataSerialized = StreamArchiveProto.Metadata.newBuilder()
.setTileCompression(StreamArchiveProto.TileCompression.TILE_COMPRESSION_NONE)
.build();
@ -83,12 +83,12 @@ class WriteableProtoStreamArchiveTest {
tileWriter.write(tile0);
tileWriter.write(tile1);
}
archive.finish(minMetadataIn);
archive.finish(minMetadataDeserialized);
}
try (InputStream in = Files.newInputStream(csvFile)) {
assertEquals(
List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataOut)),
List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(minMetadataSerialized)),
readAllEntries(in)
);
}
@ -119,12 +119,12 @@ class WriteableProtoStreamArchiveTest {
try (var tileWriter = archive.newTileWriter()) {
tileWriter.write(tile4);
}
archive.finish(maxMetadataIn);
archive.finish(maxMetadataDeserialized);
}
try (InputStream in = Files.newInputStream(csvFilePrimary)) {
assertEquals(
List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataOut)),
List.of(wrapInit(), toEntry(tile0), toEntry(tile1), wrapFinish(maxMetadataSerialized)),
readAllEntries(in)
);
}

36
planetiler-core/src/test/java/com/onthegomap/planetiler/util/CloseableIteratorTest.java 100644
Wyświetl plik

@ -0,0 +1,36 @@
package com.onthegomap.planetiler.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.List;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
class CloseableIteratorTest {
@Test
void testFilter() {
assertEquals(
List.of(2, 4),
CloseableIterator.of(Stream.of(1, 2, 3, 4, 5, 6))
.filter(i -> i == 2 || i == 4)
.stream()
.toList()
);
assertEquals(
List.of(),
CloseableIterator.of(Stream.of(100, 99, 98))
.filter(i -> i == 2 || i == 4)
.stream()
.toList()
);
assertEquals(
List.of(),
CloseableIterator.of(Stream.<Integer>of())
.filter(i -> i == 2 || i == 4)
.stream()
.toList()
);
}
}

5
planetiler-dist/src/main/java/com/onthegomap/planetiler/Main.java
Wyświetl plik

@ -2,6 +2,7 @@ package com.onthegomap.planetiler;
import static java.util.Map.entry;
import com.onthegomap.planetiler.archive.TileCopy;
import com.onthegomap.planetiler.benchmarks.LongLongMapBench;
import com.onthegomap.planetiler.benchmarks.OpenMapTilesMapping;
import com.onthegomap.planetiler.custommap.ConfiguredMapMain;
@ -65,7 +66,9 @@ public class Main {
entry("verify-monaco", VerifyMonaco::main),
entry("stats", TileSizeStats::main),
entry("top-osm-tiles", TopOsmTiles::main),
entry("compare", CompareArchives::main)
entry("compare", CompareArchives::main),
entry("tile-copy", TileCopy::main)
);
private static EntryPoint bundledSchema(String path) {