kopia lustrzana https://github.com/onthegomap/planetiler
Convert get all tile coords to iterator (#463)
rodzic
6ed0e3cc86
commit
9a704e773e
|
@ -96,7 +96,8 @@ of the intermediate features using a worker thread per core:
|
|||
|
||||
## 3) Emit Vector Tiles
|
||||
|
||||
[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) is the main driver.
|
||||
[TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java) is the main
|
||||
driver.
|
||||
First, a single-threaded reader reads features from disk:
|
||||
|
||||
- [ExternalMergeSort](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/ExternalMergeSort.java) emits
|
||||
|
@ -104,7 +105,8 @@ First, a single-threaded reader reads features from disk:
|
|||
- [FeatureGroup](planetiler-core/src/main/java/com/onthegomap/planetiler/collection/FeatureGroup.java) collects
|
||||
consecutive features in the same tile into a `TileFeatures` instance, dropping features in the same group over the
|
||||
grouping limit to limit point label density
|
||||
- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/writer/TileArchiveWriter.java) groups tiles
|
||||
- Then [TileArchiveWriter](planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java)
|
||||
groups tiles
|
||||
into variable-sized batches for workers to process (complex tiles get their own batch to ensure workers stay busy
|
||||
while the writer thread waits for finished tiles in order)
|
||||
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
package com.onthegomap.planetiler.benchmarks;
|
||||
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive.TileWriter;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
import com.onthegomap.planetiler.writer.TileArchive.TileWriter;
|
||||
import com.onthegomap.planetiler.writer.TileEncodingResult;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
|
|
|
@ -1,5 +1,8 @@
|
|||
package com.onthegomap.planetiler;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.collection.LongLongMap;
|
||||
import com.onthegomap.planetiler.collection.LongLongMultimap;
|
||||
|
@ -26,9 +29,6 @@ import com.onthegomap.planetiler.util.ResourceUsage;
|
|||
import com.onthegomap.planetiler.util.Translations;
|
||||
import com.onthegomap.planetiler.util.Wikidata;
|
||||
import com.onthegomap.planetiler.worker.RunnableThatThrows;
|
||||
import com.onthegomap.planetiler.writer.TileArchive;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileSystem;
|
||||
import java.nio.file.Files;
|
||||
|
@ -658,7 +658,7 @@ public class Planetiler {
|
|||
bounds.addFallbackProvider(new OsmNodeBoundsProvider(osmInputFile, config, stats));
|
||||
}
|
||||
|
||||
try (TileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
|
||||
try (WriteableTileArchive archive = Mbtiles.newWriteToFileDatabase(output, config.compactDb())) {
|
||||
featureGroup =
|
||||
FeatureGroup.newDiskBackedFeatureGroup(archive.tileOrder(), featureDbPath, profile, config, stats);
|
||||
stats.monitorFile("nodes", nodeDbPath);
|
||||
|
|
|
@ -0,0 +1,44 @@
|
|||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.util.CloseableIterator;
|
||||
import java.io.Closeable;
|
||||
|
||||
/**
|
||||
* Read API for on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
|
||||
* format.
|
||||
* <p>
|
||||
* See {@link WriteableTileArchive} for the write API.
|
||||
*/
|
||||
public interface ReadableTileArchive extends Closeable {
|
||||
|
||||
/** Returns the raw tile data associated with the tile at {@code coord}. */
|
||||
default byte[] getTile(TileCoord coord) {
|
||||
return getTile(coord.x(), coord.y(), coord.z());
|
||||
}
|
||||
|
||||
/** Returns the raw tile data associated with the tile at coordinate {@code x, y, z}. */
|
||||
byte[] getTile(int x, int y, int z);
|
||||
|
||||
/**
|
||||
* Returns an iterator over the coordinates of tiles in this archive.
|
||||
* <p>
|
||||
* The order should respect {@link WriteableTileArchive#tileOrder()} of the corresponding writer.
|
||||
* <p>
|
||||
* Clients should be sure to close the iterator after iterating through it, for example:
|
||||
*
|
||||
* <pre>
|
||||
* {@code
|
||||
* try (var iter = archive.getAllTileCoords()) {
|
||||
* while (iter.hasNext()) {
|
||||
* var coord = iter.next();
|
||||
* ...
|
||||
* }
|
||||
* }
|
||||
* }
|
||||
* </pre>
|
||||
*/
|
||||
CloseableIterator<TileCoord> getAllTileCoords();
|
||||
|
||||
// TODO access archive metadata
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.planetiler.writer;
|
||||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.planetiler.writer;
|
||||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import static com.onthegomap.planetiler.util.Gzip.gzip;
|
||||
import static com.onthegomap.planetiler.worker.Worker.joinFutures;
|
||||
|
@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
/**
|
||||
* Final stage of the map generation process that encodes vector tiles using {@link VectorTile} and writes them to a
|
||||
* {@link TileArchive}.
|
||||
* {@link WriteableTileArchive}.
|
||||
*/
|
||||
public class TileArchiveWriter {
|
||||
|
||||
|
@ -49,7 +49,7 @@ public class TileArchiveWriter {
|
|||
private static final long MAX_TILES_PER_BATCH = 1_000;
|
||||
private final Counter.Readable featuresProcessed;
|
||||
private final Counter memoizedTiles;
|
||||
private final TileArchive archive;
|
||||
private final WriteableTileArchive archive;
|
||||
private final PlanetilerConfig config;
|
||||
private final Stats stats;
|
||||
private final LayerStats layerStats;
|
||||
|
@ -60,7 +60,7 @@ public class TileArchiveWriter {
|
|||
private final AtomicReference<TileCoord> lastTileWritten = new AtomicReference<>();
|
||||
private final TileArchiveMetadata tileArchiveMetadata;
|
||||
|
||||
private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, TileArchive archive,
|
||||
private TileArchiveWriter(Iterable<FeatureGroup.TileFeatures> inputTiles, WriteableTileArchive archive,
|
||||
PlanetilerConfig config,
|
||||
TileArchiveMetadata tileArchiveMetadata, Stats stats, LayerStats layerStats) {
|
||||
this.inputTiles = inputTiles;
|
||||
|
@ -88,7 +88,7 @@ public class TileArchiveWriter {
|
|||
}
|
||||
|
||||
/** Reads all {@code features}, encodes them in parallel, and writes to {@code output}. */
|
||||
public static void writeOutput(FeatureGroup features, TileArchive output, DiskBacked fileSize,
|
||||
public static void writeOutput(FeatureGroup features, WriteableTileArchive output, DiskBacked fileSize,
|
||||
TileArchiveMetadata tileArchiveMetadata, PlanetilerConfig config, Stats stats) {
|
||||
var timer = stats.startStage("archive");
|
||||
|
||||
|
@ -335,28 +335,30 @@ public class TileArchiveWriter {
|
|||
}
|
||||
|
||||
private void printTileStats() {
|
||||
Format format = Format.defaultInstance();
|
||||
LOGGER.debug("Tile stats:");
|
||||
long sumSize = 0;
|
||||
long sumCount = 0;
|
||||
long maxMax = 0;
|
||||
for (int z = config.minzoom(); z <= config.maxzoom(); z++) {
|
||||
long totalCount = tilesByZoom[z].get();
|
||||
long totalSize = totalTileSizesByZoom[z].get();
|
||||
sumSize += totalSize;
|
||||
sumCount += totalCount;
|
||||
long maxSize = maxTileSizesByZoom[z].get();
|
||||
maxMax = Math.max(maxMax, maxSize);
|
||||
LOGGER.debug("z{} avg:{} max:{}",
|
||||
z,
|
||||
format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false),
|
||||
format.storage(maxSize, false));
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
Format format = Format.defaultInstance();
|
||||
LOGGER.debug("Tile stats:");
|
||||
long sumSize = 0;
|
||||
long sumCount = 0;
|
||||
long maxMax = 0;
|
||||
for (int z = config.minzoom(); z <= config.maxzoom(); z++) {
|
||||
long totalCount = tilesByZoom[z].get();
|
||||
long totalSize = totalTileSizesByZoom[z].get();
|
||||
sumSize += totalSize;
|
||||
sumCount += totalCount;
|
||||
long maxSize = maxTileSizesByZoom[z].get();
|
||||
maxMax = Math.max(maxMax, maxSize);
|
||||
LOGGER.debug("z{} avg:{} max:{}",
|
||||
z,
|
||||
format.storage(totalCount == 0 ? 0 : (totalSize / totalCount), false),
|
||||
format.storage(maxSize, false));
|
||||
}
|
||||
LOGGER.debug("all avg:{} max:{}",
|
||||
format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false),
|
||||
format.storage(maxMax, false));
|
||||
LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get()));
|
||||
LOGGER.debug(" # tiles: {}", format.integer(this.tilesEmitted()));
|
||||
}
|
||||
LOGGER.debug("all avg:{} max:{}",
|
||||
format.storage(sumCount == 0 ? 0 : (sumSize / sumCount), false),
|
||||
format.storage(maxMax, false));
|
||||
LOGGER.debug(" # features: {}", format.integer(featuresProcessed.get()));
|
||||
LOGGER.debug(" # tiles: {}", format.integer(this.tilesEmitted()));
|
||||
}
|
||||
|
||||
private long tilesEmitted() {
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.planetiler.writer;
|
||||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import java.util.Arrays;
|
|
@ -1,4 +1,4 @@
|
|||
package com.onthegomap.planetiler.writer;
|
||||
package com.onthegomap.planetiler.archive;
|
||||
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
|
@ -7,16 +7,22 @@ import java.io.Closeable;
|
|||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
/**
|
||||
* A TileArchive is a on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
|
||||
* Write API for an on-disk representation of a tileset in a portable format. Example: MBTiles, a sqlite-based archive
|
||||
* format.
|
||||
* <p>
|
||||
* See {@link ReadableTileArchive} for the read API.
|
||||
*/
|
||||
@NotThreadSafe
|
||||
public interface TileArchive extends Closeable {
|
||||
public interface WriteableTileArchive extends Closeable {
|
||||
|
||||
interface TileWriter extends Closeable {
|
||||
|
||||
void write(TileEncodingResult encodingResult);
|
||||
|
||||
// TODO: exists for compatibility reasons
|
||||
void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult);
|
||||
default void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
|
||||
write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
|
||||
}
|
||||
|
||||
@Override
|
||||
void close();
|
||||
|
@ -47,4 +53,6 @@ public interface TileArchive extends Closeable {
|
|||
* disk.
|
||||
*/
|
||||
void finish(PlanetilerConfig config);
|
||||
|
||||
// TODO update archive metadata
|
||||
}
|
|
@ -1,13 +1,14 @@
|
|||
package com.onthegomap.planetiler.geo;
|
||||
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import java.util.function.IntFunction;
|
||||
import java.util.function.ToDoubleBiFunction;
|
||||
import java.util.function.ToIntFunction;
|
||||
|
||||
/**
|
||||
* Controls the sort order of {@link com.onthegomap.planetiler.collection.FeatureGroup}, which determines the ordering
|
||||
* of {@link com.onthegomap.planetiler.writer.TileEncodingResult}s when written to
|
||||
* {@link com.onthegomap.planetiler.writer.TileArchive.TileWriter}.
|
||||
* of {@link com.onthegomap.planetiler.archive.TileEncodingResult}s when written to
|
||||
* {@link WriteableTileArchive.TileWriter}.
|
||||
*/
|
||||
public enum TileOrder {
|
||||
TMS(TileCoord::encoded, TileCoord::decode, TileCoord::progressOnLevel),
|
||||
|
|
|
@ -7,15 +7,18 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
|||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.onthegomap.planetiler.archive.ReadableTileArchive;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
import com.onthegomap.planetiler.reader.FileFormatException;
|
||||
import com.onthegomap.planetiler.util.CloseableIterator;
|
||||
import com.onthegomap.planetiler.util.Format;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import com.onthegomap.planetiler.writer.TileArchive;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.writer.TileEncodingResult;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
|
@ -31,6 +34,7 @@ import java.util.Collection;
|
|||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.TreeMap;
|
||||
|
@ -48,7 +52,7 @@ import org.sqlite.SQLiteConfig;
|
|||
*
|
||||
* @see <a href="https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md">MBTiles Specification</a>
|
||||
*/
|
||||
public final class Mbtiles implements TileArchive {
|
||||
public final class Mbtiles implements WriteableTileArchive, ReadableTileArchive {
|
||||
|
||||
// https://www.sqlite.org/src/artifact?ci=trunk&filename=magic.txt
|
||||
private static final int MBTILES_APPLICATION_ID = 0x4d504258;
|
||||
|
@ -88,13 +92,8 @@ public final class Mbtiles implements TileArchive {
|
|||
}
|
||||
|
||||
private final Connection connection;
|
||||
private PreparedStatement getTileStatement = null;
|
||||
private final boolean compactDb;
|
||||
|
||||
@Override
|
||||
public TileOrder tileOrder() {
|
||||
return TileOrder.TMS;
|
||||
}
|
||||
private PreparedStatement getTileStatement = null;
|
||||
|
||||
private Mbtiles(Connection connection, boolean compactDb) {
|
||||
this.connection = connection;
|
||||
|
@ -152,6 +151,11 @@ public final class Mbtiles implements TileArchive {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileOrder tileOrder() {
|
||||
return TileOrder.TMS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(PlanetilerConfig config, TileArchiveMetadata tileArchiveMetadata, LayerStats layerStats) {
|
||||
if (config.skipIndexCreation()) {
|
||||
|
@ -324,7 +328,7 @@ public final class Mbtiles implements TileArchive {
|
|||
}
|
||||
|
||||
/** Returns a writer that queues up inserts into the tile database(s) into large batches before executing them. */
|
||||
public TileArchive.TileWriter newTileWriter() {
|
||||
public WriteableTileArchive.TileWriter newTileWriter() {
|
||||
if (compactDb) {
|
||||
return new BatchedCompactTileWriter();
|
||||
} else {
|
||||
|
@ -333,7 +337,7 @@ public final class Mbtiles implements TileArchive {
|
|||
}
|
||||
|
||||
// TODO: exists for compatibility purposes
|
||||
public TileArchive.TileWriter newBatchedTileWriter() {
|
||||
public WriteableTileArchive.TileWriter newBatchedTileWriter() {
|
||||
return newTileWriter();
|
||||
}
|
||||
|
||||
|
@ -356,10 +360,7 @@ public final class Mbtiles implements TileArchive {
|
|||
return getTileStatement;
|
||||
}
|
||||
|
||||
public byte[] getTile(TileCoord coord) {
|
||||
return getTile(coord.x(), coord.y(), coord.z());
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getTile(int x, int y, int z) {
|
||||
try {
|
||||
PreparedStatement stmt = getTileStatement();
|
||||
|
@ -374,22 +375,9 @@ public final class Mbtiles implements TileArchive {
|
|||
}
|
||||
}
|
||||
|
||||
public List<TileCoord> getAllTileCoords() {
|
||||
List<TileCoord> result = new ArrayList<>();
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
ResultSet rs = statement.executeQuery(
|
||||
"select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE)
|
||||
);
|
||||
while (rs.next()) {
|
||||
int z = rs.getInt(TILES_COL_Z);
|
||||
int rawy = rs.getInt(TILES_COL_Y);
|
||||
int x = rs.getInt(TILES_COL_X);
|
||||
result.add(TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z));
|
||||
}
|
||||
} catch (SQLException throwables) {
|
||||
throw new IllegalStateException("Could not get all tile coordinates", throwables);
|
||||
}
|
||||
return result;
|
||||
@Override
|
||||
public CloseableIterator<TileCoord> getAllTileCoords() {
|
||||
return new TileCoordIterator();
|
||||
}
|
||||
|
||||
public Connection connection() {
|
||||
|
@ -492,14 +480,70 @@ public final class Mbtiles implements TileArchive {
|
|||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof TileDataEntry)) {
|
||||
if (!(obj instanceof TileDataEntry other)) {
|
||||
return false;
|
||||
}
|
||||
TileDataEntry other = (TileDataEntry) obj;
|
||||
return Arrays.equals(tileData, other.tileData) && tileDataId == other.tileDataId;
|
||||
}
|
||||
}
|
||||
|
||||
/** Iterates through tile coordinates one at a time without materializing the entire list in memory. */
|
||||
private class TileCoordIterator implements CloseableIterator<TileCoord> {
|
||||
private final Statement statement;
|
||||
private final ResultSet rs;
|
||||
private boolean hasNext = false;
|
||||
|
||||
private TileCoordIterator() {
|
||||
try {
|
||||
this.statement = connection.createStatement();
|
||||
this.rs = statement.executeQuery(
|
||||
"select %s, %s, %s, %s from %s".formatted(TILES_COL_Z, TILES_COL_X, TILES_COL_Y, TILES_COL_DATA, TILES_TABLE)
|
||||
);
|
||||
hasNext = rs.next();
|
||||
} catch (SQLException e) {
|
||||
throw new FileFormatException("Could not read tile coordinates from mbtiles file", e);
|
||||
} finally {
|
||||
if (!hasNext) {
|
||||
close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
try {
|
||||
statement.close();
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("Could not close mbtiles file", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return hasNext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TileCoord next() {
|
||||
if (!hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
try {
|
||||
int z = rs.getInt(TILES_COL_Z);
|
||||
int rawy = rs.getInt(TILES_COL_Y);
|
||||
int x = rs.getInt(TILES_COL_X);
|
||||
var result = TileCoord.ofXYZ(x, (1 << z) - 1 - rawy, z);
|
||||
hasNext = rs.next();
|
||||
if (!hasNext) {
|
||||
close();
|
||||
}
|
||||
return result;
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException("Could not read mbtiles file", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private abstract class BatchedTableWriterBase<T> implements AutoCloseable {
|
||||
|
||||
private static final int MAX_PARAMETERS_IN_PREPARED_STATEMENT = 999;
|
||||
|
@ -667,12 +711,6 @@ public final class Mbtiles implements TileArchive {
|
|||
tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData()));
|
||||
}
|
||||
|
||||
// TODO: exists for compatibility purposes
|
||||
@Override
|
||||
public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
|
||||
tableWriter.write(new TileEntry(encodingResult.coord(), encodingResult.tileData()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
tableWriter.close();
|
||||
|
@ -714,12 +752,6 @@ public final class Mbtiles implements TileArchive {
|
|||
batchedTileShallowTableWriter.write(new TileShallowEntry(encodingResult.coord(), tileDataId));
|
||||
}
|
||||
|
||||
// TODO: exists for compatibility purposes
|
||||
@Override
|
||||
public void write(com.onthegomap.planetiler.mbtiles.TileEncodingResult encodingResult) {
|
||||
write(new TileEncodingResult(encodingResult.coord(), encodingResult.tileData(), encodingResult.tileDataHash()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
batchedTileShallowTableWriter.close();
|
||||
|
|
|
@ -59,16 +59,19 @@ public class Verify {
|
|||
public static int getNumFeatures(Mbtiles db, String layer, int zoom, Map<String, Object> attrs, Envelope envelope,
|
||||
Class<? extends Geometry> clazz) throws GeometryException {
|
||||
int num = 0;
|
||||
for (var tileCoord : db.getAllTileCoords()) {
|
||||
Envelope tileEnv = new Envelope();
|
||||
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY()));
|
||||
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY()));
|
||||
if (tileCoord.z() == zoom) {
|
||||
byte[] data = db.getTile(tileCoord);
|
||||
for (var feature : decode(data)) {
|
||||
if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) {
|
||||
Geometry geometry = feature.geometry().decode();
|
||||
num += getGeometryCounts(geometry, clazz);
|
||||
try (var tileCoords = db.getAllTileCoords()) {
|
||||
while (tileCoords.hasNext()) {
|
||||
var tileCoord = tileCoords.next();
|
||||
Envelope tileEnv = new Envelope();
|
||||
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMinX(), envelope.getMinY()));
|
||||
tileEnv.expandToInclude(tileCoord.lngLatToTileCoords(envelope.getMaxX(), envelope.getMaxY()));
|
||||
if (tileCoord.z() == zoom) {
|
||||
byte[] data = db.getTile(tileCoord);
|
||||
for (var feature : decode(data)) {
|
||||
if (layer.equals(feature.layer()) && feature.attrs().entrySet().containsAll(attrs.entrySet())) {
|
||||
Geometry geometry = feature.geometry().decode();
|
||||
num += getGeometryCounts(geometry, clazz);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -187,7 +190,7 @@ public class Verify {
|
|||
|
||||
private void checkBasicStructure() {
|
||||
check("contains name attribute", () -> mbtiles.metadata().getAll().containsKey("name"));
|
||||
check("contains at least one tile", () -> !mbtiles.getAllTileCoords().isEmpty());
|
||||
check("contains at least one tile", () -> mbtiles.getAllTileCoords().stream().findAny().isPresent());
|
||||
checkWithMessage("all tiles are valid", () -> {
|
||||
List<String> invalidTiles = mbtiles.getAllTileCoords().stream()
|
||||
.flatMap(coord -> checkValidity(coord, decode(mbtiles.getTile(coord))).stream())
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
package com.onthegomap.planetiler.util;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.Iterator;
|
||||
import java.util.Spliterators;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
public interface CloseableIterator<T> extends Closeable, Iterator<T> {
|
||||
|
||||
@Override
|
||||
void close();
|
||||
|
||||
default Stream<T> stream() {
|
||||
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(this, 0), false).onClose(this::close);
|
||||
}
|
||||
}
|
|
@ -1,9 +1,9 @@
|
|||
package com.onthegomap.planetiler.util;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.onthegomap.planetiler.archive.WriteableTileArchive;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
import com.onthegomap.planetiler.render.RenderedFeature;
|
||||
import com.onthegomap.planetiler.writer.TileArchive;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -18,8 +18,8 @@ import javax.annotation.concurrent.ThreadSafe;
|
|||
/**
|
||||
* Tracks the feature attributes and zoom range of each layer to populate the archive output metadata.
|
||||
* <p>
|
||||
* Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link TileArchive} formats. To
|
||||
* minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
|
||||
* Matches the MBTiles spec for {@code vector_layers}, but can be reused by other {@link WriteableTileArchive} formats.
|
||||
* To minimize overhead of stat collection, each updating thread should call {@link #handlerForThread()} first to get a
|
||||
* thread-local handler that can update stats without contention.
|
||||
* </p>
|
||||
*
|
||||
|
|
|
@ -3,6 +3,8 @@ package com.onthegomap.planetiler;
|
|||
import static com.onthegomap.planetiler.TestUtils.*;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.collection.LongLongMap;
|
||||
import com.onthegomap.planetiler.collection.LongLongMultimap;
|
||||
|
@ -23,8 +25,6 @@ import com.onthegomap.planetiler.reader.osm.OsmReader;
|
|||
import com.onthegomap.planetiler.reader.osm.OsmRelationInfo;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.util.BuildInfo;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
|
|
@ -10,6 +10,7 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest;
|
|||
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.VectorTile;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.geo.GeometryType;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.geo.TileOrder;
|
||||
|
@ -17,7 +18,6 @@ import com.onthegomap.planetiler.render.RenderedFeature;
|
|||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.util.CloseableConsumer;
|
||||
import com.onthegomap.planetiler.util.Gzip;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveWriter;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
|
@ -5,15 +5,14 @@ import static org.junit.jupiter.api.Assertions.*;
|
|||
|
||||
import com.google.common.math.IntMath;
|
||||
import com.onthegomap.planetiler.TestUtils;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.geo.GeoUtils;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.util.LayerStats;
|
||||
import com.onthegomap.planetiler.writer.TileEncodingResult;
|
||||
import java.io.IOException;
|
||||
import java.math.RoundingMode;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.OptionalLong;
|
||||
|
@ -69,7 +68,7 @@ class MbtilesTest {
|
|||
assertEquals(howMany, all.size());
|
||||
assertEquals(expected, all);
|
||||
assertEquals(expected.stream().map(Mbtiles.TileEntry::tile).collect(Collectors.toSet()),
|
||||
new HashSet<>(db.getAllTileCoords()));
|
||||
db.getAllTileCoords().stream().collect(Collectors.toSet()));
|
||||
for (var expectedEntry : expected) {
|
||||
var tile = expectedEntry.tile();
|
||||
byte[] data = db.getTile(tile.x(), tile.y(), tile.z());
|
||||
|
|
|
@ -7,8 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.onthegomap.planetiler.VectorTile;
|
||||
import com.onthegomap.planetiler.archive.TileEncodingResult;
|
||||
import com.onthegomap.planetiler.geo.TileCoord;
|
||||
import com.onthegomap.planetiler.writer.TileEncodingResult;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
|
@ -2,6 +2,8 @@ package com.onthegomap.planetiler.examples;
|
|||
|
||||
import com.onthegomap.planetiler.Planetiler;
|
||||
import com.onthegomap.planetiler.Profile;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.archive.TileArchiveWriter;
|
||||
import com.onthegomap.planetiler.collection.FeatureGroup;
|
||||
import com.onthegomap.planetiler.collection.LongLongMap;
|
||||
import com.onthegomap.planetiler.collection.LongLongMultimap;
|
||||
|
@ -13,8 +15,6 @@ import com.onthegomap.planetiler.reader.osm.OsmInputFile;
|
|||
import com.onthegomap.planetiler.reader.osm.OsmReader;
|
||||
import com.onthegomap.planetiler.stats.Stats;
|
||||
import com.onthegomap.planetiler.util.FileUtils;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveMetadata;
|
||||
import com.onthegomap.planetiler.writer.TileArchiveWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
|
Ładowanie…
Reference in New Issue