2021-12-23 10:42:24 +00:00
|
|
|
package com.onthegomap.planetiler.collection;
|
2021-04-30 10:31:56 +00:00
|
|
|
|
|
|
|
import com.carrotsearch.hppc.LongLongHashMap;
|
2021-12-23 10:42:24 +00:00
|
|
|
import com.onthegomap.planetiler.Profile;
|
|
|
|
import com.onthegomap.planetiler.VectorTile;
|
|
|
|
import com.onthegomap.planetiler.config.PlanetilerConfig;
|
|
|
|
import com.onthegomap.planetiler.geo.GeometryException;
|
|
|
|
import com.onthegomap.planetiler.geo.GeometryType;
|
|
|
|
import com.onthegomap.planetiler.geo.TileCoord;
|
2023-01-27 02:43:07 +00:00
|
|
|
import com.onthegomap.planetiler.geo.TileOrder;
|
2021-12-23 10:42:24 +00:00
|
|
|
import com.onthegomap.planetiler.render.RenderedFeature;
|
|
|
|
import com.onthegomap.planetiler.stats.Stats;
|
2023-01-23 11:06:57 +00:00
|
|
|
import com.onthegomap.planetiler.util.CloseableConsumer;
|
2021-12-23 10:42:24 +00:00
|
|
|
import com.onthegomap.planetiler.util.CommonStringEncoder;
|
|
|
|
import com.onthegomap.planetiler.util.DiskBacked;
|
2023-09-22 01:44:09 +00:00
|
|
|
import com.onthegomap.planetiler.util.LayerAttrStats;
|
2022-05-24 22:46:52 +00:00
|
|
|
import com.onthegomap.planetiler.worker.Worker;
|
2022-04-26 10:26:05 +00:00
|
|
|
import java.io.Closeable;
|
2021-04-30 10:31:56 +00:00
|
|
|
import java.io.IOException;
|
2021-09-10 00:46:20 +00:00
|
|
|
import java.nio.file.Path;
|
2021-04-30 10:31:56 +00:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Arrays;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.Iterator;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Objects;
|
2021-06-07 11:46:03 +00:00
|
|
|
import java.util.concurrent.atomic.AtomicLong;
|
2021-05-13 10:25:06 +00:00
|
|
|
import java.util.function.Function;
|
2021-09-10 00:46:20 +00:00
|
|
|
import javax.annotation.concurrent.NotThreadSafe;
|
2021-04-30 10:31:56 +00:00
|
|
|
import org.msgpack.core.MessageBufferPacker;
|
|
|
|
import org.msgpack.core.MessagePack;
|
|
|
|
import org.msgpack.core.MessageUnpacker;
|
|
|
|
import org.msgpack.value.Value;
|
|
|
|
import org.msgpack.value.ValueFactory;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
|
|
|
* A utility that accepts rendered map features in any order and groups them by tile for a reader to iterate through.
|
|
|
|
* <p>
|
|
|
|
* Only support single-threaded writes and reads.
|
|
|
|
* <p>
|
|
|
|
* Limitation: layer name and attribute key strings get compressed into a single byte, so only 250 unique values are
|
|
|
|
* supported (see {@link CommonStringEncoder})
|
|
|
|
*/
|
|
|
|
@NotThreadSafe
|
2022-05-06 02:02:18 +00:00
|
|
|
public final class FeatureGroup implements Iterable<FeatureGroup.TileFeatures>, DiskBacked {
|
2021-04-30 10:31:56 +00:00
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
public static final int SORT_KEY_BITS = 23;
|
|
|
|
public static final int SORT_KEY_MAX = (1 << (SORT_KEY_BITS - 1)) - 1;
|
|
|
|
public static final int SORT_KEY_MIN = -(1 << (SORT_KEY_BITS - 1));
|
|
|
|
private static final int SORT_KEY_MASK = (1 << SORT_KEY_BITS) - 1;
|
2021-04-30 10:31:56 +00:00
|
|
|
private static final Logger LOGGER = LoggerFactory.getLogger(FeatureGroup.class);
|
2021-05-13 10:25:06 +00:00
|
|
|
private final FeatureSort sorter;
|
|
|
|
private final Profile profile;
|
2022-06-20 09:37:40 +00:00
|
|
|
private final CommonStringEncoder.AsByte commonLayerStrings = new CommonStringEncoder.AsByte();
|
|
|
|
private final CommonStringEncoder commonValueStrings = new CommonStringEncoder(100_000);
|
2021-05-13 10:25:06 +00:00
|
|
|
private final Stats stats;
|
2023-09-24 12:10:47 +00:00
|
|
|
private final PlanetilerConfig config;
|
2022-05-24 22:46:52 +00:00
|
|
|
private volatile boolean prepared = false;
|
2023-01-27 02:43:07 +00:00
|
|
|
private final TileOrder tileOrder;
|
2021-04-30 10:31:56 +00:00
|
|
|
|
2023-01-27 02:43:07 +00:00
|
|
|
|
2023-09-24 12:10:47 +00:00
|
|
|
FeatureGroup(FeatureSort sorter, TileOrder tileOrder, Profile profile, PlanetilerConfig config, Stats stats) {
|
2021-05-13 10:25:06 +00:00
|
|
|
this.sorter = sorter;
|
2023-01-27 02:43:07 +00:00
|
|
|
this.tileOrder = tileOrder;
|
2021-05-13 10:25:06 +00:00
|
|
|
this.profile = profile;
|
2023-09-24 12:10:47 +00:00
|
|
|
this.config = config;
|
2021-05-13 10:25:06 +00:00
|
|
|
this.stats = stats;
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/** Returns a feature grouper that stores all feature in-memory. Only suitable for toy use-cases like unit tests. */
|
2023-09-24 12:10:47 +00:00
|
|
|
public static FeatureGroup newInMemoryFeatureGroup(TileOrder tileOrder, Profile profile, PlanetilerConfig config,
|
|
|
|
Stats stats) {
|
|
|
|
return new FeatureGroup(FeatureSort.newInMemory(), tileOrder, profile, config, stats);
|
2021-05-13 10:25:06 +00:00
|
|
|
}
|
|
|
|
|
2023-09-24 12:10:47 +00:00
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
|
|
|
* Returns a feature grouper that writes all elements to disk in chunks, sorts each chunk, then reads back in order
|
|
|
|
* from those chunks. Suitable for making maps up to planet-scale.
|
|
|
|
*/
|
2023-01-27 02:43:07 +00:00
|
|
|
public static FeatureGroup newDiskBackedFeatureGroup(TileOrder tileOrder, Path tempDir, Profile profile,
|
2023-09-24 12:10:47 +00:00
|
|
|
PlanetilerConfig config, Stats stats) {
|
2021-09-10 00:46:20 +00:00
|
|
|
return new FeatureGroup(
|
|
|
|
new ExternalMergeSort(tempDir, config, stats),
|
2023-09-24 12:10:47 +00:00
|
|
|
tileOrder, profile, config, stats
|
2021-09-10 00:46:20 +00:00
|
|
|
);
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
2021-09-18 00:18:06 +00:00
|
|
|
* Encode key by {@code tile} asc, {@code layer} asc, {@code sortKey} asc with an extra bit to indicate whether the
|
|
|
|
* value contains grouping information.
|
2021-09-10 00:46:20 +00:00
|
|
|
*/
|
2021-09-18 00:18:06 +00:00
|
|
|
static long encodeKey(int tile, byte layer, int sortKey, boolean hasGroup) {
|
2022-03-09 02:08:03 +00:00
|
|
|
return ((long) tile << 32L) | ((long) (layer & 0xff) << 24L) | (((sortKey - SORT_KEY_MIN) & SORT_KEY_MASK) << 1L) |
|
|
|
|
(hasGroup ? 1 : 0);
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
static boolean extractHasGroupFromKey(long key) {
|
|
|
|
return (key & 1) == 1;
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
static int extractTileFromKey(long key) {
|
|
|
|
return (int) (key >> 32L);
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
static byte extractLayerIdFromKey(long key) {
|
|
|
|
return (byte) (key >> 24);
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
static int extractSortKeyFromKey(long key) {
|
|
|
|
return ((int) ((key >> 1) & SORT_KEY_MASK)) + SORT_KEY_MIN;
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
private static RenderedFeature.Group peekAtGroupInfo(byte[] encoded) {
|
2021-04-30 10:31:56 +00:00
|
|
|
try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) {
|
|
|
|
long group = unpacker.unpackLong();
|
|
|
|
int limit = unpacker.unpackInt();
|
|
|
|
return new RenderedFeature.Group(group, limit);
|
|
|
|
} catch (IOException e) {
|
|
|
|
throw new IllegalStateException(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-24 22:46:52 +00:00
|
|
|
static GeometryType decodeGeomType(byte geomTypeAndScale) {
|
|
|
|
return GeometryType.valueOf((byte) (geomTypeAndScale & 0b111));
|
|
|
|
}
|
|
|
|
|
|
|
|
static int decodeScale(byte geomTypeAndScale) {
|
|
|
|
return (geomTypeAndScale & 0xff) >>> 3;
|
|
|
|
}
|
|
|
|
|
|
|
|
static byte encodeGeomTypeAndScale(VectorTile.VectorGeometry geometry) {
|
|
|
|
assert geometry.geomType().asByte() >= 0 && geometry.geomType().asByte() <= 8;
|
|
|
|
assert geometry.scale() >= 0 && geometry.scale() < (1 << 5);
|
|
|
|
return (byte) ((geometry.geomType().asByte() & 0xff) | (geometry.scale() << 3));
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
public long numFeaturesWritten() {
|
|
|
|
return sorter.numFeaturesWritten();
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Returns a function for a single thread to use to serialize rendered features. */
|
2022-04-26 10:26:05 +00:00
|
|
|
public RenderedFeatureEncoder newRenderedFeatureEncoder() {
|
|
|
|
return new RenderedFeatureEncoder() {
|
|
|
|
// This method gets called billions of times when generating the planet, so these optimizations make a big difference:
|
|
|
|
// 1) Re-use the same buffer packer to avoid allocating and resizing new byte arrays for every feature.
|
|
|
|
private final MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
|
2023-12-14 11:53:21 +00:00
|
|
|
// 2) Avoid re-encoding values for identical filled geometries (i.e. ocean) by memoizing the encoded values
|
2021-09-10 00:46:20 +00:00
|
|
|
// FeatureRenderer ensures that a separate VectorTileEncoder.Feature is used for each zoom level
|
|
|
|
private VectorTile.Feature lastFeature = null;
|
2021-05-19 10:44:28 +00:00
|
|
|
private byte[] lastEncodedValue = null;
|
|
|
|
|
|
|
|
@Override
|
2021-09-10 00:46:20 +00:00
|
|
|
public SortableFeature apply(RenderedFeature feature) {
|
|
|
|
var group = feature.group().orElse(null);
|
2021-05-19 10:44:28 +00:00
|
|
|
var thisFeature = feature.vectorTileFeature();
|
|
|
|
byte[] encodedValue;
|
2021-09-10 00:46:20 +00:00
|
|
|
if (group != null) { // don't bother memoizing if group is present
|
2021-05-19 10:44:28 +00:00
|
|
|
encodedValue = encodeValue(thisFeature, group, packer);
|
|
|
|
} else if (lastFeature == thisFeature) {
|
|
|
|
encodedValue = lastEncodedValue;
|
|
|
|
} else { // feature changed, memoize new value
|
|
|
|
lastFeature = thisFeature;
|
2021-09-10 00:46:20 +00:00
|
|
|
lastEncodedValue = encodedValue = encodeValue(feature.vectorTileFeature(), null, packer);
|
2021-05-19 10:44:28 +00:00
|
|
|
}
|
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
return new SortableFeature(encodeKey(feature), encodedValue);
|
2021-05-19 10:44:28 +00:00
|
|
|
}
|
2022-04-26 10:26:05 +00:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public void close() throws IOException {
|
|
|
|
packer.close();
|
|
|
|
}
|
2021-05-19 10:44:28 +00:00
|
|
|
};
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
private long encodeKey(RenderedFeature feature) {
|
2021-04-30 10:31:56 +00:00
|
|
|
var vectorTileFeature = feature.vectorTileFeature();
|
2022-06-20 09:37:40 +00:00
|
|
|
byte encodedLayer = commonLayerStrings.encode(vectorTileFeature.layer());
|
2023-01-27 02:43:07 +00:00
|
|
|
|
2021-09-18 00:18:06 +00:00
|
|
|
return encodeKey(
|
2023-01-27 02:43:07 +00:00
|
|
|
this.tileOrder.encode(feature.tile()),
|
2021-09-10 00:46:20 +00:00
|
|
|
encodedLayer,
|
2021-09-18 00:18:06 +00:00
|
|
|
feature.sortKey(),
|
2021-04-30 10:31:56 +00:00
|
|
|
feature.group().isPresent()
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
private byte[] encodeValue(VectorTile.Feature vectorTileFeature, RenderedFeature.Group group,
|
2021-05-19 10:44:28 +00:00
|
|
|
MessageBufferPacker packer) {
|
2021-04-30 10:31:56 +00:00
|
|
|
packer.clear();
|
|
|
|
try {
|
2021-09-10 00:46:20 +00:00
|
|
|
// hasGroup bit in key will tell consumers whether they need to decode group info from value
|
|
|
|
if (group != null) {
|
|
|
|
packer.packLong(group.group());
|
|
|
|
packer.packInt(group.limit());
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
packer.packLong(vectorTileFeature.id());
|
2021-10-20 01:57:47 +00:00
|
|
|
packer.packByte(encodeGeomTypeAndScale(vectorTileFeature.geometry()));
|
2021-04-30 10:31:56 +00:00
|
|
|
var attrs = vectorTileFeature.attrs();
|
|
|
|
packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count());
|
|
|
|
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
|
2023-10-28 00:29:26 +00:00
|
|
|
Object value = entry.getValue();
|
|
|
|
if (value != null) {
|
2022-06-20 09:37:40 +00:00
|
|
|
packer.packInt(commonValueStrings.encode(entry.getKey()));
|
2023-10-28 00:29:26 +00:00
|
|
|
packer.packValue(switch (value) {
|
|
|
|
case String string -> ValueFactory.newString(string);
|
|
|
|
case Integer integer -> ValueFactory.newInteger(integer.longValue());
|
|
|
|
case Long longValue -> ValueFactory.newInteger(longValue);
|
|
|
|
case Float floatValue -> ValueFactory.newFloat(floatValue);
|
|
|
|
case Double doubleValue -> ValueFactory.newFloat(doubleValue);
|
|
|
|
case Boolean booleanValue -> ValueFactory.newBoolean(booleanValue);
|
|
|
|
case Object other -> ValueFactory.newString(other.toString());
|
|
|
|
});
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
// Use the same binary format for encoding geometries in output vector tiles. Benchmarking showed
|
|
|
|
// it was faster and smaller for encoding/decoding intermediate geometries than alternatives like WKB.
|
2021-04-30 10:31:56 +00:00
|
|
|
int[] commands = vectorTileFeature.geometry().commands();
|
|
|
|
packer.packArrayHeader(commands.length);
|
|
|
|
for (int command : commands) {
|
|
|
|
packer.packInt(command);
|
|
|
|
}
|
|
|
|
packer.close();
|
|
|
|
} catch (IOException e) {
|
|
|
|
throw new IllegalStateException(e);
|
|
|
|
}
|
|
|
|
return packer.toByteArray();
|
|
|
|
}
|
|
|
|
|
2022-05-06 02:02:18 +00:00
|
|
|
/** Returns a new feature writer that can be used for a single thread. */
|
2023-01-23 11:06:57 +00:00
|
|
|
public CloseableConsumer<SortableFeature> writerForThread() {
|
2022-05-06 02:02:18 +00:00
|
|
|
return sorter.writerForThread();
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public Iterator<TileFeatures> iterator() {
|
2021-09-10 00:46:20 +00:00
|
|
|
prepare();
|
2022-05-24 22:46:52 +00:00
|
|
|
return groupIntoTiles(sorter.iterator());
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Reads temp features using {@code threads} parallel threads and merges into a sorted list.
|
|
|
|
*
|
|
|
|
* @param threads The number of parallel read threads to spawn
|
|
|
|
* @return a {@link Reader} with a handle to the new read threads that were spawned, and in {@link Iterable} that can
|
|
|
|
* be used to iterate over the results.
|
|
|
|
*/
|
|
|
|
public Reader parallelIterator(int threads) {
|
|
|
|
prepare();
|
|
|
|
var parIter = sorter.parallelIterator(stats, threads);
|
|
|
|
return new Reader(parIter.reader(), () -> groupIntoTiles(parIter.iterator()));
|
|
|
|
}
|
|
|
|
|
|
|
|
private Iterator<TileFeatures> groupIntoTiles(Iterator<SortableFeature> entries) {
|
|
|
|
// entries are sorted by tile ID, so group consecutive entries in same tile into tiles
|
2021-04-30 10:31:56 +00:00
|
|
|
if (!entries.hasNext()) {
|
|
|
|
return Collections.emptyIterator();
|
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Features from sorter are ordered by tile, so iterate through features as long as
|
|
|
|
* they are in the same tile and return that group.
|
|
|
|
*/
|
|
|
|
SortableFeature firstFeature = entries.next();
|
2021-04-30 10:31:56 +00:00
|
|
|
return new Iterator<>() {
|
2021-09-10 00:46:20 +00:00
|
|
|
private SortableFeature lastFeature = firstFeature;
|
2021-09-18 00:18:06 +00:00
|
|
|
private int lastTileId = extractTileFromKey(firstFeature.key());
|
2021-04-30 10:31:56 +00:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public boolean hasNext() {
|
|
|
|
return lastFeature != null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public TileFeatures next() {
|
|
|
|
TileFeatures result = new TileFeatures(lastTileId);
|
2021-09-10 00:46:20 +00:00
|
|
|
result.add(lastFeature);
|
2021-04-30 10:31:56 +00:00
|
|
|
int lastTile = lastTileId;
|
|
|
|
|
|
|
|
while (entries.hasNext()) {
|
2021-09-10 00:46:20 +00:00
|
|
|
SortableFeature next = entries.next();
|
2021-04-30 10:31:56 +00:00
|
|
|
lastFeature = next;
|
2021-09-18 00:18:06 +00:00
|
|
|
lastTileId = extractTileFromKey(lastFeature.key());
|
2021-04-30 10:31:56 +00:00
|
|
|
if (lastTile != lastTileId) {
|
|
|
|
return result;
|
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
result.add(next);
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
lastFeature = null;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2021-08-22 09:37:57 +00:00
|
|
|
@Override
|
2021-09-10 00:46:20 +00:00
|
|
|
public long diskUsageBytes() {
|
|
|
|
return sorter.diskUsageBytes();
|
2021-05-13 10:25:06 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/** Sorts features to prepare for grouping after all features have been written. */
|
|
|
|
public void prepare() {
|
|
|
|
if (!prepared) {
|
|
|
|
synchronized (this) {
|
|
|
|
if (!prepared) {
|
|
|
|
sorter.sort();
|
|
|
|
prepared = true;
|
|
|
|
}
|
|
|
|
}
|
2021-05-13 10:25:06 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-24 22:46:52 +00:00
|
|
|
public interface RenderedFeatureEncoder extends Function<RenderedFeature, SortableFeature>, Closeable {}
|
|
|
|
|
|
|
|
public record Reader(Worker readWorker, Iterable<TileFeatures> result) {}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/** Features contained in a single tile. */
|
|
|
|
public class TileFeatures {
|
2021-05-13 10:25:06 +00:00
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
private final TileCoord tileCoord;
|
|
|
|
private final List<SortableFeature> entries = new ArrayList<>();
|
|
|
|
private final AtomicLong numFeaturesProcessed = new AtomicLong(0);
|
2021-04-30 10:31:56 +00:00
|
|
|
private LongLongHashMap counts = null;
|
2022-04-23 09:58:49 +00:00
|
|
|
private byte lastLayer = Byte.MAX_VALUE;
|
2021-04-30 10:31:56 +00:00
|
|
|
|
2023-01-27 02:43:07 +00:00
|
|
|
private TileFeatures(int lastTileId) {
|
|
|
|
this.tileCoord = tileOrder.decode(lastTileId);
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2023-09-24 12:10:47 +00:00
|
|
|
private static void unscaleAndRemovePointsOutsideBuffer(List<VectorTile.Feature> features, double maxPointBuffer) {
|
|
|
|
boolean checkPoints = maxPointBuffer <= 256 && maxPointBuffer >= -128;
|
2022-05-24 22:46:52 +00:00
|
|
|
for (int i = 0; i < features.size(); i++) {
|
|
|
|
var feature = features.get(i);
|
|
|
|
if (feature != null) {
|
|
|
|
VectorTile.VectorGeometry geometry = feature.geometry();
|
2023-09-24 12:10:47 +00:00
|
|
|
var orig = geometry;
|
2022-05-24 22:46:52 +00:00
|
|
|
if (geometry.scale() != 0) {
|
2023-09-24 12:10:47 +00:00
|
|
|
geometry = geometry.unscale();
|
|
|
|
}
|
|
|
|
if (checkPoints && geometry.geomType() == GeometryType.POINT && !geometry.isEmpty()) {
|
|
|
|
geometry = geometry.filterPointsOutsideBuffer(maxPointBuffer);
|
|
|
|
}
|
|
|
|
if (geometry.isEmpty()) {
|
|
|
|
features.set(i, null);
|
|
|
|
} else if (geometry != orig) {
|
|
|
|
features.set(i, feature.copyWithNewGeometry(geometry));
|
2022-05-24 22:46:52 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/** Returns the number of features read including features discarded from being over the limit in a group. */
|
2021-07-29 11:07:58 +00:00
|
|
|
public long getNumFeaturesProcessed() {
|
2021-06-07 11:46:03 +00:00
|
|
|
return numFeaturesProcessed.get();
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/** Returns the number of features to output, excluding features discarded from being over the limit in a group. */
|
2021-07-29 11:07:58 +00:00
|
|
|
public long getNumFeaturesToEmit() {
|
|
|
|
return entries.size();
|
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
public TileCoord tileCoord() {
|
|
|
|
return tileCoord;
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
/**
|
|
|
|
* Returns true if {@code other} contains features with identical layer, geometry, and attributes, as this tile -
|
|
|
|
* even if the tiles have separate coordinates.
|
|
|
|
* <p>
|
|
|
|
* Used as an optimization to avoid re-encoding the same ocean tiles over and over again.
|
|
|
|
*/
|
2021-04-30 10:31:56 +00:00
|
|
|
public boolean hasSameContents(TileFeatures other) {
|
|
|
|
if (other == null || other.entries.size() != entries.size()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
for (int i = 0; i < entries.size(); i++) {
|
2021-09-10 00:46:20 +00:00
|
|
|
SortableFeature a = entries.get(i);
|
|
|
|
SortableFeature b = other.entries.get(i);
|
2021-09-18 00:18:06 +00:00
|
|
|
long layerA = extractLayerIdFromKey(a.key());
|
|
|
|
long layerB = extractLayerIdFromKey(b.key());
|
2021-04-30 10:54:27 +00:00
|
|
|
if (layerA != layerB || !Arrays.equals(a.value(), b.value())) {
|
2021-04-30 10:31:56 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2022-05-24 21:46:56 +00:00
|
|
|
|
2022-04-23 09:58:49 +00:00
|
|
|
private VectorTile.Feature decodeVectorTileFeature(SortableFeature entry) {
|
|
|
|
try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(entry.value())) {
|
|
|
|
long group;
|
|
|
|
if (extractHasGroupFromKey(entry.key())) {
|
|
|
|
group = unpacker.unpackLong();
|
|
|
|
unpacker.unpackInt(); // groupLimit - features over the limit were already discarded
|
|
|
|
} else {
|
|
|
|
group = VectorTile.Feature.NO_GROUP;
|
|
|
|
}
|
|
|
|
long id = unpacker.unpackLong();
|
|
|
|
byte geomTypeAndScale = unpacker.unpackByte();
|
|
|
|
GeometryType geomType = decodeGeomType(geomTypeAndScale);
|
|
|
|
int scale = decodeScale(geomTypeAndScale);
|
|
|
|
int mapSize = unpacker.unpackMapHeader();
|
2023-10-28 00:29:26 +00:00
|
|
|
Map<String, Object> attrs = HashMap.newHashMap(mapSize);
|
2022-04-23 09:58:49 +00:00
|
|
|
for (int i = 0; i < mapSize; i++) {
|
2022-06-20 09:37:40 +00:00
|
|
|
String key = commonValueStrings.decode(unpacker.unpackInt());
|
2022-04-23 09:58:49 +00:00
|
|
|
Value v = unpacker.unpackValue();
|
|
|
|
if (v.isStringValue()) {
|
|
|
|
attrs.put(key, v.asStringValue().asString());
|
|
|
|
} else if (v.isIntegerValue()) {
|
|
|
|
attrs.put(key, v.asIntegerValue().toLong());
|
|
|
|
} else if (v.isFloatValue()) {
|
|
|
|
attrs.put(key, v.asFloatValue().toDouble());
|
|
|
|
} else if (v.isBooleanValue()) {
|
|
|
|
attrs.put(key, v.asBooleanValue().getBoolean());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
int commandSize = unpacker.unpackArrayHeader();
|
|
|
|
int[] commands = new int[commandSize];
|
|
|
|
for (int i = 0; i < commandSize; i++) {
|
|
|
|
commands[i] = unpacker.unpackInt();
|
|
|
|
}
|
2022-06-20 09:37:40 +00:00
|
|
|
String layer = commonLayerStrings.decode(extractLayerIdFromKey(entry.key()));
|
2022-04-23 09:58:49 +00:00
|
|
|
return new VectorTile.Feature(
|
|
|
|
layer,
|
|
|
|
id,
|
|
|
|
new VectorTile.VectorGeometry(commands, geomType, scale),
|
|
|
|
attrs,
|
|
|
|
group
|
|
|
|
);
|
|
|
|
} catch (IOException e) {
|
|
|
|
throw new IllegalStateException(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-24 12:10:47 +00:00
|
|
|
public VectorTile getVectorTile() {
|
2023-12-14 11:53:21 +00:00
|
|
|
return getVectorTile(null);
|
|
|
|
}
|
|
|
|
|
|
|
|
public VectorTile getVectorTile(LayerAttrStats.Updater layerStats) {
|
2023-09-24 12:10:47 +00:00
|
|
|
VectorTile tile = new VectorTile();
|
2023-12-14 11:53:21 +00:00
|
|
|
if (layerStats != null) {
|
|
|
|
tile.trackLayerStats(layerStats.forZoom(tileCoord.z()));
|
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
List<VectorTile.Feature> items = new ArrayList<>(entries.size());
|
2021-04-30 10:31:56 +00:00
|
|
|
String currentLayer = null;
|
2021-09-18 00:18:06 +00:00
|
|
|
for (SortableFeature entry : entries) {
|
2021-04-30 10:31:56 +00:00
|
|
|
var feature = decodeVectorTileFeature(entry);
|
|
|
|
String layer = feature.layer();
|
|
|
|
|
|
|
|
if (currentLayer == null) {
|
|
|
|
currentLayer = layer;
|
|
|
|
} else if (!currentLayer.equals(layer)) {
|
2023-09-24 12:10:47 +00:00
|
|
|
postProcessAndAddLayerFeatures(tile, currentLayer, items);
|
2021-04-30 10:31:56 +00:00
|
|
|
currentLayer = layer;
|
|
|
|
items.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
items.add(feature);
|
|
|
|
}
|
2023-09-24 12:10:47 +00:00
|
|
|
postProcessAndAddLayerFeatures(tile, currentLayer, items);
|
|
|
|
return tile;
|
2021-04-30 10:31:56 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
private void postProcessAndAddLayerFeatures(VectorTile encoder, String layer,
|
|
|
|
List<VectorTile.Feature> features) {
|
2021-06-01 10:29:55 +00:00
|
|
|
try {
|
2021-09-10 00:46:20 +00:00
|
|
|
List<VectorTile.Feature> postProcessed = profile
|
|
|
|
.postProcessLayerFeatures(layer, tileCoord.z(), features);
|
|
|
|
features = postProcessed == null ? features : postProcessed;
|
2021-10-20 01:57:47 +00:00
|
|
|
// lines are stored using a higher precision so that rounding does not
|
|
|
|
// introduce artificial intersections between endpoints to confuse line merging,
|
|
|
|
// so we have to reduce the precision here, now that line merging is done.
|
2023-09-24 12:10:47 +00:00
|
|
|
unscaleAndRemovePointsOutsideBuffer(features, config.maxPointBuffer());
|
|
|
|
// also remove points more than --max-point-buffer pixels outside the tile if the
|
|
|
|
// user has requested a narrower buffer than the profile provides by default
|
2022-04-23 09:58:49 +00:00
|
|
|
} catch (Throwable e) { // NOSONAR - OK to catch Throwable since we re-throw Errors
|
2021-09-10 00:46:20 +00:00
|
|
|
// failures in tile post-processing happen very late so err on the side of caution and
|
|
|
|
// log failures, only throwing when it's a fatal error
|
2021-07-29 01:47:13 +00:00
|
|
|
if (e instanceof GeometryException geoe) {
|
|
|
|
geoe.log(stats, "postprocess_layer",
|
2023-10-31 02:14:46 +00:00
|
|
|
"Caught error postprocessing features for " + layer + " layer on " + tileCoord, config.logJtsExceptions());
|
2022-04-23 09:58:49 +00:00
|
|
|
} else if (e instanceof Error err) {
|
|
|
|
LOGGER.error("Caught fatal error postprocessing features {} {}", layer, tileCoord, e);
|
2021-07-29 01:47:13 +00:00
|
|
|
throw err;
|
2022-04-23 09:58:49 +00:00
|
|
|
} else {
|
|
|
|
LOGGER.error("Caught error postprocessing features {} {}", layer, tileCoord, e);
|
2021-07-29 01:47:13 +00:00
|
|
|
}
|
2021-06-01 10:29:55 +00:00
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
encoder.addLayerFeatures(layer, features);
|
2021-06-01 10:29:55 +00:00
|
|
|
}
|
|
|
|
|
2021-09-10 00:46:20 +00:00
|
|
|
void add(SortableFeature entry) {
|
2021-06-07 11:46:03 +00:00
|
|
|
numFeaturesProcessed.incrementAndGet();
|
2021-09-18 00:18:06 +00:00
|
|
|
long key = entry.key();
|
|
|
|
if (extractHasGroupFromKey(key)) {
|
|
|
|
byte thisLayer = extractLayerIdFromKey(key);
|
2021-04-30 10:31:56 +00:00
|
|
|
if (counts == null) {
|
2022-03-01 13:43:19 +00:00
|
|
|
counts = Hppc.newLongLongHashMap();
|
2022-04-23 09:58:49 +00:00
|
|
|
lastLayer = thisLayer;
|
|
|
|
} else if (thisLayer != lastLayer) {
|
|
|
|
lastLayer = thisLayer;
|
2021-04-30 10:31:56 +00:00
|
|
|
counts.clear();
|
|
|
|
}
|
2021-09-10 00:46:20 +00:00
|
|
|
var groupInfo = peekAtGroupInfo(entry.value());
|
2021-04-30 10:31:56 +00:00
|
|
|
long old = counts.getOrDefault(groupInfo.group(), 0);
|
|
|
|
if (groupInfo.limit() > 0 && old >= groupInfo.limit()) {
|
2021-09-10 00:46:20 +00:00
|
|
|
// discard if there are to many features in this group already
|
2021-04-30 10:31:56 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
counts.put(groupInfo.group(), old + 1);
|
|
|
|
}
|
|
|
|
entries.add(entry);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String toString() {
|
|
|
|
return "TileFeatures{" +
|
2021-09-10 00:46:20 +00:00
|
|
|
"tile=" + tileCoord +
|
2021-04-30 10:31:56 +00:00
|
|
|
", num entries=" + entries.size() +
|
|
|
|
'}';
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|