From dd505aeb3386bd15651b99678efdd270a2e225de Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 30 Apr 2021 06:31:56 -0400 Subject: [PATCH] refactor feature data records --- pom.xml | 6 - .../onthegomap/flatmap/FeatureRenderer.java | 12 +- .../com/onthegomap/flatmap/LayerFeature.java | 30 -- .../com/onthegomap/flatmap/MbtilesWriter.java | 9 +- .../onthegomap/flatmap/OpenMapTilesMain.java | 8 +- .../com/onthegomap/flatmap/OsmInputFile.java | 4 +- .../java/com/onthegomap/flatmap/Profile.java | 15 +- .../onthegomap/flatmap/RenderedFeature.java | 20 + .../com/onthegomap/flatmap/Translations.java | 3 +- .../onthegomap/flatmap/VectorTileEncoder.java | 116 +++-- .../collections/ExternalMergeSort.java | 4 +- .../flatmap/collections/FeatureGroup.java | 319 ++++++++++++++ .../{MergeSort.java => FeatureSort.java} | 6 +- .../collections/MergeSortFeatureMap.java | 404 ------------------ .../flatmap/monitoring/ProgressLoggers.java | 5 +- .../flatmap/profiles/OpenMapTilesProfile.java | 9 +- .../flatmap/reader/NaturalEarthReader.java | 4 +- .../flatmap/reader/OpenStreetMapReader.java | 8 +- .../com/onthegomap/flatmap/reader/Reader.java | 11 +- .../flatmap/reader/ShapefileReader.java | 10 +- .../onthegomap/flatmap/OsmInputFileTest.java | 4 +- .../com/onthegomap/flatmap/TestUtils.java | 6 - .../flatmap/VectorTileEncoderTest.java | 119 +++--- .../com/onthegomap/flatmap/WikidataTest.java | 8 +- ...tureMapTest.java => FeatureGroupTest.java} | 88 ++-- ...ergeSortTest.java => FeatureSortTest.java} | 26 +- .../monitoring/ProgressLoggersTest.java | 3 +- .../reader/NaturalEarthReaderTest.java | 6 +- .../flatmap/reader/ShapefileReaderTest.java | 7 +- 29 files changed, 598 insertions(+), 672 deletions(-) delete mode 100644 src/main/java/com/onthegomap/flatmap/LayerFeature.java create mode 100644 src/main/java/com/onthegomap/flatmap/RenderedFeature.java create mode 100644 src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java rename src/main/java/com/onthegomap/flatmap/collections/{MergeSort.java => FeatureSort.java} (85%) delete mode 100644 src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java rename src/test/java/com/onthegomap/flatmap/collections/{MergeSortFeatureMapTest.java => FeatureGroupTest.java} (71%) rename src/test/java/com/onthegomap/flatmap/collections/{MergeSortTest.java => FeatureSortTest.java} (70%) diff --git a/pom.xml b/pom.xml index 3eaae282..ad4e4087 100644 --- a/pom.xml +++ b/pom.xml @@ -130,12 +130,6 @@ 3.9.0 test - - junit - junit - RELEASE - test - diff --git a/src/main/java/com/onthegomap/flatmap/FeatureRenderer.java b/src/main/java/com/onthegomap/flatmap/FeatureRenderer.java index eed64bb1..53f9ee07 100644 --- a/src/main/java/com/onthegomap/flatmap/FeatureRenderer.java +++ b/src/main/java/com/onthegomap/flatmap/FeatureRenderer.java @@ -1,6 +1,6 @@ package com.onthegomap.flatmap; -import com.onthegomap.flatmap.collections.MergeSort; +import com.onthegomap.flatmap.collections.FeatureSort; import java.util.function.Consumer; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; @@ -22,11 +22,11 @@ public class FeatureRenderer { this.config = config; } - public void renderFeature(RenderableFeature feature, Consumer consumer) { + public void renderFeature(RenderableFeature feature, Consumer consumer) { renderGeometry(feature.getGeometry(), feature, consumer); } - public void renderGeometry(Geometry geom, RenderableFeature feature, Consumer consumer) { + public void renderGeometry(Geometry geom, RenderableFeature feature, Consumer consumer) { // TODO what about converting between area and line? if (geom instanceof Point point) { addPointFeature(feature, point, consumer); @@ -44,15 +44,15 @@ public class FeatureRenderer { } } - private void addPointFeature(RenderableFeature feature, Point point, Consumer consumer) { + private void addPointFeature(RenderableFeature feature, Point point, Consumer consumer) { // TODO render features into tile } - private void addPointFeature(RenderableFeature feature, MultiPoint points, Consumer consumer) { + private void addPointFeature(RenderableFeature feature, MultiPoint points, Consumer consumer) { // TODO render features into tile } - private void addLinearFeature(RenderableFeature feature, Geometry geom, Consumer consumer) { + private void addLinearFeature(RenderableFeature feature, Geometry geom, Consumer consumer) { // TODO render lines / areas into tile } } diff --git a/src/main/java/com/onthegomap/flatmap/LayerFeature.java b/src/main/java/com/onthegomap/flatmap/LayerFeature.java deleted file mode 100644 index a99c8f34..00000000 --- a/src/main/java/com/onthegomap/flatmap/LayerFeature.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.onthegomap.flatmap; - -import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap.FeatureMapKey; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap.FeatureMapValue; -import java.util.Map; - -public record LayerFeature( - boolean hasGroup, - long group, - int zorder, - Map attrs, - byte geomType, - int[] commands, - long id -) implements VectorTileFeature { - - public static LayerFeature of(FeatureMapKey key, FeatureMapValue value) { - return new LayerFeature( - key.hasGroup(), - value.group(), - key.zOrder(), - value.attrs(), - value.geomType(), - value.commands(), - value.featureId() - ); - } - -} diff --git a/src/main/java/com/onthegomap/flatmap/MbtilesWriter.java b/src/main/java/com/onthegomap/flatmap/MbtilesWriter.java index adfd5aef..d42d383e 100644 --- a/src/main/java/com/onthegomap/flatmap/MbtilesWriter.java +++ b/src/main/java/com/onthegomap/flatmap/MbtilesWriter.java @@ -1,7 +1,6 @@ package com.onthegomap.flatmap; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures; +import com.onthegomap.flatmap.collections.FeatureGroup; import com.onthegomap.flatmap.geo.TileCoord; import com.onthegomap.flatmap.monitoring.ProgressLoggers; import com.onthegomap.flatmap.monitoring.Stats; @@ -33,7 +32,7 @@ public class MbtilesWriter { } - public static void writeOutput(long featureCount, MergeSortFeatureMap features, File output, FlatMapConfig config) { + public static void writeOutput(long featureCount, FeatureGroup features, File output, FlatMapConfig config) { Stats stats = config.stats(); output.delete(); MbtilesWriter writer = new MbtilesWriter(config.stats()); @@ -56,8 +55,8 @@ public class MbtilesWriter { topology.awaitAndLog(loggers, config.logInterval()); } - public void tileEncoder(Supplier prev, Consumer next) throws Exception { - MergeSortFeatureMap.TileFeatures tileFeatures, last = null; + public void tileEncoder(Supplier prev, Consumer next) throws Exception { + FeatureGroup.TileFeatures tileFeatures, last = null; byte[] lastBytes = null, lastEncoded = null; while ((tileFeatures = prev.get()) != null) { featuresProcessed.addAndGet(tileFeatures.getNumFeatures()); diff --git a/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java b/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java index 81b39445..6f1b4cf8 100644 --- a/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java +++ b/src/main/java/com/onthegomap/flatmap/OpenMapTilesMain.java @@ -1,8 +1,8 @@ package com.onthegomap.flatmap; +import com.onthegomap.flatmap.collections.FeatureGroup; +import com.onthegomap.flatmap.collections.FeatureSort; import com.onthegomap.flatmap.collections.LongLongMap; -import com.onthegomap.flatmap.collections.MergeSort; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap; import com.onthegomap.flatmap.profiles.OpenMapTilesProfile; import com.onthegomap.flatmap.reader.NaturalEarthReader; import com.onthegomap.flatmap.reader.OpenStreetMapReader; @@ -66,8 +66,8 @@ public class OpenMapTilesMain { FileUtils.forceMkdir(tmpDir.toFile()); File nodeDb = tmpDir.resolve("node.db").toFile(); LongLongMap nodeLocations = new LongLongMap.MapdbSortedTable(nodeDb); - MergeSort featureDb = MergeSort.newExternalMergeSort(tmpDir.resolve("feature.db"), threads, stats); - MergeSortFeatureMap featureMap = new MergeSortFeatureMap(featureDb, profile); + FeatureSort featureDb = FeatureSort.newExternalMergeSort(tmpDir.resolve("feature.db"), threads, stats); + FeatureGroup featureMap = new FeatureGroup(featureDb, profile); FlatMapConfig config = new FlatMapConfig(profile, envelope, threads, stats, logInterval); FeatureRenderer renderer = new FeatureRenderer(config); diff --git a/src/main/java/com/onthegomap/flatmap/OsmInputFile.java b/src/main/java/com/onthegomap/flatmap/OsmInputFile.java index efe1679e..ffc60a8c 100644 --- a/src/main/java/com/onthegomap/flatmap/OsmInputFile.java +++ b/src/main/java/com/onthegomap/flatmap/OsmInputFile.java @@ -5,7 +5,7 @@ import com.graphhopper.reader.ReaderElement; import com.graphhopper.reader.osm.pbf.PbfDecoder; import com.graphhopper.reader.osm.pbf.PbfStreamSplitter; import com.graphhopper.reader.osm.pbf.Sink; -import com.onthegomap.flatmap.worker.Topology.SourceStep; +import com.onthegomap.flatmap.worker.Topology; import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.File; @@ -80,7 +80,7 @@ public class OsmInputFile { } } - public SourceStep read(int threads) { + public Topology.SourceStep read(int threads) { return next -> readTo(next, threads); } diff --git a/src/main/java/com/onthegomap/flatmap/Profile.java b/src/main/java/com/onthegomap/flatmap/Profile.java index b4034395..120d5eea 100644 --- a/src/main/java/com/onthegomap/flatmap/Profile.java +++ b/src/main/java/com/onthegomap/flatmap/Profile.java @@ -1,24 +1,24 @@ package com.onthegomap.flatmap; import com.graphhopper.reader.ReaderRelation; -import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature; -import com.onthegomap.flatmap.reader.OpenStreetMapReader.RelationInfo; +import com.onthegomap.flatmap.reader.OpenStreetMapReader; import java.util.List; public interface Profile { - List preprocessOsmRelation(ReaderRelation relation); + List preprocessOsmRelation(ReaderRelation relation); void processFeature(SourceFeature sourceFeature, RenderableFeatures features); void release(); - List postProcessLayerFeatures(String layer, int zoom, List items); + List postProcessLayerFeatures(String layer, int zoom, + List items); class NullProfile implements Profile { @Override - public List preprocessOsmRelation(ReaderRelation relation) { + public List preprocessOsmRelation(ReaderRelation relation) { return null; } @@ -33,8 +33,9 @@ public interface Profile { } @Override - public List postProcessLayerFeatures(String layer, int zoom, - List items) { + public List postProcessLayerFeatures(String layer, + int zoom, + List items) { return items; } } diff --git a/src/main/java/com/onthegomap/flatmap/RenderedFeature.java b/src/main/java/com/onthegomap/flatmap/RenderedFeature.java new file mode 100644 index 00000000..2e8f0e45 --- /dev/null +++ b/src/main/java/com/onthegomap/flatmap/RenderedFeature.java @@ -0,0 +1,20 @@ +package com.onthegomap.flatmap; + +import com.onthegomap.flatmap.geo.TileCoord; +import java.util.Optional; + +public record RenderedFeature( + TileCoord tile, + VectorTileEncoder.Feature vectorTileFeature, + int zOrder, + Optional group +) { + + public RenderedFeature { + assert vectorTileFeature != null; + } + + public static record Group(long group, int limit) { + + } +} diff --git a/src/main/java/com/onthegomap/flatmap/Translations.java b/src/main/java/com/onthegomap/flatmap/Translations.java index f09618fe..a87ae271 100644 --- a/src/main/java/com/onthegomap/flatmap/Translations.java +++ b/src/main/java/com/onthegomap/flatmap/Translations.java @@ -1,7 +1,6 @@ package com.onthegomap.flatmap; import com.graphhopper.reader.ReaderElement; -import com.onthegomap.flatmap.Wikidata.WikidataTranslations; import java.util.List; import java.util.Map; @@ -16,7 +15,7 @@ public class Translations { return null; } - public void addTranslationProvider(WikidataTranslations load) { + public void addTranslationProvider(Wikidata.WikidataTranslations load) { // TODO } diff --git a/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java b/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java index 7ae48496..f22add6f 100644 --- a/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java +++ b/src/main/java/com/onthegomap/flatmap/VectorTileEncoder.java @@ -24,6 +24,7 @@ import com.google.common.primitives.Ints; import com.google.protobuf.InvalidProtocolBufferException; import com.onthegomap.flatmap.geo.GeoUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -66,13 +67,13 @@ public class VectorTileEncoder { private static final double SCALE = ((double) EXTENT) / SIZE; private final Map layers = new LinkedHashMap<>(); - public static int[] getCommands(Geometry input) { + private static int[] getCommands(Geometry input) { var encoder = new CommandEncoder(); encoder.accept(input); return encoder.result.toArray(); } - public static VectorTile.Tile.GeomType toGeomType(Geometry geometry) { + private static VectorTile.Tile.GeomType toGeomType(Geometry geometry) { if (geometry instanceof Point || geometry instanceof MultiPoint) { return VectorTile.Tile.GeomType.POINT; } else if (geometry instanceof LineString || geometry instanceof MultiLineString) { @@ -97,7 +98,7 @@ public class VectorTileEncoder { return ((n >> 1) ^ (-(n & 1))); } - public static Geometry decodeCommands(byte geomTypeByte, int[] commands) { + private static Geometry decodeCommands(byte geomTypeByte, int[] commands) { VectorTile.Tile.GeomType geomType = Objects.requireNonNull(VectorTile.Tile.GeomType.forNumber(geomTypeByte)); GeometryFactory gf = GeoUtils.gf; int x = 0; @@ -226,13 +227,13 @@ public class VectorTileEncoder { return geometry; } - public static List decode(byte[] encoded) { + public static List decode(byte[] encoded) { try { VectorTile.Tile tile = VectorTile.Tile.parseFrom(encoded); - List features = new ArrayList<>(); + List features = new ArrayList<>(); for (VectorTile.Tile.Layer layer : tile.getLayersList()) { String layerName = layer.getName(); - int extent = layer.getExtent(); + assert layer.getExtent() == 4096; List keys = layer.getKeysList(); List values = new ArrayList<>(); @@ -266,12 +267,11 @@ public class VectorTileEncoder { attrs.put(key, value); } Geometry geometry = decodeCommands(feature.getType(), feature.getGeometryList()); - features.add(new DecodedFeature( + features.add(new Feature( layerName, - extent, - geometry, - attrs, - feature.getId() + feature.getId(), + encodeGeometry(geometry), + attrs )); } } @@ -285,18 +285,11 @@ public class VectorTileEncoder { return decodeCommands((byte) type.getNumber(), geometryList.stream().mapToInt(i -> i).toArray()); } - public interface VectorTileFeature { - - int[] commands(); - - long id(); - - byte geomType(); - - Map attrs(); + public static VectorGeometry encodeGeometry(Geometry geometry) { + return new VectorGeometry(getCommands(geometry), (byte) toGeomType(geometry).getNumber()); } - public VectorTileEncoder addLayerFeatures(String layerName, List features) { + public VectorTileEncoder addLayerFeatures(String layerName, List features) { if (features.isEmpty()) { return this; } @@ -307,8 +300,8 @@ public class VectorTileEncoder { layers.put(layerName, layer); } - for (VectorTileFeature inFeature : features) { - if (inFeature.commands().length > 0) { + for (Feature inFeature : features) { + if (inFeature.geometry().commands().length > 0) { EncodedFeature outFeature = new EncodedFeature(inFeature); for (Map.Entry e : inFeature.attrs().entrySet()) { @@ -370,8 +363,8 @@ public class VectorTileEncoder { featureBuilder.setId(feature.id); } - featureBuilder.setType(VectorTile.Tile.GeomType.forNumber(feature.geometryType)); - featureBuilder.addAllGeometry(Ints.asList(feature.geometry)); + featureBuilder.setType(VectorTile.Tile.GeomType.forNumber(feature.geometry().geomType())); + featureBuilder.addAllGeometry(Ints.asList(feature.geometry().commands())); tileLayer.addFeatures(featureBuilder.build()); } @@ -391,6 +384,63 @@ public class VectorTileEncoder { } } + public static record VectorGeometry(int[] commands, byte geomType) { + + public Geometry decode() { + return decodeCommands(geomType, commands); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VectorGeometry that = (VectorGeometry) o; + + if (geomType != that.geomType) { + return false; + } + return Arrays.equals(commands, that.commands); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(commands); + result = 31 * result + (int) geomType; + return result; + } + + @Override + public String toString() { + return "VectorGeometry[" + + "commands=int[" + commands.length + + "], geomType=" + geomType + + " (" + GeomType.forNumber(geomType) + + ")]"; + } + } + + public static record Feature( + String layer, + long id, + VectorGeometry geometry, + Map attrs + ) { + + public Feature copyWithNewGeometry(Geometry newGeometry) { + return new Feature( + layer, + id, + encodeGeometry(newGeometry), + attrs + ); + } + } + private static class CommandEncoder { private final IntArrayList result = new IntArrayList(); @@ -503,23 +553,13 @@ public class VectorTileEncoder { } } - private static final record EncodedFeature(IntArrayList tags, long id, byte geometryType, int[] geometry) { + private static final record EncodedFeature(IntArrayList tags, long id, VectorGeometry geometry) { - EncodedFeature(VectorTileFeature in) { - this(new IntArrayList(), in.id(), in.geomType(), in.commands()); + EncodedFeature(Feature in) { + this(new IntArrayList(), in.id(), in.geometry()); } } - public static final record DecodedFeature( - String layerName, - int extent, - Geometry geometry, - Map attributes, - long id - ) { - - } - private static final class Layer { private final List encodedFeatures = new ArrayList<>(); diff --git a/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java b/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java index e7fdb8da..273e5706 100644 --- a/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java +++ b/src/main/java/com/onthegomap/flatmap/collections/ExternalMergeSort.java @@ -27,9 +27,9 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class ExternalMergeSort implements MergeSort { +class ExternalMergeSort implements FeatureSort { - private static final Logger LOGGER = LoggerFactory.getLogger(MergeSort.class); + private static final Logger LOGGER = LoggerFactory.getLogger(FeatureSort.class); private static final long MAX_CHUNK_SIZE = 1_000_000_000; // 1GB diff --git a/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java b/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java new file mode 100644 index 00000000..03891666 --- /dev/null +++ b/src/main/java/com/onthegomap/flatmap/collections/FeatureGroup.java @@ -0,0 +1,319 @@ +package com.onthegomap.flatmap.collections; + +import com.carrotsearch.hppc.LongLongHashMap; +import com.graphhopper.coll.GHLongLongHashMap; +import com.onthegomap.flatmap.Profile; +import com.onthegomap.flatmap.RenderedFeature; +import com.onthegomap.flatmap.VectorTileEncoder; +import com.onthegomap.flatmap.geo.TileCoord; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import org.msgpack.core.MessageBufferPacker; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.Value; +import org.msgpack.value.ValueFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record FeatureGroup(FeatureSort sorter, Profile profile, CommonStringEncoder commonStrings) + implements Consumer, Iterable { + + public static final int Z_ORDER_BITS = 23; + public static final int Z_ORDER_MAX = (1 << (Z_ORDER_BITS - 1)) - 1; + public static final int Z_ORDER_MIN = -(1 << (Z_ORDER_BITS - 1)); + private static final int Z_ORDER_MASK = (1 << Z_ORDER_BITS) - 1; + private static final Logger LOGGER = LoggerFactory.getLogger(FeatureGroup.class); + private static final ThreadLocal messagePackers = ThreadLocal + .withInitial(MessagePack::newDefaultBufferPacker); + + public FeatureGroup(FeatureSort sorter, Profile profile) { + this(sorter, profile, new CommonStringEncoder()); + } + + static long encodeSortKey(int tile, byte layer, int zOrder, boolean hasGroup) { + zOrder = -zOrder - 1; + return ((long) tile << 32L) | ((long) (layer & 0xff) << 24L) | (((zOrder - Z_ORDER_MIN) & Z_ORDER_MASK) << 1L) | ( + hasGroup ? 1 : 0); + } + + static boolean extractHasGroupFromSortKey(long sortKey) { + return (sortKey & 1) == 1; + } + + static int extractTileFromSortKey(long sortKey) { + return (int) (sortKey >> 32L); + } + + static byte extractLayerIdFromSortKey(long sortKey) { + return (byte) (sortKey >> 24); + } + + static int extractZorderFromKey(long sortKey) { + return Z_ORDER_MAX - ((int) ((sortKey >> 1) & Z_ORDER_MASK)); + } + + private static RenderedFeature.Group decodeGroupInfo(byte[] encoded) { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { + long group = unpacker.unpackLong(); + int limit = unpacker.unpackInt(); + return new RenderedFeature.Group(group, limit); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + public FeatureSort.Entry encode(RenderedFeature feature) { + return new FeatureSort.Entry( + encodeSortKey(feature), + encodeValue(feature) + ); + } + + private long encodeSortKey(RenderedFeature feature) { + var vectorTileFeature = feature.vectorTileFeature(); + commonStrings.encode(vectorTileFeature.layer()); + return encodeSortKey( + feature.tile().encoded(), + commonStrings.encode(vectorTileFeature.layer()), + feature.zOrder(), + feature.group().isPresent() + ); + } + + private byte[] encodeValue(RenderedFeature feature) { + MessageBufferPacker packer = messagePackers.get(); + packer.clear(); + try { + var groupInfoOption = feature.group(); + if (groupInfoOption.isPresent()) { + var groupInfo = groupInfoOption.get(); + packer.packLong(groupInfo.group()); + packer.packInt(groupInfo.limit()); + } + var vectorTileFeature = feature.vectorTileFeature(); + packer.packLong(vectorTileFeature.id()); + packer.packByte(vectorTileFeature.geometry().geomType()); + var attrs = vectorTileFeature.attrs(); + packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count()); + for (Map.Entry entry : attrs.entrySet()) { + if (entry.getValue() != null) { + packer.packByte(commonStrings.encode(entry.getKey())); + Object value = entry.getValue(); + if (value instanceof String) { + packer.packValue(ValueFactory.newString((String) value)); + } else if (value instanceof Integer) { + packer.packValue(ValueFactory.newInteger(((Integer) value).longValue())); + } else if (value instanceof Long) { + packer.packValue(ValueFactory.newInteger((Long) value)); + } else if (value instanceof Float) { + packer.packValue(ValueFactory.newFloat((Float) value)); + } else if (value instanceof Double) { + packer.packValue(ValueFactory.newFloat((Double) value)); + } else if (value instanceof Boolean) { + packer.packValue(ValueFactory.newBoolean((Boolean) value)); + } else { + packer.packValue(ValueFactory.newString(value.toString())); + } + } + } + int[] commands = vectorTileFeature.geometry().commands(); + packer.packArrayHeader(commands.length); + for (int command : commands) { + packer.packInt(command); + } + packer.close(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + return packer.toByteArray(); + } + + private VectorTileEncoder.Feature decodeVectorTileFeature(FeatureSort.Entry entry) { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(entry.value())) { + if (extractHasGroupFromSortKey(entry.sortKey())) { + unpacker.unpackLong(); // group + unpacker.unpackInt(); // groupLimit + } + long id = unpacker.unpackLong(); + byte geomType = unpacker.unpackByte(); + int mapSize = unpacker.unpackMapHeader(); + Map attrs = new HashMap<>(mapSize); + for (int i = 0; i < mapSize; i++) { + String key = commonStrings.decode(unpacker.unpackByte()); + Value v = unpacker.unpackValue(); + if (v.isStringValue()) { + attrs.put(key, v.asStringValue().asString()); + } else if (v.isIntegerValue()) { + attrs.put(key, v.asIntegerValue().toLong()); + } else if (v.isFloatValue()) { + attrs.put(key, v.asFloatValue().toDouble()); + } else if (v.isBooleanValue()) { + attrs.put(key, v.asBooleanValue().getBoolean()); + } + } + int commandSize = unpacker.unpackArrayHeader(); + int[] commands = new int[commandSize]; + for (int i = 0; i < commandSize; i++) { + commands[i] = unpacker.unpackInt(); + } + return new VectorTileEncoder.Feature( + commonStrings.decode(extractLayerIdFromSortKey(entry.sortKey())), + id, + new VectorTileEncoder.VectorGeometry(commands, geomType), + attrs + ); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void accept(FeatureSort.Entry entry) { + sorter.add(entry); + } + + @Override + public Iterator iterator() { + Iterator entries = sorter.iterator(); + if (!entries.hasNext()) { + return Collections.emptyIterator(); + } + FeatureSort.Entry firstFeature = entries.next(); + return new Iterator<>() { + private FeatureSort.Entry lastFeature = firstFeature; + private int lastTileId = extractTileFromSortKey(firstFeature.sortKey()); + + @Override + public boolean hasNext() { + return lastFeature != null; + } + + @Override + public TileFeatures next() { + TileFeatures result = new TileFeatures(lastTileId); + result.accept(lastFeature); + int lastTile = lastTileId; + + while (entries.hasNext()) { + FeatureSort.Entry next = entries.next(); + lastFeature = next; + lastTileId = extractTileFromSortKey(lastFeature.sortKey()); + if (lastTile != lastTileId) { + return result; + } + result.accept(next); + } + lastFeature = null; + return result; + } + }; + } + + public long getStorageSize() { + return sorter.getStorageSize(); + } + + public class TileFeatures implements Consumer { + + private final TileCoord tile; + private final List entries = new ArrayList<>(); + + private LongLongHashMap counts = null; + private byte layer = Byte.MAX_VALUE; + + public TileFeatures(int tile) { + this.tile = TileCoord.decode(tile); + } + + public long getNumFeatures() { + return 0; + } + + public TileCoord coord() { + return tile; + } + + public boolean hasSameContents(TileFeatures other) { + if (other == null || other.entries.size() != entries.size()) { + return false; + } + for (int i = 0; i < entries.size(); i++) { + byte[] a = entries.get(i).value(); + byte[] b = other.entries.get(i).value(); + if (!Arrays.equals(a, b)) { + return false; + } + } + return true; + } + + public VectorTileEncoder getTile() { + VectorTileEncoder encoder = new VectorTileEncoder(); + List items = new ArrayList<>(entries.size()); + String currentLayer = null; + for (int index = entries.size() - 1; index >= 0; index--) { + FeatureSort.Entry entry = entries.get(index); + + var feature = decodeVectorTileFeature(entry); + String layer = feature.layer(); + + if (currentLayer == null) { + currentLayer = layer; + } else if (!currentLayer.equals(layer)) { + encoder.addLayerFeatures( + currentLayer, + profile.postProcessLayerFeatures(currentLayer, tile.z(), items) + ); + currentLayer = layer; + items.clear(); + } + + items.add(feature); + } + encoder.addLayerFeatures( + currentLayer, + profile.postProcessLayerFeatures(currentLayer, tile.z(), items) + ); + return encoder; + } + + @Override + public void accept(FeatureSort.Entry entry) { + long sortKey = entry.sortKey(); + if (extractHasGroupFromSortKey(sortKey)) { + byte thisLayer = extractLayerIdFromSortKey(sortKey); + if (counts == null) { + counts = new GHLongLongHashMap(); + layer = thisLayer; + } else if (thisLayer != layer) { + layer = thisLayer; + counts.clear(); + } + var groupInfo = decodeGroupInfo(entry.value()); + long old = counts.getOrDefault(groupInfo.group(), 0); + if (groupInfo.limit() > 0 && old >= groupInfo.limit()) { + return; + } + counts.put(groupInfo.group(), old + 1); + } + entries.add(entry); + } + + @Override + public String toString() { + return "TileFeatures{" + + "tile=" + tile + + ", num entries=" + entries.size() + + '}'; + } + } +} diff --git a/src/main/java/com/onthegomap/flatmap/collections/MergeSort.java b/src/main/java/com/onthegomap/flatmap/collections/FeatureSort.java similarity index 85% rename from src/main/java/com/onthegomap/flatmap/collections/MergeSort.java rename to src/main/java/com/onthegomap/flatmap/collections/FeatureSort.java index ca39d426..09f2b886 100644 --- a/src/main/java/com/onthegomap/flatmap/collections/MergeSort.java +++ b/src/main/java/com/onthegomap/flatmap/collections/FeatureSort.java @@ -7,13 +7,13 @@ import java.util.Arrays; import java.util.List; import org.jetbrains.annotations.NotNull; -public interface MergeSort extends Iterable { +public interface FeatureSort extends Iterable { - static MergeSort newExternalMergeSort(Path tempDir, int threads, Stats stats) { + static FeatureSort newExternalMergeSort(Path tempDir, int threads, Stats stats) { return new ExternalMergeSort(tempDir, threads, stats); } - static MergeSort newExternalMergeSort(Path dir, int workers, int chunkSizeLimit, Stats stats) { + static FeatureSort newExternalMergeSort(Path dir, int workers, int chunkSizeLimit, Stats stats) { return new ExternalMergeSort(dir, workers, chunkSizeLimit, stats); } diff --git a/src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java b/src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java deleted file mode 100644 index 40f03748..00000000 --- a/src/main/java/com/onthegomap/flatmap/collections/MergeSortFeatureMap.java +++ /dev/null @@ -1,404 +0,0 @@ -package com.onthegomap.flatmap.collections; - -import com.carrotsearch.hppc.LongLongHashMap; -import com.graphhopper.coll.GHLongLongHashMap; -import com.onthegomap.flatmap.LayerFeature; -import com.onthegomap.flatmap.Profile; -import com.onthegomap.flatmap.VectorTileEncoder; -import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures; -import com.onthegomap.flatmap.geo.TileCoord; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Consumer; -import org.jetbrains.annotations.NotNull; -import org.locationtech.jts.geom.Geometry; -import org.msgpack.core.MessageBufferPacker; -import org.msgpack.core.MessagePack; -import org.msgpack.core.MessageUnpacker; -import org.msgpack.value.Value; -import org.msgpack.value.ValueFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public record MergeSortFeatureMap(MergeSort mergeSort, Profile profile, CommonStringEncoder commonStrings) - implements Consumer, Iterable { - - private static final Logger LOGGER = LoggerFactory.getLogger(MergeSortFeatureMap.class); - - public MergeSortFeatureMap(MergeSort mergeSort, Profile profile) { - this(mergeSort, profile, new CommonStringEncoder()); - } - - public MergeSort.Entry encode( - long featureId, - TileCoord tile, - String layer, - Map attrs, - Geometry geom, - int zOrder, - boolean hasGroup, - int groupLimit - ) { - return new MergeSort.Entry( - FeatureMapKey.encode(tile.encoded(), commonStrings.encode(layer), zOrder, hasGroup), - FeatureMapValue.from(featureId, attrs, geom, hasGroup, groupLimit).encode(commonStrings) - ); - } - - @Override - public void accept(MergeSort.Entry entry) { - mergeSort.add(entry); - } - - @Override - public Iterator iterator() { - Iterator entries = mergeSort.iterator(); - if (!entries.hasNext()) { - return Collections.emptyIterator(); - } - MergeSort.Entry firstFeature = entries.next(); - return new Iterator<>() { - private MergeSort.Entry lastFeature = firstFeature; - private int lastTileId = FeatureMapKey.extractTileFromKey(firstFeature.sortKey()); - - @Override - public boolean hasNext() { - return lastFeature != null; - } - - @Override - public TileFeatures next() { - TileFeatures result = new TileFeatures(lastTileId); - result.accept(lastFeature); - int lastTile = lastTileId; - - while (entries.hasNext()) { - MergeSort.Entry next = entries.next(); - lastFeature = next; - lastTileId = FeatureMapKey.extractTileFromKey(lastFeature.sortKey()); - if (lastTile != lastTileId) { - return result; - } - result.accept(next); - } - lastFeature = null; - return result; - } - }; - } - - public long getStorageSize() { - return mergeSort.getStorageSize(); - } - - public class TileFeatures implements Consumer { - - private final TileCoord tile; - private final List entries = new ArrayList<>(); - - private LongLongHashMap counts = null; - private byte layer = Byte.MAX_VALUE; - - public TileFeatures(int tile) { - this.tile = TileCoord.decode(tile); - } - - public long getNumFeatures() { - return 0; - } - - public TileCoord coord() { - return tile; - } - - public boolean hasSameContents(TileFeatures other) { - if (other == null || other.entries.size() != entries.size()) { - return false; - } - for (int i = 0; i < entries.size(); i++) { - byte[] a = entries.get(i).value(); - byte[] b = other.entries.get(i).value(); - if (!Arrays.equals(a, b)) { - return false; - } - } - return true; - } - - public VectorTileEncoder getTile() { - VectorTileEncoder encoder = new VectorTileEncoder(); - List items = new ArrayList<>(entries.size()); - String currentLayer = null; - for (int index = entries.size() - 1; index >= 0; index--) { - MergeSort.Entry entry = entries.get(index); - - FeatureMapKey key = FeatureMapKey.decode(entry.sortKey()); - FeatureMapValue value = FeatureMapValue.decode(entry.value(), key.hasGroup(), commonStrings); - String layer = commonStrings.decode(key.layer); - - if (currentLayer == null) { - currentLayer = layer; - } else if (!currentLayer.equals(layer)) { - encoder.addLayerFeatures( - currentLayer, - profile.postProcessLayerFeatures(currentLayer, tile.z(), items) - ); - currentLayer = layer; - items.clear(); - } - - items.add(LayerFeature.of(key, value)); - } - encoder.addLayerFeatures( - currentLayer, - profile.postProcessLayerFeatures(currentLayer, tile.z(), items) - ); - return encoder; - } - - @Override - public void accept(MergeSort.Entry entry) { - long sortKey = entry.sortKey(); - if (FeatureMapKey.extractHasGroupFromKey(sortKey)) { - byte thisLayer = FeatureMapKey.extractLayerIdFromKey(sortKey); - if (counts == null) { - counts = new GHLongLongHashMap(); - layer = thisLayer; - } else if (thisLayer != layer) { - layer = thisLayer; - counts.clear(); - } - var groupInfo = FeatureMapValue.decodeGroupInfo(entry.value()); - long old = counts.getOrDefault(groupInfo.group, 0); - if (groupInfo.limit > 0 && old >= groupInfo.limit) { - return; - } - counts.put(groupInfo.group, old + 1); - } - entries.add(entry); - } - - @Override - public String toString() { - return "TileFeatures{" + - "tile=" + tile + - ", num entries=" + entries.size() + - '}'; - } - } - - private static final ThreadLocal messagePackers = ThreadLocal - .withInitial(MessagePack::newDefaultBufferPacker); - - public record RenderedFeature( - long featureId, - TileCoord tile, - String layer, - int zOrder, - Optional groupInfo - ) { - - } - - public record FeatureMapKey(long encoded, TileCoord tile, byte layer, int zOrder, boolean hasGroup) implements - Comparable { - - private static final int Z_ORDER_MASK = (1 << 23) - 1; - public static final int Z_ORDER_MAX = (1 << 22) - 1; - public static final int Z_ORDER_MIN = -(1 << 22); - public static final int Z_ORDER_BITS = 23; - - public static FeatureMapKey of(int tile, byte layer, int zOrder, boolean hasGroup) { - return new FeatureMapKey(encode(tile, layer, zOrder, hasGroup), TileCoord.decode(tile), layer, zOrder, hasGroup); - } - - public static FeatureMapKey decode(long encoded) { - return of( - extractTileFromKey(encoded), - extractLayerIdFromKey(encoded), - extractZorderFromKey(encoded), - extractHasGroupFromKey(encoded) - ); - } - - public static long encode(int tile, byte layer, int zOrder, boolean hasGroup) { - zOrder = -zOrder - 1; - return ((long) tile << 32L) | ((long) (layer & 0xff) << 24L) | (((zOrder - Z_ORDER_MIN) & Z_ORDER_MASK) << 1L) | ( - hasGroup ? 1 : 0); - } - - public static boolean extractHasGroupFromKey(long sortKey) { - return (sortKey & 1) == 1; - } - - public static int extractTileFromKey(long sortKey) { - return (int) (sortKey >> 32L); - } - - public static byte extractLayerIdFromKey(long sortKey) { - return (byte) (sortKey >> 24); - } - - public static int extractZorderFromKey(long sortKey) { - return Z_ORDER_MAX - ((int) ((sortKey >> 1) & Z_ORDER_MASK)); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - FeatureMapKey that = (FeatureMapKey) o; - - return encoded == that.encoded; - } - - @Override - public int hashCode() { - return (int) (encoded ^ (encoded >>> 32)); - } - - @Override - public int compareTo(@NotNull FeatureMapKey o) { - return Long.compare(encoded, o.encoded); - } - } - - public static record FeatureMapValue( - long featureId, - Map attrs, - int[] commands, - byte geomType, - boolean hasGrouping, - int groupLimit, - long group - ) { - - public static record GroupInfo(long group, int limit) { - - } - - public static GroupInfo decodeGroupInfo(byte[] encoded) { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { - long group = unpacker.unpackLong(); - int limit = unpacker.unpackInt(); - return new GroupInfo(group, limit); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - public static FeatureMapValue from( - long featureId, - Map attrs, - Geometry geom, - boolean hasGrouping, - int groupLimit - ) { - long group = geom.getUserData() instanceof Number number ? number.longValue() : 0; - byte geomType = (byte) VectorTileEncoder.toGeomType(geom).getNumber(); - int[] commands = VectorTileEncoder.getCommands(geom); - return new FeatureMapValue( - featureId, - attrs, - commands, - geomType, - hasGrouping, - groupLimit, - group - ); - } - - public static FeatureMapValue decode(byte[] encoded, boolean hasGroup, CommonStringEncoder commonStrings) { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(encoded)) { - long group = 0; - int groupLimit = -1; - if (hasGroup) { - group = unpacker.unpackLong(); - groupLimit = unpacker.unpackInt(); - } - long id = unpacker.unpackLong(); - byte geomType = unpacker.unpackByte(); - int mapSize = unpacker.unpackMapHeader(); - Map attrs = new HashMap<>(mapSize); - for (int i = 0; i < mapSize; i++) { - String key = commonStrings.decode(unpacker.unpackByte()); - Value v = unpacker.unpackValue(); - if (v.isStringValue()) { - attrs.put(key, v.asStringValue().asString()); - } else if (v.isIntegerValue()) { - attrs.put(key, v.asIntegerValue().toLong()); - } else if (v.isFloatValue()) { - attrs.put(key, v.asFloatValue().toDouble()); - } else if (v.isBooleanValue()) { - attrs.put(key, v.asBooleanValue().getBoolean()); - } - } - int commandSize = unpacker.unpackArrayHeader(); - int[] commands = new int[commandSize]; - for (int i = 0; i < commandSize; i++) { - commands[i] = unpacker.unpackInt(); - } - return new FeatureMapValue(id, attrs, commands, geomType, hasGroup, groupLimit, group); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - public byte[] encode(CommonStringEncoder commonStrings) { - MessageBufferPacker packer = messagePackers.get(); - packer.clear(); - try { - if (hasGrouping) { - packer.packLong(group); - packer.packInt(groupLimit); - } - packer.packLong(featureId); - packer.packByte(geomType); - packer.packMapHeader((int) attrs.values().stream().filter(Objects::nonNull).count()); - for (Map.Entry entry : attrs.entrySet()) { - if (entry.getValue() != null) { - packer.packByte(commonStrings.encode(entry.getKey())); - Object value = entry.getValue(); - if (value instanceof String) { - packer.packValue(ValueFactory.newString((String) value)); - } else if (value instanceof Integer) { - packer.packValue(ValueFactory.newInteger(((Integer) value).longValue())); - } else if (value instanceof Long) { - packer.packValue(ValueFactory.newInteger((Long) value)); - } else if (value instanceof Float) { - packer.packValue(ValueFactory.newFloat((Float) value)); - } else if (value instanceof Double) { - packer.packValue(ValueFactory.newFloat((Double) value)); - } else if (value instanceof Boolean) { - packer.packValue(ValueFactory.newBoolean((Boolean) value)); - } else { - packer.packValue(ValueFactory.newString(value.toString())); - } - } - } - packer.packArrayHeader(commands.length); - for (int command : commands) { - packer.packInt(command); - } - packer.close(); - } catch (IOException e) { - throw new IllegalStateException(e); - } - return packer.toByteArray(); - } - } -} diff --git a/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java b/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java index e5ed8122..193c65f0 100644 --- a/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java +++ b/src/main/java/com/onthegomap/flatmap/monitoring/ProgressLoggers.java @@ -8,7 +8,6 @@ import static com.onthegomap.flatmap.Format.padRight; import com.graphhopper.util.Helper; import com.onthegomap.flatmap.Format; -import com.onthegomap.flatmap.monitoring.ProcessInfo.ThreadState; import com.onthegomap.flatmap.worker.Topology; import com.onthegomap.flatmap.worker.WorkQueue; import com.onthegomap.flatmap.worker.Worker; @@ -150,7 +149,7 @@ public class ProgressLoggers { public ProgressLoggers addThreadPoolStats(String name, String prefix) { boolean first = loggers.isEmpty() || !(loggers.get(loggers.size() - 1) instanceof TopologyLogger); try { - Map lastThreads = ProcessInfo.getThreadStats(); + Map lastThreads = ProcessInfo.getThreadStats(); AtomicLong lastTime = new AtomicLong(System.nanoTime()); loggers.add(new TopologyLogger(() -> { var oldAndNewThreads = new TreeMap<>(lastThreads); @@ -165,7 +164,7 @@ public class ProgressLoggers { if (!newThreads.containsKey(thread.id())) { return " -%"; } - long last = lastThreads.getOrDefault(thread.id(), ThreadState.DEFAULT).cpuTimeNanos(); + long last = lastThreads.getOrDefault(thread.id(), ProcessInfo.ThreadState.DEFAULT).cpuTimeNanos(); return padLeft(formatPercent(1d * (thread.cpuTimeNanos() - last) / timeDiff), 3); }).collect(Collectors.joining(" ", "(", ")")); diff --git a/src/main/java/com/onthegomap/flatmap/profiles/OpenMapTilesProfile.java b/src/main/java/com/onthegomap/flatmap/profiles/OpenMapTilesProfile.java index 92fee4b2..cef313f9 100644 --- a/src/main/java/com/onthegomap/flatmap/profiles/OpenMapTilesProfile.java +++ b/src/main/java/com/onthegomap/flatmap/profiles/OpenMapTilesProfile.java @@ -1,12 +1,11 @@ package com.onthegomap.flatmap.profiles; - import com.graphhopper.reader.ReaderRelation; import com.onthegomap.flatmap.Profile; import com.onthegomap.flatmap.RenderableFeatures; import com.onthegomap.flatmap.SourceFeature; import com.onthegomap.flatmap.VectorTileEncoder; -import com.onthegomap.flatmap.reader.OpenStreetMapReader.RelationInfo; +import com.onthegomap.flatmap.reader.OpenStreetMapReader; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,13 +19,13 @@ public class OpenMapTilesProfile implements Profile { } @Override - public List postProcessLayerFeatures(String layer, int zoom, - List items) { + public List postProcessLayerFeatures(String layer, int zoom, + List items) { return items; } @Override - public List preprocessOsmRelation(ReaderRelation relation) { + public List preprocessOsmRelation(ReaderRelation relation) { return null; } diff --git a/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java b/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java index 3b41cc74..b672171b 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/NaturalEarthReader.java @@ -3,7 +3,7 @@ package com.onthegomap.flatmap.reader; import com.onthegomap.flatmap.SourceFeature; import com.onthegomap.flatmap.geo.GeoUtils; import com.onthegomap.flatmap.monitoring.Stats; -import com.onthegomap.flatmap.worker.Topology.SourceStep; +import com.onthegomap.flatmap.worker.Topology; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -90,7 +90,7 @@ public class NaturalEarthReader extends Reader { } @Override - public SourceStep read() { + public Topology.SourceStep read() { return next -> { var tables = tableNames(); for (int i = 0; i < tables.size(); i++) { diff --git a/src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java b/src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java index 16639784..0230ed60 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/OpenStreetMapReader.java @@ -15,10 +15,10 @@ import com.onthegomap.flatmap.Profile; import com.onthegomap.flatmap.RenderableFeature; import com.onthegomap.flatmap.RenderableFeatures; import com.onthegomap.flatmap.SourceFeature; +import com.onthegomap.flatmap.collections.FeatureGroup; +import com.onthegomap.flatmap.collections.FeatureSort; import com.onthegomap.flatmap.collections.LongLongMap; import com.onthegomap.flatmap.collections.LongLongMultimap; -import com.onthegomap.flatmap.collections.MergeSort; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap; import com.onthegomap.flatmap.geo.GeoUtils; import com.onthegomap.flatmap.monitoring.ProgressLoggers; import com.onthegomap.flatmap.monitoring.Stats; @@ -107,7 +107,7 @@ public class OpenStreetMapReader implements Closeable { topology.awaitAndLog(loggers, config.logInterval()); } - public long pass2(FeatureRenderer renderer, MergeSortFeatureMap writer, int readerThreads, int processThreads, + public long pass2(FeatureRenderer renderer, FeatureGroup writer, int readerThreads, int processThreads, FlatMapConfig config) { Profile profile = config.profile(); AtomicLong nodesProcessed = new AtomicLong(0); @@ -119,7 +119,7 @@ public class OpenStreetMapReader implements Closeable { var topology = Topology.start("osm_pass2", stats) .fromGenerator("pbf", osmInputFile.read(readerThreads)) .addBuffer("reader_queue", 50_000, 1_000) - .addWorker("process", processThreads, (prev, next) -> { + .addWorker("process", processThreads, (prev, next) -> { RenderableFeatures features = new RenderableFeatures(); ReaderElement readerElement; while ((readerElement = prev.get()) != null) { diff --git a/src/main/java/com/onthegomap/flatmap/reader/Reader.java b/src/main/java/com/onthegomap/flatmap/reader/Reader.java index f980f899..8410721f 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/Reader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/Reader.java @@ -6,12 +6,11 @@ import com.onthegomap.flatmap.Profile; import com.onthegomap.flatmap.RenderableFeature; import com.onthegomap.flatmap.RenderableFeatures; import com.onthegomap.flatmap.SourceFeature; -import com.onthegomap.flatmap.collections.MergeSort; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap; +import com.onthegomap.flatmap.collections.FeatureGroup; +import com.onthegomap.flatmap.collections.FeatureSort; import com.onthegomap.flatmap.monitoring.ProgressLoggers; import com.onthegomap.flatmap.monitoring.Stats; import com.onthegomap.flatmap.worker.Topology; -import com.onthegomap.flatmap.worker.Topology.SourceStep; import java.io.Closeable; import java.util.concurrent.atomic.AtomicLong; import org.locationtech.jts.geom.Envelope; @@ -27,7 +26,7 @@ public abstract class Reader implements Closeable { this.stats = stats; } - public final void process(String name, FeatureRenderer renderer, MergeSortFeatureMap writer, FlatMapConfig config) { + public final void process(String name, FeatureRenderer renderer, FeatureGroup writer, FlatMapConfig config) { long featureCount = getCount(); int threads = config.threads(); Envelope env = config.envelope(); @@ -38,7 +37,7 @@ public abstract class Reader implements Closeable { var topology = Topology.start(name, stats) .fromGenerator("read", read()) .addBuffer("read_queue", 1000) - .addWorker("process", threads, (prev, next) -> { + .addWorker("process", threads, (prev, next) -> { RenderableFeatures features = new RenderableFeatures(); SourceFeature sourceFeature; while ((sourceFeature = prev.get()) != null) { @@ -70,7 +69,7 @@ public abstract class Reader implements Closeable { public abstract long getCount(); - public abstract SourceStep read(); + public abstract Topology.SourceStep read(); @Override public abstract void close(); diff --git a/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java b/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java index ba846e3d..1ebf6e5a 100644 --- a/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java +++ b/src/main/java/com/onthegomap/flatmap/reader/ShapefileReader.java @@ -3,9 +3,9 @@ package com.onthegomap.flatmap.reader; import com.onthegomap.flatmap.FeatureRenderer; import com.onthegomap.flatmap.FlatMapConfig; import com.onthegomap.flatmap.SourceFeature; -import com.onthegomap.flatmap.collections.MergeSortFeatureMap; +import com.onthegomap.flatmap.collections.FeatureGroup; import com.onthegomap.flatmap.monitoring.Stats; -import com.onthegomap.flatmap.worker.Topology.SourceStep; +import com.onthegomap.flatmap.worker.Topology; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -33,14 +33,14 @@ public class ShapefileReader extends Reader implements Closeable { private MathTransform transform; public static void process(String sourceProjection, String name, File input, FeatureRenderer renderer, - MergeSortFeatureMap writer, FlatMapConfig config) { + FeatureGroup writer, FlatMapConfig config) { try (var reader = new ShapefileReader(sourceProjection, input, config.stats())) { reader.process(name, renderer, writer, config); } } public static void process(String name, File input, FeatureRenderer renderer, - MergeSortFeatureMap writer, FlatMapConfig config) { + FeatureGroup writer, FlatMapConfig config) { process(null, name, input, renderer, writer, config); } @@ -105,7 +105,7 @@ public class ShapefileReader extends Reader implements Closeable { } @Override - public SourceStep read() { + public Topology.SourceStep read() { return next -> { try (var iter = inputSource.features()) { while (iter.hasNext()) { diff --git a/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java b/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java index 001779e3..ad5d95c5 100644 --- a/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java +++ b/src/test/java/com/onthegomap/flatmap/OsmInputFileTest.java @@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import com.graphhopper.reader.ReaderElement; -import com.onthegomap.flatmap.monitoring.Stats.InMemory; +import com.onthegomap.flatmap.monitoring.Stats; import com.onthegomap.flatmap.worker.Topology; import java.io.File; import java.util.concurrent.atomic.AtomicInteger; @@ -27,7 +27,7 @@ public class OsmInputFileTest { AtomicInteger nodes = new AtomicInteger(0); AtomicInteger ways = new AtomicInteger(0); AtomicInteger rels = new AtomicInteger(0); - Topology.start("test", new InMemory()) + Topology.start("test", new Stats.InMemory()) .fromGenerator("pbf", file.read(2)) .addBuffer("reader_queue", 1_000, 100) .sinkToConsumer("counter", 1, elem -> { diff --git a/src/test/java/com/onthegomap/flatmap/TestUtils.java b/src/test/java/com/onthegomap/flatmap/TestUtils.java index 21c6d15b..986413fa 100644 --- a/src/test/java/com/onthegomap/flatmap/TestUtils.java +++ b/src/test/java/com/onthegomap/flatmap/TestUtils.java @@ -36,12 +36,6 @@ public class TestUtils { return GeoUtils.gf.createPoint(new CoordinateXY(x, y)); } - public static Point newPointWithUserData(double x, double y, Object userData) { - Point point = GeoUtils.gf.createPoint(new CoordinateXY(x, y)); - point.setUserData(userData); - return point; - } - public static MultiPoint newMultiPoint(Point... points) { return GeoUtils.gf.createMultiPoint(points); } diff --git a/src/test/java/com/onthegomap/flatmap/VectorTileEncoderTest.java b/src/test/java/com/onthegomap/flatmap/VectorTileEncoderTest.java index eaf1e7f5..178f71bc 100644 --- a/src/test/java/com/onthegomap/flatmap/VectorTileEncoderTest.java +++ b/src/test/java/com/onthegomap/flatmap/VectorTileEncoderTest.java @@ -30,9 +30,6 @@ import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.primitives.Ints; -import com.onthegomap.flatmap.VectorTileEncoder.DecodedFeature; -import com.onthegomap.flatmap.VectorTileEncoder.VectorTileFeature; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,7 +43,6 @@ import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; import vector_tile.VectorTile; -import vector_tile.VectorTile.Tile.GeomType; /** * This class is copied from https://github.com/ElectronicChartCentre/java-vector-tile/blob/master/src/test/java/no/ecc/vectortile/VectorTileEncoderTest.java @@ -56,13 +52,14 @@ public class VectorTileEncoderTest { // Tests adapted from https://github.com/ElectronicChartCentre/java-vector-tile/blob/master/src/test/java/no/ecc/vectortile/VectorTileEncoderTest.java private static List getCommands(Geometry geom) { - return Ints.asList(VectorTileEncoder.getCommands(TRANSFORM_TO_TILE.transform(geom))); + return Ints.asList(VectorTileEncoder.encodeGeometry(TRANSFORM_TO_TILE.transform(geom)).commands()); } @Test public void testToGeomType() { - Geometry geometry = gf.createLineString(); - assertEquals(VectorTile.Tile.GeomType.LINESTRING, VectorTileEncoder.toGeomType(geometry)); + Geometry geometry = gf.createLineString(new Coordinate[]{new CoordinateXY(1, 2), new CoordinateXY(3, 4)}); + assertEquals((byte) VectorTile.Tile.GeomType.LINESTRING.getNumber(), + VectorTileEncoder.encodeGeometry(geometry).geomType()); } @Test @@ -101,22 +98,13 @@ public class VectorTileEncoderTest { ))); } - private static record SimpleVectorTileFeature( - int[] commands, - long id, - byte geomType, - Map attrs - ) implements VectorTileFeature { - - } - - private static VectorTileFeature newVectorTileFeature(Geometry geom, Map attrs) { - return new SimpleVectorTileFeature(VectorTileEncoder.getCommands(geom), 1, - (byte) VectorTileEncoder.toGeomType(geom).getNumber(), attrs); + private static VectorTileEncoder.Feature newVectorTileFeature(String layer, Geometry geom, + Map attrs) { + return new VectorTileEncoder.Feature(layer, 1, VectorTileEncoder.encodeGeometry(geom), attrs); } @Test - public void testNullAttributeValue() throws IOException { + public void testNullAttributeValue() { VectorTileEncoder vtm = new VectorTileEncoder(); Map attrs = new HashMap<>(); attrs.put("key1", "value1"); @@ -124,21 +112,23 @@ public class VectorTileEncoderTest { attrs.put("key3", "value3"); vtm.addLayerFeatures("DEPCNT", List.of( - newVectorTileFeature(newPoint(3, 6), attrs) + newVectorTileFeature("DEPCNT", newPoint(3, 6), attrs) )); byte[] encoded = vtm.encode(); assertNotSame(0, encoded.length); var decoded = VectorTileEncoder.decode(encoded); - assertEquals(List.of(new DecodedFeature("DEPCNT", 4096, newPoint(3, 6), Map.of( - "key1", "value1", - "key3", "value3" - ), 1)), decoded); + assertEquals(List + .of(new VectorTileEncoder.Feature("DEPCNT", 1, VectorTileEncoder.encodeGeometry(newPoint(3, 6)), Map.of( + "key1", "value1", + "key3", "value3" + ))), decoded); + assertSameGeometries(List.of(newPoint(3, 6)), decoded); } @Test - public void testAttributeTypes() throws IOException { + public void testAttributeTypes() { VectorTileEncoder vtm = new VectorTileEncoder(); Map attrs = Map.of( @@ -152,14 +142,14 @@ public class VectorTileEncoderTest { "key8", Boolean.FALSE ); - vtm.addLayerFeatures("DEPCNT", List.of(newVectorTileFeature(newPoint(3, 6), attrs))); + vtm.addLayerFeatures("DEPCNT", List.of(newVectorTileFeature("DEPCNT", newPoint(3, 6), attrs))); byte[] encoded = vtm.encode(); assertNotSame(0, encoded.length); - List decoded = VectorTileEncoder.decode(encoded); + List decoded = VectorTileEncoder.decode(encoded); assertEquals(1, decoded.size()); - Map decodedAttributes = decoded.get(0).attributes(); + Map decodedAttributes = decoded.get(0).attrs(); assertEquals("value1", decodedAttributes.get("key1")); assertEquals(123L, decodedAttributes.get("key2")); assertEquals(234.1f, decodedAttributes.get("key3")); @@ -202,7 +192,7 @@ public class VectorTileEncoderTest { } @Test - public void testMultiPolygon() throws IOException { + public void testMultiPolygon() { MultiPolygon mp = newMultiPolygon( (Polygon) newPoint(13, 16).buffer(3), (Polygon) newPoint(24, 25).buffer(5) @@ -212,19 +202,19 @@ public class VectorTileEncoderTest { Map attrs = Map.of("key1", "value1"); VectorTileEncoder vtm = new VectorTileEncoder(); - vtm.addLayerFeatures("mp", List.of(newVectorTileFeature(mp, attrs))); + vtm.addLayerFeatures("mp", List.of(newVectorTileFeature("mp", mp, attrs))); byte[] encoded = vtm.encode(); assertTrue(encoded.length > 0); var features = VectorTileEncoder.decode(encoded); assertEquals(1, features.size()); - MultiPolygon mp2 = (MultiPolygon) features.get(0).geometry(); + MultiPolygon mp2 = (MultiPolygon) features.get(0).geometry().decode(); assertEquals(mp.getNumGeometries(), mp2.getNumGeometries()); } @Test - public void testGeometryCollectionSilentlyIgnored() throws IOException { + public void testGeometryCollectionSilentlyIgnored() { GeometryCollection gc = newGeometryCollection( newPoint(13, 16).buffer(3), newPoint(24, 25) @@ -232,7 +222,7 @@ public class VectorTileEncoderTest { Map attributes = Map.of("key1", "value1"); VectorTileEncoder vtm = new VectorTileEncoder(); - vtm.addLayerFeatures("gc", List.of(newVectorTileFeature(gc, attributes))); + vtm.addLayerFeatures("gc", List.of(newVectorTileFeature("gc", gc, attributes))); byte[] encoded = vtm.encode(); @@ -243,12 +233,12 @@ public class VectorTileEncoderTest { // New tests added: @Test - public void testRoundTripPoint() throws IOException { + public void testRoundTripPoint() { testRoundTripGeometry(gf.createPoint(new CoordinateXY(1, 2))); } @Test - public void testRoundTripMultipoint() throws IOException { + public void testRoundTripMultipoint() { testRoundTripGeometry(gf.createMultiPointFromCoords(new Coordinate[]{ new CoordinateXY(1, 2), new CoordinateXY(3, 4) @@ -256,7 +246,7 @@ public class VectorTileEncoderTest { } @Test - public void testRoundTripLineString() throws IOException { + public void testRoundTripLineString() { testRoundTripGeometry(gf.createLineString(new Coordinate[]{ new CoordinateXY(1, 2), new CoordinateXY(3, 4) @@ -264,7 +254,7 @@ public class VectorTileEncoderTest { } @Test - public void testRoundTripPolygon() throws IOException { + public void testRoundTripPolygon() { testRoundTripGeometry(gf.createPolygon( gf.createLinearRing(new Coordinate[]{ new CoordinateXY(0, 0), @@ -286,7 +276,7 @@ public class VectorTileEncoderTest { } @Test - public void testRoundTripMultiPolygon() throws IOException { + public void testRoundTripMultiPolygon() { testRoundTripGeometry(gf.createMultiPolygon(new Polygon[]{ gf.createPolygon(new Coordinate[]{ new CoordinateXY(0, 0), @@ -306,7 +296,7 @@ public class VectorTileEncoderTest { } @Test - public void testRoundTripAttributes() throws IOException { + public void testRoundTripAttributes() { testRoundTripAttrs(Map.of( "string", "string", "long", 1L, @@ -317,52 +307,53 @@ public class VectorTileEncoderTest { } @Test - public void testMultipleFeaturesMultipleLayer() throws IOException { + public void testMultipleFeaturesMultipleLayer() { Point point = gf.createPoint(new CoordinateXY(0, 0)); Map attrs1 = Map.of("a", 1L, "b", 2L); Map attrs2 = Map.of("b", 3L, "c", 2L); byte[] encoded = new VectorTileEncoder().addLayerFeatures("layer1", List.of( - new LayerFeature(false, 0, 0, attrs1, - (byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 1L), - new LayerFeature(false, 0, 0, attrs2, - (byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 2L) + new VectorTileEncoder.Feature("layer1", 1L, VectorTileEncoder.encodeGeometry(point), attrs1), + new VectorTileEncoder.Feature("layer1", 2L, VectorTileEncoder.encodeGeometry(point), attrs2) )).addLayerFeatures("layer2", List.of( - new LayerFeature(false, 0, 0, attrs1, - (byte) GeomType.POINT.getNumber(), VectorTileEncoder.getCommands(point), 3L) + new VectorTileEncoder.Feature("layer2", 3L, VectorTileEncoder.encodeGeometry(point), attrs1) )).encode(); - List decoded = VectorTileEncoder.decode(encoded); - assertEquals(attrs1, decoded.get(0).attributes()); - assertEquals("layer1", decoded.get(0).layerName()); + List decoded = VectorTileEncoder.decode(encoded); + assertEquals(attrs1, decoded.get(0).attrs()); + assertEquals("layer1", decoded.get(0).layer()); - assertEquals(attrs2, decoded.get(1).attributes()); - assertEquals("layer1", decoded.get(1).layerName()); + assertEquals(attrs2, decoded.get(1).attrs()); + assertEquals("layer1", decoded.get(1).layer()); - assertEquals(attrs1, decoded.get(2).attributes()); - assertEquals("layer2", decoded.get(2).layerName()); + assertEquals(attrs1, decoded.get(2).attrs()); + assertEquals("layer2", decoded.get(2).layer()); } - private void testRoundTripAttrs(Map attrs) throws IOException { + private void testRoundTripAttrs(Map attrs) { testRoundTrip(gf.createPoint(new CoordinateXY(0, 0)), "layer", attrs, 1); } - private void testRoundTripGeometry(Geometry input) throws IOException { + private void testRoundTripGeometry(Geometry input) { testRoundTrip(input, "layer", Map.of(), 1); } - private void testRoundTrip(Geometry input, String layer, Map attrs, long id) throws IOException { - int[] commands = VectorTileEncoder.getCommands(input); - byte geomType = (byte) VectorTileEncoder.toGeomType(input).ordinal(); - Geometry output = VectorTileEncoder.decodeCommands(geomType, commands); + private void testRoundTrip(Geometry input, String layer, Map attrs, long id) { + VectorTileEncoder.VectorGeometry encodedGeom = VectorTileEncoder.encodeGeometry(input); + Geometry output = encodedGeom.decode(); assertTrue(input.equalsExact(output), "\n" + input + "\n!=\n" + output); byte[] encoded = new VectorTileEncoder().addLayerFeatures(layer, List.of( - new LayerFeature(false, 0, 0, attrs, - (byte) VectorTileEncoder.toGeomType(input).getNumber(), VectorTileEncoder.getCommands(input), id) + new VectorTileEncoder.Feature(layer, id, VectorTileEncoder.encodeGeometry(input), attrs) )).encode(); - List decoded = VectorTileEncoder.decode(encoded); - DecodedFeature expected = new DecodedFeature(layer, 4096, input, attrs, id); + List decoded = VectorTileEncoder.decode(encoded); + VectorTileEncoder.Feature expected = new VectorTileEncoder.Feature(layer, id, + VectorTileEncoder.encodeGeometry(input), attrs); assertEquals(List.of(expected), decoded); + assertSameGeometries(List.of(input), decoded); + } + + private void assertSameGeometries(List expected, List actual) { + assertEquals(expected, actual.stream().map(d -> d.geometry().decode()).toList()); } } diff --git a/src/test/java/com/onthegomap/flatmap/WikidataTest.java b/src/test/java/com/onthegomap/flatmap/WikidataTest.java index f86696c7..8d789cc8 100644 --- a/src/test/java/com/onthegomap/flatmap/WikidataTest.java +++ b/src/test/java/com/onthegomap/flatmap/WikidataTest.java @@ -6,8 +6,6 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.graphhopper.reader.ReaderElement; import com.graphhopper.reader.ReaderNode; -import com.onthegomap.flatmap.Wikidata.Client; -import com.onthegomap.flatmap.Wikidata.WikidataTranslations; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringReader; @@ -33,7 +31,7 @@ public class WikidataTest { @Test public void testWikidataTranslations() { var expected = Map.of("en", "en value", "es", "es value"); - WikidataTranslations translations = new WikidataTranslations(); + Wikidata.WikidataTranslations translations = new Wikidata.WikidataTranslations(); assertNull(translations.get(1)); translations.put(1, "en", "en value"); translations.put(1, "es", "es value"); @@ -49,7 +47,7 @@ public class WikidataTest { @TestFactory public List testFetchWikidata() throws IOException, InterruptedException { StringWriter writer = new StringWriter(); - Client client = Mockito.mock(Client.class, Mockito.RETURNS_SMART_NULLS); + Wikidata.Client client = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS); Wikidata fixture = new Wikidata(writer, client, 2); fixture.fetch(1L); Mockito.verifyNoInteractions(client); @@ -118,7 +116,7 @@ public class WikidataTest { }), dynamicTest("do not re-request on subsequent loads", () -> { StringWriter writer2 = new StringWriter(); - Client client2 = Mockito.mock(Client.class, Mockito.RETURNS_SMART_NULLS); + Wikidata.Client client2 = Mockito.mock(Wikidata.Client.class, Mockito.RETURNS_SMART_NULLS); Wikidata fixture2 = new Wikidata(writer2, client2, 2); fixture2.loadExisting(Wikidata.load(new StringReader(writer.toString()))); fixture2.fetch(1L); diff --git a/src/test/java/com/onthegomap/flatmap/collections/MergeSortFeatureMapTest.java b/src/test/java/com/onthegomap/flatmap/collections/FeatureGroupTest.java similarity index 71% rename from src/test/java/com/onthegomap/flatmap/collections/MergeSortFeatureMapTest.java rename to src/test/java/com/onthegomap/flatmap/collections/FeatureGroupTest.java index 1258a667..4d065a65 100644 --- a/src/test/java/com/onthegomap/flatmap/collections/MergeSortFeatureMapTest.java +++ b/src/test/java/com/onthegomap/flatmap/collections/FeatureGroupTest.java @@ -1,13 +1,13 @@ package com.onthegomap.flatmap.collections; import static com.onthegomap.flatmap.TestUtils.newPoint; -import static com.onthegomap.flatmap.TestUtils.newPointWithUserData; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.DynamicTest.dynamicTest; import com.onthegomap.flatmap.Profile; +import com.onthegomap.flatmap.RenderedFeature; import com.onthegomap.flatmap.VectorTileEncoder; import com.onthegomap.flatmap.geo.TileCoord; import java.util.ArrayList; @@ -16,6 +16,7 @@ import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.DynamicTest; @@ -25,10 +26,10 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.locationtech.jts.geom.Geometry; -public class MergeSortFeatureMapTest { +public class FeatureGroupTest { - private final List list = new ArrayList<>(); - private final MergeSort sorter = new MergeSort() { + private final List list = new ArrayList<>(); + private final FeatureSort sorter = new FeatureSort() { @Override public void sort() { list.sort(Comparator.naturalOrder()); @@ -50,7 +51,7 @@ public class MergeSortFeatureMapTest { return list.iterator(); } }; - private MergeSortFeatureMap features = new MergeSortFeatureMap(sorter, new Profile.NullProfile()); + private FeatureGroup features = new FeatureGroup(sorter, new Profile.NullProfile()); @Test public void testEmpty() { @@ -61,27 +62,36 @@ public class MergeSortFeatureMapTest { long id = 0; private void put(int tile, String layer, Map attrs, Geometry geom) { - MergeSort.Entry key = features.encode(id++, TileCoord.decode(tile), layer, attrs, geom, 0, false, 0); - features.accept(key); + putWithZorder(tile, layer, attrs, geom, 0); } private void putWithZorder(int tile, String layer, Map attrs, Geometry geom, int zOrder) { - MergeSort.Entry key = features.encode(id++, TileCoord.decode(tile), layer, attrs, geom, zOrder, false, 0); - features.accept(key); + putWithGroupAndZorder(tile, layer, attrs, geom, zOrder, false, 0, 0); } - private void putWithGroup(int tile, String layer, Map attrs, Geometry geom, int zOrder, int limit) { - MergeSort.Entry key = features.encode(id++, TileCoord.decode(tile), layer, attrs, geom, zOrder, true, limit); - features.accept(key); + private void putWithGroup(int tile, String layer, Map attrs, Geometry geom, int zOrder, long group, + int limit) { + putWithGroupAndZorder(tile, layer, attrs, geom, zOrder, true, group, limit); + } + + private void putWithGroupAndZorder(int tile, String layer, Map attrs, Geometry geom, int zOrder, + boolean hasGroup, long group, int limit) { + RenderedFeature feature = new RenderedFeature( + TileCoord.decode(tile), + new VectorTileEncoder.Feature(layer, id++, VectorTileEncoder.encodeGeometry(geom), attrs), + zOrder, + hasGroup ? Optional.of(new RenderedFeature.Group(group, limit)) : Optional.empty() + ); + features.accept(features.encode(feature)); } private Map>> getFeatures() { Map>> map = new TreeMap<>(); - for (MergeSortFeatureMap.TileFeatures tile : features) { + for (FeatureGroup.TileFeatures tile : features) { for (var feature : VectorTileEncoder.decode(tile.getTile().encode())) { map.computeIfAbsent(tile.coord().encoded(), (i) -> new TreeMap<>()) - .computeIfAbsent(feature.layerName(), l -> new ArrayList<>()) - .add(new Feature(feature.attributes(), feature.geometry())); + .computeIfAbsent(feature.layer(), l -> new ArrayList<>()) + .add(new Feature(feature.attrs(), feature.geometry().decode())); } } return map; @@ -144,13 +154,13 @@ public class MergeSortFeatureMapTest { public void testLimitPoints() { int x = 5, y = 6; putWithGroup( - 1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 1), 0, 2 + 1, "layer", Map.of("id", 3), newPoint(x, y), 0, 1, 2 ); putWithGroup( - 1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 2 + 1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 2 ); putWithGroup( - 1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 2 + 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2 ); sorter.sort(); assertEquals(new TreeMap<>(Map.of( @@ -168,13 +178,13 @@ public class MergeSortFeatureMapTest { public void testLimitPointsInDifferentGroups() { int x = 5, y = 6; putWithGroup( - 1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 2), 0, 2 + 1, "layer", Map.of("id", 3), newPoint(x, y), 0, 2, 2 ); putWithGroup( - 1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 2 + 1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 2 ); putWithGroup( - 1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 2 + 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2 ); sorter.sort(); assertEquals(new TreeMap<>(Map.of( @@ -192,13 +202,13 @@ public class MergeSortFeatureMapTest { public void testDontLimitPointsWithGroup() { int x = 5, y = 6; putWithGroup( - 1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 1), 0, 0 + 1, "layer", Map.of("id", 3), newPoint(x, y), 0, 1, 0 ); putWithGroup( - 1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 0 + 1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 0 ); putWithGroup( - 1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 0 + 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 0 ); sorter.sort(); assertEquals(new TreeMap<>(Map.of( @@ -214,23 +224,23 @@ public class MergeSortFeatureMapTest { @Test public void testProfileChangesGeometry() { - features = new MergeSortFeatureMap(sorter, new Profile.NullProfile() { + features = new FeatureGroup(sorter, new Profile.NullProfile() { @Override - public List postProcessLayerFeatures(String layer, int zoom, - List items) { + public List postProcessLayerFeatures(String layer, int zoom, + List items) { Collections.reverse(items); return items; } }); int x = 5, y = 6; putWithGroup( - 1, "layer", Map.of("id", 3), newPointWithUserData(x, y, 1), 0, 2 + 1, "layer", Map.of("id", 3), newPoint(x, y), 0, 1, 2 ); putWithGroup( - 1, "layer", Map.of("id", 1), newPointWithUserData(1, 2, 1), 2, 2 + 1, "layer", Map.of("id", 1), newPoint(1, 2), 2, 1, 2 ); putWithGroup( - 1, "layer", Map.of("id", 2), newPointWithUserData(3, 4, 1), 1, 2 + 1, "layer", Map.of("id", 2), newPoint(3, 4), 1, 1, 2 ); sorter.sort(); assertEquals(new TreeMap<>(Map.of( @@ -260,14 +270,12 @@ public class MergeSortFeatureMapTest { for (byte layer : layers) { for (int zOrder : zOrders) { for (boolean hasGroup : hasGroups) { - MergeSortFeatureMap.FeatureMapKey key = MergeSortFeatureMap.FeatureMapKey - .of(tile.encoded(), layer, zOrder, hasGroup); - result.add(dynamicTest(key.toString(), () -> { - MergeSortFeatureMap.FeatureMapKey decoded = MergeSortFeatureMap.FeatureMapKey.decode(key.encoded()); - assertEquals(decoded.tile(), tile, "tile"); - assertEquals(decoded.layer(), layer, "layer"); - assertEquals(decoded.zOrder(), zOrder, "zOrder"); - assertEquals(decoded.hasGroup(), hasGroup, "hasGroup"); + long sortKey = FeatureGroup.encodeSortKey(tile.encoded(), layer, zOrder, hasGroup); + result.add(dynamicTest(tile + " " + layer + " " + zOrder + " " + hasGroup, () -> { + assertEquals(tile.encoded(), FeatureGroup.extractTileFromSortKey(sortKey), "tile"); + assertEquals(layer, FeatureGroup.extractLayerIdFromSortKey(sortKey), "layer"); + assertEquals(zOrder, FeatureGroup.extractZorderFromKey(sortKey), "zOrder"); + assertEquals(hasGroup, FeatureGroup.extractHasGroupFromSortKey(sortKey), "hasGroup"); })); } } @@ -292,9 +300,9 @@ public class MergeSortFeatureMapTest { int tileB, byte layerB, int zOrderB, boolean hasGroupB ) { assertTrue( - MergeSortFeatureMap.FeatureMapKey.encode(tileA, layerA, zOrderA, hasGroupA) + FeatureGroup.encodeSortKey(tileA, layerA, zOrderA, hasGroupA) < - MergeSortFeatureMap.FeatureMapKey.encode(tileB, layerB, zOrderB, hasGroupB) + FeatureGroup.encodeSortKey(tileB, layerB, zOrderB, hasGroupB) ); } } diff --git a/src/test/java/com/onthegomap/flatmap/collections/MergeSortTest.java b/src/test/java/com/onthegomap/flatmap/collections/FeatureSortTest.java similarity index 70% rename from src/test/java/com/onthegomap/flatmap/collections/MergeSortTest.java rename to src/test/java/com/onthegomap/flatmap/collections/FeatureSortTest.java index 588dad77..99056c17 100644 --- a/src/test/java/com/onthegomap/flatmap/collections/MergeSortTest.java +++ b/src/test/java/com/onthegomap/flatmap/collections/FeatureSortTest.java @@ -11,29 +11,29 @@ import java.util.Random; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public class MergeSortTest { +public class FeatureSortTest { @TempDir Path tmpDir; - private static MergeSort.Entry newEntry(int i) { - return new MergeSort.Entry(i, new byte[]{(byte) i}); + private static FeatureSort.Entry newEntry(int i) { + return new FeatureSort.Entry(i, new byte[]{(byte) i}); } - private MergeSort newSorter(int workers, int chunkSizeLimit) { - return MergeSort.newExternalMergeSort(tmpDir, workers, chunkSizeLimit, new Stats.InMemory()); + private FeatureSort newSorter(int workers, int chunkSizeLimit) { + return FeatureSort.newExternalMergeSort(tmpDir, workers, chunkSizeLimit, new Stats.InMemory()); } @Test public void testEmpty() { - MergeSort sorter = newSorter(1, 100); + FeatureSort sorter = newSorter(1, 100); sorter.sort(); assertEquals(List.of(), sorter.toList()); } @Test public void testSingle() { - MergeSort sorter = newSorter(1, 100); + FeatureSort sorter = newSorter(1, 100); sorter.add(newEntry(1)); sorter.sort(); assertEquals(List.of(newEntry(1)), sorter.toList()); @@ -41,7 +41,7 @@ public class MergeSortTest { @Test public void testTwoItemsOneChunk() { - MergeSort sorter = newSorter(1, 100); + FeatureSort sorter = newSorter(1, 100); sorter.add(newEntry(2)); sorter.add(newEntry(1)); sorter.sort(); @@ -50,7 +50,7 @@ public class MergeSortTest { @Test public void testTwoItemsTwoChunks() { - MergeSort sorter = newSorter(1, 0); + FeatureSort sorter = newSorter(1, 0); sorter.add(newEntry(2)); sorter.add(newEntry(1)); sorter.sort(); @@ -59,7 +59,7 @@ public class MergeSortTest { @Test public void testTwoWorkers() { - MergeSort sorter = newSorter(2, 0); + FeatureSort sorter = newSorter(2, 0); sorter.add(newEntry(4)); sorter.add(newEntry(3)); sorter.add(newEntry(2)); @@ -70,14 +70,14 @@ public class MergeSortTest { @Test public void testManyItems() { - List sorted = new ArrayList<>(); - List shuffled = new ArrayList<>(); + List sorted = new ArrayList<>(); + List shuffled = new ArrayList<>(); for (int i = 0; i < 10_000; i++) { shuffled.add(newEntry(i)); sorted.add(newEntry(i)); } Collections.shuffle(shuffled, new Random(0)); - MergeSort sorter = newSorter(2, 20_000); + FeatureSort sorter = newSorter(2, 20_000); shuffled.forEach(sorter::add); sorter.sort(); assertEquals(sorted, sorter.toList()); diff --git a/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java b/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java index 5cc017e5..bc1ad2f8 100644 --- a/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java +++ b/src/test/java/com/onthegomap/flatmap/monitoring/ProgressLoggersTest.java @@ -2,7 +2,6 @@ package com.onthegomap.flatmap.monitoring; import static org.junit.jupiter.api.Assertions.assertEquals; -import com.onthegomap.flatmap.monitoring.Stats.InMemory; import com.onthegomap.flatmap.worker.Topology; import java.time.Duration; import java.util.concurrent.CountDownLatch; @@ -15,7 +14,7 @@ public class ProgressLoggersTest { @Timeout(10) public void testLogTopology() { var latch = new CountDownLatch(1); - var topology = Topology.start("topo", new InMemory()) + var topology = Topology.start("topo", new Stats.InMemory()) .fromGenerator("reader", next -> latch.await()) .addBuffer("reader_queue", 10) .addWorker("worker", 2, (a, b) -> latch.await()) diff --git a/src/test/java/com/onthegomap/flatmap/reader/NaturalEarthReaderTest.java b/src/test/java/com/onthegomap/flatmap/reader/NaturalEarthReaderTest.java index 2df5475d..28812991 100644 --- a/src/test/java/com/onthegomap/flatmap/reader/NaturalEarthReaderTest.java +++ b/src/test/java/com/onthegomap/flatmap/reader/NaturalEarthReaderTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.flatmap.geo.GeoUtils; -import com.onthegomap.flatmap.monitoring.Stats.InMemory; +import com.onthegomap.flatmap.monitoring.Stats; import com.onthegomap.flatmap.worker.Topology; import java.io.File; import java.util.ArrayList; @@ -24,12 +24,12 @@ public class NaturalEarthReaderTest { @Timeout(30) public void testReadNaturalEarth(String filename, @TempDir File tempDir) { var file = new File("src/test/resources/" + filename); - try (var reader = new NaturalEarthReader(file, tempDir, new InMemory())) { + try (var reader = new NaturalEarthReader(file, tempDir, new Stats.InMemory())) { for (int i = 1; i <= 2; i++) { assertEquals(19, reader.getCount(), "iter " + i); List points = new ArrayList<>(); - Topology.start("test", new InMemory()) + Topology.start("test", new Stats.InMemory()) .fromGenerator("naturalearth", reader.read()) .addBuffer("reader_queue", 100, 1) .sinkToConsumer("counter", 1, elem -> { diff --git a/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java b/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java index c330e744..977b2e0c 100644 --- a/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java +++ b/src/test/java/com/onthegomap/flatmap/reader/ShapefileReaderTest.java @@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.onthegomap.flatmap.geo.GeoUtils; -import com.onthegomap.flatmap.monitoring.Stats.InMemory; +import com.onthegomap.flatmap.monitoring.Stats; import com.onthegomap.flatmap.worker.Topology; import java.io.File; import java.util.ArrayList; @@ -16,7 +16,8 @@ import org.locationtech.jts.geom.Geometry; public class ShapefileReaderTest { - private ShapefileReader reader = new ShapefileReader(new File("src/test/resources/shapefile.zip"), new InMemory()); + private ShapefileReader reader = new ShapefileReader(new File("src/test/resources/shapefile.zip"), + new Stats.InMemory()); @AfterEach public void close() { @@ -34,7 +35,7 @@ public class ShapefileReaderTest { public void testReadShapefile() { for (int i = 1; i <= 2; i++) { List points = new ArrayList<>(); - Topology.start("test", new InMemory()) + Topology.start("test", new Stats.InMemory()) .fromGenerator("shapefile", reader.read()) .addBuffer("reader_queue", 100, 1) .sinkToConsumer("counter", 1, elem -> {