diff --git a/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java b/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java index 17314df2..ac05c04f 100644 --- a/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java +++ b/core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java @@ -1,10 +1,12 @@ package com.onthegomap.flatmap; +import com.carrotsearch.hppc.IntObjectHashMap; import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongIntHashMap; import com.carrotsearch.hppc.LongLongHashMap; import com.carrotsearch.hppc.LongObjectHashMap; +import com.carrotsearch.hppc.ObjectIntHashMap; public class MemoryEstimator { @@ -35,6 +37,18 @@ public class MemoryEstimator { return object == null ? 0 : (24L + 8L * object.buffer.length); } + public static long size(String string) { + return string == null ? 0 : 54 + string.getBytes().length; + } + + public static long sizeWithoutValues(IntObjectHashMap object) { + return object == null ? 0 : (24L + 4L * object.keys.length + 24L + 8L * object.values.length); + } + + public static long sizeWithoutKeys(ObjectIntHashMap object) { + return object == null ? 0 : (24L + 8L * object.keys.length + 24L + 4L * object.values.length); + } + public interface HasEstimate { long estimateMemoryUsageBytes(); diff --git a/core/src/main/java/com/onthegomap/flatmap/SourceFeature.java b/core/src/main/java/com/onthegomap/flatmap/SourceFeature.java index 87e5a79e..d75f6a87 100644 --- a/core/src/main/java/com/onthegomap/flatmap/SourceFeature.java +++ b/core/src/main/java/com/onthegomap/flatmap/SourceFeature.java @@ -14,11 +14,11 @@ public abstract class SourceFeature { private final Map properties; private final String source; private final String sourceLayer; - private final List relationInfos; + private final List> relationInfos; private final long id; protected SourceFeature(Map properties, String source, String sourceLayer, - List relationInfos, long id) { + List> relationInfos, long id) { this.properties = properties; this.source = source; this.sourceLayer = sourceLayer; @@ -156,15 +156,18 @@ public abstract class SourceFeature { return sourceLayer; } - public List relationInfo(Class relationInfoClass) { - List result = null; + public List> relationInfo( + Class relationInfoClass) { + List> result = null; if (relationInfos != null) { - for (OpenStreetMapReader.RelationInfo info : relationInfos) { - if (relationInfoClass.isInstance(info)) { + for (OpenStreetMapReader.RelationMember info : relationInfos) { + if (relationInfoClass.isInstance(info.relation())) { if (result == null) { result = new ArrayList<>(); } - result.add(relationInfoClass.cast(info)); + @SuppressWarnings("unchecked") + OpenStreetMapReader.RelationMember casted = (OpenStreetMapReader.RelationMember) info; + result.add(casted); } } } diff --git a/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java b/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java index fb20cb45..554c67c4 100644 --- a/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java +++ b/core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java @@ -1,10 +1,14 @@ package com.onthegomap.flatmap.read; +import com.carrotsearch.hppc.IntObjectHashMap; import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.LongDoubleHashMap; import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.ObjectIntHashMap; +import com.graphhopper.coll.GHIntObjectHashMap; import com.graphhopper.coll.GHLongHashSet; import com.graphhopper.coll.GHLongObjectHashMap; +import com.graphhopper.coll.GHObjectIntHashMap; import com.graphhopper.reader.ReaderElement; import com.graphhopper.reader.ReaderElementUtils; import com.graphhopper.reader.ReaderNode; @@ -124,7 +128,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima relationInfoSizes.addAndGet(info.estimateMemoryUsageBytes()); for (ReaderRelation.Member member : rel.getMembers()) { if (member.getType() == ReaderRelation.Member.WAY) { - wayToRelations.put(member.getRef(), rel.getId()); + wayToRelations.put(member.getRef(), new RelationMembership(member.getRole(), rel.getId()).encode()); } } } @@ -233,13 +237,15 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima boolean closed = nodes.size() > 1 && nodes.get(0) == nodes.get(nodes.size() - 1); String area = way.getTag("area"); LongArrayList relationIds = wayToRelations.get(way.getId()); - List rels = null; + List> rels = null; if (!relationIds.isEmpty()) { rels = new ArrayList<>(relationIds.size()); for (int r = 0; r < relationIds.size(); r++) { - RelationInfo rel = relationInfo.get(relationIds.get(r)); + long encoded = relationIds.get(r); + RelationMembership parsed = RelationMembership.parse(encoded); + RelationInfo rel = relationInfo.get(parsed.relationId); if (rel != null) { - rels.add(rel); + rels.add(new RelationMember<>(parsed.role, rel)); } } } @@ -257,6 +263,9 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima size += MemoryEstimator.size(multipolygonWayGeometries); size += MemoryEstimator.size(wayToRelations); size += MemoryEstimator.sizeWithoutValues(relationInfo); + size += MemoryEstimator.sizeWithoutValues(roleIdsReverse); + size += MemoryEstimator.sizeWithoutKeys(roleIds); + size += roleSizes.get(); size += relationInfoSizes.get(); return size; } @@ -268,11 +277,48 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima wayToRelations = null; waysInMultipolygon = null; relationInfo = null; + roleIds.release(); + roleIdsReverse.release(); nodeDb.close(); } + public static record RelationMember(String role, T relation) {} + + private static final ObjectIntHashMap roleIds = new GHObjectIntHashMap<>(); + private static final IntObjectHashMap roleIdsReverse = new GHIntObjectHashMap<>(); + private static final AtomicLong roleSizes = new AtomicLong(0); + private static final int ROLE_BITS = 16; + private static final int MAX_ROLES = (1 << ROLE_BITS) - 10; + private static final int ROLE_SHIFT = 64 - ROLE_BITS; + private static final int ROLE_MASK = (1 << ROLE_BITS) - 1; + private static final long NOT_ROLE_MASK = (1L << ROLE_SHIFT) - 1L; + + private record RelationMembership(String role, long relationId) { + + public static RelationMembership parse(long encoded) { + int role = (int) ((encoded >>> ROLE_SHIFT) & ROLE_MASK); + return new RelationMembership(roleIdsReverse.get(role), encoded & NOT_ROLE_MASK); + } + + public long encode() { + int roleId = roleIds.getOrDefault(role, -1); + if (roleId == -1) { + roleSizes.addAndGet(MemoryEstimator.size(role)); + roleId = roleIds.size() + 1; + roleIds.put(role, roleId); + roleIdsReverse.put(roleId, role); + if (roleId > MAX_ROLES) { + throw new IllegalStateException("Too many roles to encode: " + role); + } + } + return relationId | ((long) roleId << ROLE_SHIFT); + } + } + public interface RelationInfo extends MemoryEstimator.HasEstimate { + long id(); + @Override default long estimateMemoryUsageBytes() { return 0; @@ -286,7 +332,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima final boolean point; public ProxyFeature(ReaderElement elem, boolean point, boolean line, boolean polygon, - List relationInfo) { + List> relationInfo) { super(ReaderElementUtils.getProperties(elem), name, null, relationInfo, elem.getId()); this.point = point; this.line = line; @@ -361,7 +407,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima private final LongArrayList nodeIds; public WaySourceFeature(ReaderWay way, boolean closed, String area, NodeLocationProvider nodeCache, - List relationInfo) { + List> relationInfo) { super(way, false, (!closed || !"yes".equals(area)) && way.getNodes().size() >= 2, (closed && !"no".equals(area)) && way.getNodes().size() >= 4, diff --git a/core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java b/core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java index e6ae3ece..d95f1bf5 100644 --- a/core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java +++ b/core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java @@ -836,7 +836,7 @@ public class FlatMapTest { @Test public void testOsmLineInRelation() throws Exception { - record TestRelationInfo(String name) implements OpenStreetMapReader.RelationInfo {} + record TestRelationInfo(long id, String name) implements OpenStreetMapReader.RelationInfo {} var results = runWithOsmElements( Map.of("threads", "1"), List.of( @@ -859,18 +859,18 @@ public class FlatMapTest { ), (relation) -> { if (relation.hasTag("name", "relation name")) { - return List.of(new TestRelationInfo(relation.getTag("name"))); + return List.of(new TestRelationInfo(relation.getId(), relation.getTag("name"))); } return null; }, (in, features) -> { - List relationInfos = in.relationInfo(TestRelationInfo.class); + var relationInfos = in.relationInfo(TestRelationInfo.class); + var firstRelation = relationInfos.stream().findFirst(); if (in.canBeLine()) { features.line("layer") .setZoomRange(0, 0) - .setAttr("relname", relationInfos.stream() - .findFirst().map(TestRelationInfo::name) - .orElse(null)) - .inheritFromSource("attr"); + .setAttr("relname", firstRelation.map(d -> d.relation().name).orElse(null)) + .inheritFromSource("attr") + .setAttr("relrole", firstRelation.map(OpenStreetMapReader.RelationMember::role).orElse(null)); } } ); @@ -882,7 +882,8 @@ public class FlatMapTest { )), feature(newLineString(128, 0.25 * 256, 128, 0.125 * 256), Map.of( "attr", "value2", - "relname", "relation name" + "relname", "relation name", + "relrole", "role" )) ) )), sortListValues(results.tiles)); diff --git a/core/src/test/java/com/onthegomap/flatmap/read/OpenStreetMapReaderTest.java b/core/src/test/java/com/onthegomap/flatmap/read/OpenStreetMapReaderTest.java index 06573f8f..eb49502c 100644 --- a/core/src/test/java/com/onthegomap/flatmap/read/OpenStreetMapReaderTest.java +++ b/core/src/test/java/com/onthegomap/flatmap/read/OpenStreetMapReaderTest.java @@ -645,21 +645,15 @@ public class OpenStreetMapReaderTest { @Test public void testWayInRelation() { - record OtherRelInfo() implements OpenStreetMapReader.RelationInfo {} - record TestRelInfo(String name) implements OpenStreetMapReader.RelationInfo { - - @Override - public long estimateMemoryUsageBytes() { - return 10 + name.length(); - } - } + record OtherRelInfo(long id) implements OpenStreetMapReader.RelationInfo {} + record TestRelInfo(long id, String name) implements OpenStreetMapReader.RelationInfo {} OpenStreetMapReader reader = new OpenStreetMapReader( osmSource, longLongMap, new Profile.NullProfile() { @Override public List preprocessOsmRelation(ReaderRelation relation) { - return List.of(new TestRelInfo("name")); + return List.of(new TestRelInfo(1, "name")); } }, stats @@ -671,7 +665,7 @@ public class OpenStreetMapReaderTest { way.getNodes().add(node1.getId(), node2.getId()); way.setTag("key", "value"); var relation = new ReaderRelation(4); - relation.add(new ReaderRelation.Member(ReaderRelation.Member.WAY, 3, "")); + relation.add(new ReaderRelation.Member(ReaderRelation.Member.WAY, 3, "rolename")); reader.processPass1(node1); reader.processPass1(node2); @@ -681,7 +675,8 @@ public class OpenStreetMapReaderTest { SourceFeature feature = reader.processWayPass2(nodeCache, way); assertEquals(List.of(), feature.relationInfo(OtherRelInfo.class)); - assertEquals(List.of(new TestRelInfo("name")), feature.relationInfo(TestRelInfo.class)); + assertEquals(List.of(new OpenStreetMapReader.RelationMember<>("rolename", new TestRelInfo(1, "name"))), + feature.relationInfo(TestRelInfo.class)); } private OpenStreetMapReader newOsmReader() { diff --git a/examples/src/main/java/com/onthegomap/flatmap/examples/BikeRouteOverlay.java b/examples/src/main/java/com/onthegomap/flatmap/examples/BikeRouteOverlay.java index be3a2055..30c5a5b8 100644 --- a/examples/src/main/java/com/onthegomap/flatmap/examples/BikeRouteOverlay.java +++ b/examples/src/main/java/com/onthegomap/flatmap/examples/BikeRouteOverlay.java @@ -14,8 +14,8 @@ import java.util.List; public class BikeRouteOverlay implements Profile { - private static record RouteRelationInfo(String name, String ref, String route, String network) implements - OpenStreetMapReader.RelationInfo {} + private static record RouteRelationInfo(long id, String name, String ref, String route, String network) + implements OpenStreetMapReader.RelationInfo {} @Override public List preprocessOsmRelation(ReaderRelation relation) { @@ -23,6 +23,7 @@ public class BikeRouteOverlay implements Profile { String type = relation.getTag("route"); if ("mtb".equals(type) || "bicycle".equals(type)) { return List.of(new RouteRelationInfo( + relation.getId(), relation.getTag("name"), relation.getTag("ref"), type, @@ -42,10 +43,11 @@ public class BikeRouteOverlay implements Profile { @Override public void processFeature(SourceFeature sourceFeature, FeatureCollector features) { if (sourceFeature.canBeLine()) { - for (RouteRelationInfo routeInfo : sourceFeature.relationInfo(RouteRelationInfo.class)) { - features.line(routeInfo.route + "-route-" + routeInfo.network) - .setAttr("name", routeInfo.name) - .setAttr("ref", routeInfo.ref) + for (var routeInfo : sourceFeature.relationInfo(RouteRelationInfo.class)) { + RouteRelationInfo relation = routeInfo.relation(); + features.line(relation.route + "-route-" + relation.network) + .setAttr("name", relation.name) + .setAttr("ref", relation.ref) .setZoomRange(0, 14) .setMinPixelSize(0); } diff --git a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/Generate.java b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/Generate.java index ec4799c0..a50ec4f3 100644 --- a/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/Generate.java +++ b/openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/Generate.java @@ -160,9 +160,6 @@ public class Generate { emitLayerDefinitions(config.tileset, layers, packageName, output); emitTableDefinitions(tables, packageName, output); - -// layers.forEach((layer) -> LOGGER.info("layer: " + layer.layer.id)); -// tables.forEach((key, val) -> LOGGER.info("table: " + key)); } private static void emitTableDefinitions(Map tables, String packageName, Path output)