thread relation member roles through

core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java

@@ -1,10 +1,12 @@
 package com.onthegomap.flatmap;
 
+import com.carrotsearch.hppc.IntObjectHashMap;
 import com.carrotsearch.hppc.LongArrayList;
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongIntHashMap;
 import com.carrotsearch.hppc.LongLongHashMap;
 import com.carrotsearch.hppc.LongObjectHashMap;
+import com.carrotsearch.hppc.ObjectIntHashMap;
 
 public class MemoryEstimator {
 
@@ -35,6 +37,18 @@ public class MemoryEstimator {
     return object == null ? 0 : (24L + 8L * object.buffer.length);
   }
 
+  public static long size(String string) {
+    return string == null ? 0 : 54 + string.getBytes().length;
+  }
+
+  public static long sizeWithoutValues(IntObjectHashMap<?> object) {
+    return object == null ? 0 : (24L + 4L * object.keys.length + 24L + 8L * object.values.length);
+  }
+
+  public static long sizeWithoutKeys(ObjectIntHashMap<?> object) {
+    return object == null ? 0 : (24L + 8L * object.keys.length + 24L + 4L * object.values.length);
+  }
+
   public interface HasEstimate {
 
     long estimateMemoryUsageBytes();

core/src/main/java/com/onthegomap/flatmap/SourceFeature.java

@@ -14,11 +14,11 @@ public abstract class SourceFeature {
   private final Map<String, Object> properties;
   private final String source;
   private final String sourceLayer;
-  private final List<OpenStreetMapReader.RelationInfo> relationInfos;
+  private final List<OpenStreetMapReader.RelationMember<?>> relationInfos;
   private final long id;
 
   protected SourceFeature(Map<String, Object> properties, String source, String sourceLayer,
-    List<OpenStreetMapReader.RelationInfo> relationInfos, long id) {
+    List<OpenStreetMapReader.RelationMember<?>> relationInfos, long id) {
     this.properties = properties;
     this.source = source;
     this.sourceLayer = sourceLayer;
@@ -156,15 +156,18 @@ public abstract class SourceFeature {
     return sourceLayer;
   }
 
-  public <T extends OpenStreetMapReader.RelationInfo> List<T> relationInfo(Class<T> relationInfoClass) {
+  public <T extends OpenStreetMapReader.RelationInfo> List<OpenStreetMapReader.RelationMember<T>> relationInfo(
-    List<T> result = null;
+    Class<T> relationInfoClass) {
+    List<OpenStreetMapReader.RelationMember<T>> result = null;
     if (relationInfos != null) {
-      for (OpenStreetMapReader.RelationInfo info : relationInfos) {
+      for (OpenStreetMapReader.RelationMember<?> info : relationInfos) {
-        if (relationInfoClass.isInstance(info)) {
+        if (relationInfoClass.isInstance(info.relation())) {
           if (result == null) {
             result = new ArrayList<>();
           }
-          result.add(relationInfoClass.cast(info));
+          @SuppressWarnings("unchecked")
+          OpenStreetMapReader.RelationMember<T> casted = (OpenStreetMapReader.RelationMember<T>) info;
+          result.add(casted);
         }
       }
     }

core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java

@@ -1,10 +1,14 @@
 package com.onthegomap.flatmap.read;
 
+import com.carrotsearch.hppc.IntObjectHashMap;
 import com.carrotsearch.hppc.LongArrayList;
 import com.carrotsearch.hppc.LongDoubleHashMap;
 import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.ObjectIntHashMap;
+import com.graphhopper.coll.GHIntObjectHashMap;
 import com.graphhopper.coll.GHLongHashSet;
 import com.graphhopper.coll.GHLongObjectHashMap;
+import com.graphhopper.coll.GHObjectIntHashMap;
 import com.graphhopper.reader.ReaderElement;
 import com.graphhopper.reader.ReaderElementUtils;
 import com.graphhopper.reader.ReaderNode;
@@ -124,7 +128,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
           relationInfoSizes.addAndGet(info.estimateMemoryUsageBytes());
           for (ReaderRelation.Member member : rel.getMembers()) {
             if (member.getType() == ReaderRelation.Member.WAY) {
-              wayToRelations.put(member.getRef(), rel.getId());
+              wayToRelations.put(member.getRef(), new RelationMembership(member.getRole(), rel.getId()).encode());
             }
           }
         }
@@ -233,13 +237,15 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
     boolean closed = nodes.size() > 1 && nodes.get(0) == nodes.get(nodes.size() - 1);
     String area = way.getTag("area");
     LongArrayList relationIds = wayToRelations.get(way.getId());
-    List<RelationInfo> rels = null;
+    List<RelationMember<?>> rels = null;
     if (!relationIds.isEmpty()) {
       rels = new ArrayList<>(relationIds.size());
       for (int r = 0; r < relationIds.size(); r++) {
-        RelationInfo rel = relationInfo.get(relationIds.get(r));
+        long encoded = relationIds.get(r);
+        RelationMembership parsed = RelationMembership.parse(encoded);
+        RelationInfo rel = relationInfo.get(parsed.relationId);
         if (rel != null) {
-          rels.add(rel);
+          rels.add(new RelationMember<>(parsed.role, rel));
         }
       }
     }
@@ -257,6 +263,9 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
     size += MemoryEstimator.size(multipolygonWayGeometries);
     size += MemoryEstimator.size(wayToRelations);
     size += MemoryEstimator.sizeWithoutValues(relationInfo);
+    size += MemoryEstimator.sizeWithoutValues(roleIdsReverse);
+    size += MemoryEstimator.sizeWithoutKeys(roleIds);
+    size += roleSizes.get();
     size += relationInfoSizes.get();
     return size;
   }
@@ -268,11 +277,48 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
     wayToRelations = null;
     waysInMultipolygon = null;
     relationInfo = null;
+    roleIds.release();
+    roleIdsReverse.release();
     nodeDb.close();
   }
 
+  public static record RelationMember<T extends RelationInfo>(String role, T relation) {}
+
+  private static final ObjectIntHashMap<String> roleIds = new GHObjectIntHashMap<>();
+  private static final IntObjectHashMap<String> roleIdsReverse = new GHIntObjectHashMap<>();
+  private static final AtomicLong roleSizes = new AtomicLong(0);
+  private static final int ROLE_BITS = 16;
+  private static final int MAX_ROLES = (1 << ROLE_BITS) - 10;
+  private static final int ROLE_SHIFT = 64 - ROLE_BITS;
+  private static final int ROLE_MASK = (1 << ROLE_BITS) - 1;
+  private static final long NOT_ROLE_MASK = (1L << ROLE_SHIFT) - 1L;
+
+  private record RelationMembership(String role, long relationId) {
+
+    public static RelationMembership parse(long encoded) {
+      int role = (int) ((encoded >>> ROLE_SHIFT) & ROLE_MASK);
+      return new RelationMembership(roleIdsReverse.get(role), encoded & NOT_ROLE_MASK);
+    }
+
+    public long encode() {
+      int roleId = roleIds.getOrDefault(role, -1);
+      if (roleId == -1) {
+        roleSizes.addAndGet(MemoryEstimator.size(role));
+        roleId = roleIds.size() + 1;
+        roleIds.put(role, roleId);
+        roleIdsReverse.put(roleId, role);
+        if (roleId > MAX_ROLES) {
+          throw new IllegalStateException("Too many roles to encode: " + role);
+        }
+      }
+      return relationId | ((long) roleId << ROLE_SHIFT);
+    }
+  }
+
   public interface RelationInfo extends MemoryEstimator.HasEstimate {
 
+    long id();
+
     @Override
     default long estimateMemoryUsageBytes() {
       return 0;
@@ -286,7 +332,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
     final boolean point;
 
     public ProxyFeature(ReaderElement elem, boolean point, boolean line, boolean polygon,
-      List<RelationInfo> relationInfo) {
+      List<RelationMember<?>> relationInfo) {
       super(ReaderElementUtils.getProperties(elem), name, null, relationInfo, elem.getId());
       this.point = point;
       this.line = line;
@@ -361,7 +407,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
     private final LongArrayList nodeIds;
 
     public WaySourceFeature(ReaderWay way, boolean closed, String area, NodeLocationProvider nodeCache,
-      List<RelationInfo> relationInfo) {
+      List<RelationMember<?>> relationInfo) {
       super(way, false,
         (!closed || !"yes".equals(area)) && way.getNodes().size() >= 2,
         (closed && !"no".equals(area)) && way.getNodes().size() >= 4,

core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java

@@ -836,7 +836,7 @@ public class FlatMapTest {
 
   @Test
   public void testOsmLineInRelation() throws Exception {
-    record TestRelationInfo(String name) implements OpenStreetMapReader.RelationInfo {}
+    record TestRelationInfo(long id, String name) implements OpenStreetMapReader.RelationInfo {}
     var results = runWithOsmElements(
       Map.of("threads", "1"),
       List.of(
@@ -859,18 +859,18 @@ public class FlatMapTest {
       ),
       (relation) -> {
         if (relation.hasTag("name", "relation name")) {
-          return List.of(new TestRelationInfo(relation.getTag("name")));
+          return List.of(new TestRelationInfo(relation.getId(), relation.getTag("name")));
         }
         return null;
       }, (in, features) -> {
-        List<TestRelationInfo> relationInfos = in.relationInfo(TestRelationInfo.class);
+        var relationInfos = in.relationInfo(TestRelationInfo.class);
+        var firstRelation = relationInfos.stream().findFirst();
         if (in.canBeLine()) {
           features.line("layer")
             .setZoomRange(0, 0)
-            .setAttr("relname", relationInfos.stream()
+            .setAttr("relname", firstRelation.map(d -> d.relation().name).orElse(null))
-              .findFirst().map(TestRelationInfo::name)
+            .inheritFromSource("attr")
-              .orElse(null))
+            .setAttr("relrole", firstRelation.map(OpenStreetMapReader.RelationMember::role).orElse(null));
-            .inheritFromSource("attr");
         }
       }
     );
@@ -882,7 +882,8 @@ public class FlatMapTest {
         )),
         feature(newLineString(128, 0.25 * 256, 128, 0.125 * 256), Map.of(
           "attr", "value2",
-          "relname", "relation name"
+          "relname", "relation name",
+          "relrole", "role"
         ))
       )
     )), sortListValues(results.tiles));

core/src/test/java/com/onthegomap/flatmap/read/OpenStreetMapReaderTest.java

@@ -645,21 +645,15 @@ public class OpenStreetMapReaderTest {
 
   @Test
   public void testWayInRelation() {
-    record OtherRelInfo() implements OpenStreetMapReader.RelationInfo {}
+    record OtherRelInfo(long id) implements OpenStreetMapReader.RelationInfo {}
-    record TestRelInfo(String name) implements OpenStreetMapReader.RelationInfo {
+    record TestRelInfo(long id, String name) implements OpenStreetMapReader.RelationInfo {}
-
-      @Override
-      public long estimateMemoryUsageBytes() {
-        return 10 + name.length();
-      }
-    }
     OpenStreetMapReader reader = new OpenStreetMapReader(
       osmSource,
       longLongMap,
       new Profile.NullProfile() {
         @Override
         public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
-          return List.of(new TestRelInfo("name"));
+          return List.of(new TestRelInfo(1, "name"));
         }
       },
       stats
@@ -671,7 +665,7 @@ public class OpenStreetMapReaderTest {
     way.getNodes().add(node1.getId(), node2.getId());
     way.setTag("key", "value");
     var relation = new ReaderRelation(4);
-    relation.add(new ReaderRelation.Member(ReaderRelation.Member.WAY, 3, ""));
+    relation.add(new ReaderRelation.Member(ReaderRelation.Member.WAY, 3, "rolename"));
 
     reader.processPass1(node1);
     reader.processPass1(node2);
@@ -681,7 +675,8 @@ public class OpenStreetMapReaderTest {
     SourceFeature feature = reader.processWayPass2(nodeCache, way);
 
     assertEquals(List.of(), feature.relationInfo(OtherRelInfo.class));
-    assertEquals(List.of(new TestRelInfo("name")), feature.relationInfo(TestRelInfo.class));
+    assertEquals(List.of(new OpenStreetMapReader.RelationMember<>("rolename", new TestRelInfo(1, "name"))),
+      feature.relationInfo(TestRelInfo.class));
   }
 
   private OpenStreetMapReader newOsmReader() {

examples/src/main/java/com/onthegomap/flatmap/examples/BikeRouteOverlay.java

@@ -14,8 +14,8 @@ import java.util.List;
 
 public class BikeRouteOverlay implements Profile {
 
-  private static record RouteRelationInfo(String name, String ref, String route, String network) implements
+  private static record RouteRelationInfo(long id, String name, String ref, String route, String network)
-    OpenStreetMapReader.RelationInfo {}
+    implements OpenStreetMapReader.RelationInfo {}
 
   @Override
   public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
@@ -23,6 +23,7 @@ public class BikeRouteOverlay implements Profile {
       String type = relation.getTag("route");
       if ("mtb".equals(type) || "bicycle".equals(type)) {
         return List.of(new RouteRelationInfo(
+          relation.getId(),
           relation.getTag("name"),
           relation.getTag("ref"),
           type,
@@ -42,10 +43,11 @@ public class BikeRouteOverlay implements Profile {
   @Override
   public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
     if (sourceFeature.canBeLine()) {
-      for (RouteRelationInfo routeInfo : sourceFeature.relationInfo(RouteRelationInfo.class)) {
+      for (var routeInfo : sourceFeature.relationInfo(RouteRelationInfo.class)) {
-        features.line(routeInfo.route + "-route-" + routeInfo.network)
+        RouteRelationInfo relation = routeInfo.relation();
-          .setAttr("name", routeInfo.name)
+        features.line(relation.route + "-route-" + relation.network)
-          .setAttr("ref", routeInfo.ref)
+          .setAttr("name", relation.name)
+          .setAttr("ref", relation.ref)
           .setZoomRange(0, 14)
           .setMinPixelSize(0);
       }

openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/Generate.java

@@ -160,9 +160,6 @@ public class Generate {
 
     emitLayerDefinitions(config.tileset, layers, packageName, output);
     emitTableDefinitions(tables, packageName, output);
-
-//    layers.forEach((layer) -> LOGGER.info("layer: " + layer.layer.id));
-//    tables.forEach((key, val) -> LOGGER.info("table: " + key));
   }
 
   private static void emitTableDefinitions(Map<String, Imposm3Table> tables, String packageName, Path output)