mirror
/
planetiler
kopia lustrzana https://github.com/onthegomap/planetiler
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

thread relation member roles through

pull/1/head
Mike Barry 2021-06-18 07:21:43 -04:00
rodzic 1ad28676a6
commit 32d109f52a
7 zmienionych plików z 99 dodań i 41 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

14
core/src/main/java/com/onthegomap/flatmap/MemoryEstimator.java
Wyświetl plik

@ -1,10 +1,12 @@
package com.onthegomap.flatmap;
import com.carrotsearch.hppc.IntObjectHashMap;
import com.carrotsearch.hppc.LongArrayList;
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongIntHashMap;
import com.carrotsearch.hppc.LongLongHashMap;
import com.carrotsearch.hppc.LongObjectHashMap;
import com.carrotsearch.hppc.ObjectIntHashMap;
public class MemoryEstimator {
@ -35,6 +37,18 @@ public class MemoryEstimator {
return object == null ? 0 : (24L + 8L * object.buffer.length);
}
public static long size(String string) {
return string == null ? 0 : 54 + string.getBytes().length;
}
public static long sizeWithoutValues(IntObjectHashMap<?> object) {
return object == null ? 0 : (24L + 4L * object.keys.length + 24L + 8L * object.values.length);
}
public static long sizeWithoutKeys(ObjectIntHashMap<?> object) {
return object == null ? 0 : (24L + 8L * object.keys.length + 24L + 4L * object.values.length);
}
public interface HasEstimate {
long estimateMemoryUsageBytes();

17
core/src/main/java/com/onthegomap/flatmap/SourceFeature.java
Wyświetl plik

@ -14,11 +14,11 @@ public abstract class SourceFeature {
private final Map<String, Object> properties;
private final String source;
private final String sourceLayer;
private final List<OpenStreetMapReader.RelationInfo> relationInfos;
private final List<OpenStreetMapReader.RelationMember<?>> relationInfos;
private final long id;
protected SourceFeature(Map<String, Object> properties, String source, String sourceLayer,
List<OpenStreetMapReader.RelationInfo> relationInfos, long id) {
List<OpenStreetMapReader.RelationMember<?>> relationInfos, long id) {
this.properties = properties;
this.source = source;
this.sourceLayer = sourceLayer;
@ -156,15 +156,18 @@ public abstract class SourceFeature {
return sourceLayer;
}
public <T extends OpenStreetMapReader.RelationInfo> List<T> relationInfo(Class<T> relationInfoClass) {
List<T> result = null;
public <T extends OpenStreetMapReader.RelationInfo> List<OpenStreetMapReader.RelationMember<T>> relationInfo(
Class<T> relationInfoClass) {
List<OpenStreetMapReader.RelationMember<T>> result = null;
if (relationInfos != null) {
for (OpenStreetMapReader.RelationInfo info : relationInfos) {
if (relationInfoClass.isInstance(info)) {
for (OpenStreetMapReader.RelationMember<?> info : relationInfos) {
if (relationInfoClass.isInstance(info.relation())) {
if (result == null) {
result = new ArrayList<>();
}
result.add(relationInfoClass.cast(info));
@SuppressWarnings("unchecked")
OpenStreetMapReader.RelationMember<T> casted = (OpenStreetMapReader.RelationMember<T>) info;
result.add(casted);
}
}
}

58
core/src/main/java/com/onthegomap/flatmap/read/OpenStreetMapReader.java
Wyświetl plik

@ -1,10 +1,14 @@
package com.onthegomap.flatmap.read;
import com.carrotsearch.hppc.IntObjectHashMap;
import com.carrotsearch.hppc.LongArrayList;
import com.carrotsearch.hppc.LongDoubleHashMap;
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.ObjectIntHashMap;
import com.graphhopper.coll.GHIntObjectHashMap;
import com.graphhopper.coll.GHLongHashSet;
import com.graphhopper.coll.GHLongObjectHashMap;
import com.graphhopper.coll.GHObjectIntHashMap;
import com.graphhopper.reader.ReaderElement;
import com.graphhopper.reader.ReaderElementUtils;
import com.graphhopper.reader.ReaderNode;
@ -124,7 +128,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
relationInfoSizes.addAndGet(info.estimateMemoryUsageBytes());
for (ReaderRelation.Member member : rel.getMembers()) {
if (member.getType() == ReaderRelation.Member.WAY) {
wayToRelations.put(member.getRef(), rel.getId());
wayToRelations.put(member.getRef(), new RelationMembership(member.getRole(), rel.getId()).encode());
}
}
}
@ -233,13 +237,15 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
boolean closed = nodes.size() > 1 && nodes.get(0) == nodes.get(nodes.size() - 1);
String area = way.getTag("area");
LongArrayList relationIds = wayToRelations.get(way.getId());
List<RelationInfo> rels = null;
List<RelationMember<?>> rels = null;
if (!relationIds.isEmpty()) {
rels = new ArrayList<>(relationIds.size());
for (int r = 0; r < relationIds.size(); r++) {
RelationInfo rel = relationInfo.get(relationIds.get(r));
long encoded = relationIds.get(r);
RelationMembership parsed = RelationMembership.parse(encoded);
RelationInfo rel = relationInfo.get(parsed.relationId);
if (rel != null) {
rels.add(rel);
rels.add(new RelationMember<>(parsed.role, rel));
}
}
}
@ -257,6 +263,9 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
size += MemoryEstimator.size(multipolygonWayGeometries);
size += MemoryEstimator.size(wayToRelations);
size += MemoryEstimator.sizeWithoutValues(relationInfo);
size += MemoryEstimator.sizeWithoutValues(roleIdsReverse);
size += MemoryEstimator.sizeWithoutKeys(roleIds);
size += roleSizes.get();
size += relationInfoSizes.get();
return size;
}
@ -268,11 +277,48 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
wayToRelations = null;
waysInMultipolygon = null;
relationInfo = null;
roleIds.release();
roleIdsReverse.release();
nodeDb.close();
}
public static record RelationMember<T extends RelationInfo>(String role, T relation) {}
private static final ObjectIntHashMap<String> roleIds = new GHObjectIntHashMap<>();
private static final IntObjectHashMap<String> roleIdsReverse = new GHIntObjectHashMap<>();
private static final AtomicLong roleSizes = new AtomicLong(0);
private static final int ROLE_BITS = 16;
private static final int MAX_ROLES = (1 << ROLE_BITS) - 10;
private static final int ROLE_SHIFT = 64 - ROLE_BITS;
private static final int ROLE_MASK = (1 << ROLE_BITS) - 1;
private static final long NOT_ROLE_MASK = (1L << ROLE_SHIFT) - 1L;
private record RelationMembership(String role, long relationId) {
public static RelationMembership parse(long encoded) {
int role = (int) ((encoded >>> ROLE_SHIFT) & ROLE_MASK);
return new RelationMembership(roleIdsReverse.get(role), encoded & NOT_ROLE_MASK);
}
public long encode() {
int roleId = roleIds.getOrDefault(role, -1);
if (roleId == -1) {
roleSizes.addAndGet(MemoryEstimator.size(role));
roleId = roleIds.size() + 1;
roleIds.put(role, roleId);
roleIdsReverse.put(roleId, role);
if (roleId > MAX_ROLES) {
throw new IllegalStateException("Too many roles to encode: " + role);
}
}
return relationId | ((long) roleId << ROLE_SHIFT);
}
}
public interface RelationInfo extends MemoryEstimator.HasEstimate {
long id();
@Override
default long estimateMemoryUsageBytes() {
return 0;
@ -286,7 +332,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
final boolean point;
public ProxyFeature(ReaderElement elem, boolean point, boolean line, boolean polygon,
List<RelationInfo> relationInfo) {
List<RelationMember<?>> relationInfo) {
super(ReaderElementUtils.getProperties(elem), name, null, relationInfo, elem.getId());
this.point = point;
this.line = line;
@ -361,7 +407,7 @@ public class OpenStreetMapReader implements Closeable, MemoryEstimator.HasEstima
private final LongArrayList nodeIds;
public WaySourceFeature(ReaderWay way, boolean closed, String area, NodeLocationProvider nodeCache,
List<RelationInfo> relationInfo) {
List<RelationMember<?>> relationInfo) {
super(way, false,
(!closed || !"yes".equals(area)) && way.getNodes().size() >= 2,
(closed && !"no".equals(area)) && way.getNodes().size() >= 4,

17
core/src/test/java/com/onthegomap/flatmap/FlatMapTest.java
Wyświetl plik

@ -836,7 +836,7 @@ public class FlatMapTest {
@Test
public void testOsmLineInRelation() throws Exception {
record TestRelationInfo(String name) implements OpenStreetMapReader.RelationInfo {}
record TestRelationInfo(long id, String name) implements OpenStreetMapReader.RelationInfo {}
var results = runWithOsmElements(
Map.of("threads", "1"),
List.of(
@ -859,18 +859,18 @@ public class FlatMapTest {
),
(relation) -> {
if (relation.hasTag("name", "relation name")) {
return List.of(new TestRelationInfo(relation.getTag("name")));
return List.of(new TestRelationInfo(relation.getId(), relation.getTag("name")));
}
return null;
}, (in, features) -> {
List<TestRelationInfo> relationInfos = in.relationInfo(TestRelationInfo.class);
var relationInfos = in.relationInfo(TestRelationInfo.class);
var firstRelation = relationInfos.stream().findFirst();
if (in.canBeLine()) {
features.line("layer")
.setZoomRange(0, 0)
.setAttr("relname", relationInfos.stream()
.findFirst().map(TestRelationInfo::name)
.orElse(null))
.inheritFromSource("attr");
.setAttr("relname", firstRelation.map(d -> d.relation().name).orElse(null))
.inheritFromSource("attr")
.setAttr("relrole", firstRelation.map(OpenStreetMapReader.RelationMember::role).orElse(null));
}
}
);
@ -882,7 +882,8 @@ public class FlatMapTest {
)),
feature(newLineString(128, 0.25 * 256, 128, 0.125 * 256), Map.of(
"attr", "value2",
"relname", "relation name"
"relname", "relation name",
"relrole", "role"
))
)
)), sortListValues(results.tiles));

17
core/src/test/java/com/onthegomap/flatmap/read/OpenStreetMapReaderTest.java
Wyświetl plik

@ -645,21 +645,15 @@ public class OpenStreetMapReaderTest {
@Test
public void testWayInRelation() {
record OtherRelInfo() implements OpenStreetMapReader.RelationInfo {}
record TestRelInfo(String name) implements OpenStreetMapReader.RelationInfo {
@Override
public long estimateMemoryUsageBytes() {
return 10 + name.length();
}
}
record OtherRelInfo(long id) implements OpenStreetMapReader.RelationInfo {}
record TestRelInfo(long id, String name) implements OpenStreetMapReader.RelationInfo {}
OpenStreetMapReader reader = new OpenStreetMapReader(
osmSource,
longLongMap,
new Profile.NullProfile() {
@Override
public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
return List.of(new TestRelInfo("name"));
return List.of(new TestRelInfo(1, "name"));
}
},
stats
@ -671,7 +665,7 @@ public class OpenStreetMapReaderTest {
way.getNodes().add(node1.getId(), node2.getId());
way.setTag("key", "value");
var relation = new ReaderRelation(4);
relation.add(new ReaderRelation.Member(ReaderRelation.Member.WAY, 3, ""));
relation.add(new ReaderRelation.Member(ReaderRelation.Member.WAY, 3, "rolename"));
reader.processPass1(node1);
reader.processPass1(node2);
@ -681,7 +675,8 @@ public class OpenStreetMapReaderTest {
SourceFeature feature = reader.processWayPass2(nodeCache, way);
assertEquals(List.of(), feature.relationInfo(OtherRelInfo.class));
assertEquals(List.of(new TestRelInfo("name")), feature.relationInfo(TestRelInfo.class));
assertEquals(List.of(new OpenStreetMapReader.RelationMember<>("rolename", new TestRelInfo(1, "name"))),
feature.relationInfo(TestRelInfo.class));
}
private OpenStreetMapReader newOsmReader() {

14
examples/src/main/java/com/onthegomap/flatmap/examples/BikeRouteOverlay.java
Wyświetl plik

@ -14,8 +14,8 @@ import java.util.List;
public class BikeRouteOverlay implements Profile {
private static record RouteRelationInfo(String name, String ref, String route, String network) implements
OpenStreetMapReader.RelationInfo {}
private static record RouteRelationInfo(long id, String name, String ref, String route, String network)
implements OpenStreetMapReader.RelationInfo {}
@Override
public List<OpenStreetMapReader.RelationInfo> preprocessOsmRelation(ReaderRelation relation) {
@ -23,6 +23,7 @@ public class BikeRouteOverlay implements Profile {
String type = relation.getTag("route");
if ("mtb".equals(type) || "bicycle".equals(type)) {
return List.of(new RouteRelationInfo(
relation.getId(),
relation.getTag("name"),
relation.getTag("ref"),
type,
@ -42,10 +43,11 @@ public class BikeRouteOverlay implements Profile {
@Override
public void processFeature(SourceFeature sourceFeature, FeatureCollector features) {
if (sourceFeature.canBeLine()) {
for (RouteRelationInfo routeInfo : sourceFeature.relationInfo(RouteRelationInfo.class)) {
features.line(routeInfo.route + "-route-" + routeInfo.network)
.setAttr("name", routeInfo.name)
.setAttr("ref", routeInfo.ref)
for (var routeInfo : sourceFeature.relationInfo(RouteRelationInfo.class)) {
RouteRelationInfo relation = routeInfo.relation();
features.line(relation.route + "-route-" + relation.network)
.setAttr("name", relation.name)
.setAttr("ref", relation.ref)
.setZoomRange(0, 14)
.setMinPixelSize(0);
}

3
openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/Generate.java
Wyświetl plik

@ -160,9 +160,6 @@ public class Generate {
emitLayerDefinitions(config.tileset, layers, packageName, output);
emitTableDefinitions(tables, packageName, output);
// layers.forEach((layer) -> LOGGER.info("layer: " + layer.layer.id));
// tables.forEach((key, val) -> LOGGER.info("table: " + key));
}
private static void emitTableDefinitions(Map<String, Imposm3Table> tables, String packageName, Path output)