kopia lustrzana https://github.com/onthegomap/planetiler
optimize sqlite writer
Mike Barry
rodzic
e07cd872f3
commit
e381f54f62
|
|
@ -11,17 +11,17 @@ public class DummyOsmMirror implements OsmMirror {
|
|||
private class Bulk implements BulkWriter {
|
||||
|
||||
@Override
|
||||
public void putNode(OsmElement.Node node) {
|
||||
public void putNode(Serialized.Node node) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putWay(OsmElement.Way way) {
|
||||
public void putWay(Serialized.Way way) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putRelation(OsmElement.Relation node) {
|
||||
public void putRelation(Serialized.Relation node) {
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,41 +20,41 @@ public class InMemoryOsmMirror implements OsmMirror {
|
|||
|
||||
private class Bulk implements BulkWriter {
|
||||
@Override
|
||||
public void putNode(OsmElement.Node node) {
|
||||
nodes.put(node.id(), node);
|
||||
public void putNode(Serialized.Node node) {
|
||||
nodes.put(node.item().id(), node.item());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putWay(OsmElement.Way way) {
|
||||
var previous = ways.put(way.id(), way);
|
||||
public void putWay(Serialized.Way way) {
|
||||
var previous = ways.put(way.item().id(), way.item());
|
||||
if (previous != null) {
|
||||
for (var node : previous.nodes()) {
|
||||
nodesToParentWay.get(node.value).removeAll(way.id());
|
||||
nodesToParentWay.get(node.value).removeAll(way.item().id());
|
||||
}
|
||||
}
|
||||
for (var node : way.nodes()) {
|
||||
nodesToParentWay.computeIfAbsent(node.value, c -> new LongArrayList()).add(way.id());
|
||||
for (var node : way.item().nodes()) {
|
||||
nodesToParentWay.computeIfAbsent(node.value, c -> new LongArrayList()).add(way.item().id());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putRelation(OsmElement.Relation relation) {
|
||||
var previous = relations.put(relation.id(), relation);
|
||||
public void putRelation(Serialized.Relation relation) {
|
||||
var previous = relations.put(relation.item().id(), relation.item());
|
||||
if (previous != null) {
|
||||
for (var member : previous.members()) {
|
||||
(switch (member.type()) {
|
||||
case NODE -> nodesToParentRelation;
|
||||
case WAY -> waysToParentRelation;
|
||||
case RELATION -> relationsToParentRelation;
|
||||
}).get(member.ref()).removeAll(relation.id());
|
||||
}).get(member.ref()).removeAll(relation.item().id());
|
||||
}
|
||||
}
|
||||
for (var member : relation.members()) {
|
||||
for (var member : relation.item().members()) {
|
||||
(switch (member.type()) {
|
||||
case NODE -> nodesToParentRelation;
|
||||
case WAY -> waysToParentRelation;
|
||||
case RELATION -> relationsToParentRelation;
|
||||
}).computeIfAbsent(member.ref(), c -> new LongArrayList()).add(relation.id());
|
||||
}).computeIfAbsent(member.ref(), c -> new LongArrayList()).add(relation.item().id());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ public interface OsmMirror extends AutoCloseable {
|
|||
int processThreads = arguments.threads();
|
||||
OsmInputFile in = new OsmInputFile(input);
|
||||
var blockCounter = new AtomicLong();
|
||||
boolean id = !type.contains("sqlite");
|
||||
try (var blocks = in.get()) {
|
||||
record Batch(CompletableFuture<List<Serialized<? extends OsmElement>>> results, OsmBlockSource.Block block) {}
|
||||
var queue = new WorkQueue<Batch>("batches", processThreads * 2, 1, stats);
|
||||
|
|
@ -76,17 +77,17 @@ public interface OsmMirror extends AutoCloseable {
|
|||
packer.clear();
|
||||
if (item instanceof OsmElement.Node node) {
|
||||
if (doNodes) {
|
||||
OsmMirrorUtil.pack(packer, node);
|
||||
OsmMirrorUtil.pack(packer, node, id);
|
||||
result.add(new Serialized.Node(node, packer.toByteArray()));
|
||||
}
|
||||
} else if (item instanceof OsmElement.Way way) {
|
||||
if (doWays) {
|
||||
OsmMirrorUtil.pack(packer, way);
|
||||
OsmMirrorUtil.pack(packer, way, id);
|
||||
result.add(new Serialized.Way(way, packer.toByteArray()));
|
||||
}
|
||||
} else if (item instanceof OsmElement.Relation relation) {
|
||||
if (doRelations) {
|
||||
OsmMirrorUtil.pack(packer, relation);
|
||||
OsmMirrorUtil.pack(packer, relation, id);
|
||||
result.add(new Serialized.Relation(relation, packer.toByteArray()));
|
||||
}
|
||||
}
|
||||
|
|
@ -177,29 +178,11 @@ public interface OsmMirror extends AutoCloseable {
|
|||
|
||||
interface BulkWriter extends Closeable {
|
||||
|
||||
default void putNode(Serialized.Node node) {
|
||||
putNode(node.item());
|
||||
}
|
||||
void putNode(Serialized.Node node);
|
||||
|
||||
default void putWay(Serialized.Way way) {
|
||||
putWay(way.item());
|
||||
}
|
||||
void putWay(Serialized.Way way);
|
||||
|
||||
default void putRelation(Serialized.Relation node) {
|
||||
putRelation(node.item());
|
||||
}
|
||||
|
||||
default void putNode(OsmElement.Node node) {
|
||||
putNode(new Serialized.Node(node, OsmMirrorUtil.encode(node)));
|
||||
}
|
||||
|
||||
default void putWay(OsmElement.Way way) {
|
||||
putWay(new Serialized.Way(way, OsmMirrorUtil.encode(way)));
|
||||
}
|
||||
|
||||
default void putRelation(OsmElement.Relation node) {
|
||||
putRelation(new Serialized.Relation(node, OsmMirrorUtil.encode(node)));
|
||||
}
|
||||
void putRelation(Serialized.Relation node);
|
||||
}
|
||||
|
||||
BulkWriter newBulkWriter();
|
||||
|
|
|
|||
|
|
@ -66,9 +66,9 @@ public class OsmMirrorUtil {
|
|||
return result;
|
||||
}
|
||||
|
||||
public static byte[] encode(OsmElement.Node node) {
|
||||
public static byte[] encode(OsmElement.Node node, boolean id) {
|
||||
try (var messagePack = MessagePack.newDefaultBufferPacker()) {
|
||||
pack(messagePack, node);
|
||||
pack(messagePack, node, id);
|
||||
return messagePack.toByteArray();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
|
|
@ -83,9 +83,9 @@ public class OsmMirrorUtil {
|
|||
}
|
||||
}
|
||||
|
||||
public static byte[] encode(OsmElement.Way way) {
|
||||
public static byte[] encode(OsmElement.Way way, boolean id) {
|
||||
try (var messagePack = MessagePack.newDefaultBufferPacker()) {
|
||||
pack(messagePack, way);
|
||||
pack(messagePack, way, id);
|
||||
return messagePack.toByteArray();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
|
|
@ -100,9 +100,9 @@ public class OsmMirrorUtil {
|
|||
}
|
||||
}
|
||||
|
||||
public static byte[] encode(OsmElement.Relation relation) {
|
||||
public static byte[] encode(OsmElement.Relation relation, boolean id) {
|
||||
try (var messagePack = MessagePack.newDefaultBufferPacker()) {
|
||||
pack(messagePack, relation);
|
||||
pack(messagePack, relation, id);
|
||||
return messagePack.toByteArray();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
|
|
@ -118,8 +118,10 @@ public class OsmMirrorUtil {
|
|||
}
|
||||
|
||||
|
||||
static void pack(MessagePacker messagePack, OsmElement.Node node) throws IOException {
|
||||
messagePack.packLong(node.id());
|
||||
static void pack(MessagePacker messagePack, OsmElement.Node node, boolean id) throws IOException {
|
||||
if (id) {
|
||||
messagePack.packLong(node.id());
|
||||
}
|
||||
messagePack.packInt(node.version());
|
||||
messagePack.packLong(node.encodedLocation());
|
||||
packTags(node.tags(), messagePack);
|
||||
|
|
@ -127,6 +129,10 @@ public class OsmMirrorUtil {
|
|||
|
||||
private static OsmElement.Node unpackNode(MessageUnpacker messagePack) throws IOException {
|
||||
long id = messagePack.unpackLong();
|
||||
return unpackNode(messagePack, id);
|
||||
}
|
||||
|
||||
private static OsmElement.Node unpackNode(MessageUnpacker messagePack, long id) throws IOException {
|
||||
int version = messagePack.unpackInt();
|
||||
long location = messagePack.unpackLong();
|
||||
Map<String, Object> tags = unpackTags(messagePack);
|
||||
|
|
@ -151,8 +157,10 @@ public class OsmMirrorUtil {
|
|||
return (n >>> 1) ^ -(n & 1);
|
||||
}
|
||||
|
||||
static void pack(MessagePacker messagePack, OsmElement.Way way) throws IOException {
|
||||
messagePack.packLong(way.id());
|
||||
static void pack(MessagePacker messagePack, OsmElement.Way way, boolean id) throws IOException {
|
||||
if (id) {
|
||||
messagePack.packLong(way.id());
|
||||
}
|
||||
messagePack.packInt(way.version());
|
||||
messagePack.packArrayHeader(way.nodes().size());
|
||||
LongArrayList nodes = way.nodes();
|
||||
|
|
@ -167,6 +175,10 @@ public class OsmMirrorUtil {
|
|||
|
||||
private static OsmElement.Way unpackWay(MessageUnpacker messagePack) throws IOException {
|
||||
long id = messagePack.unpackLong();
|
||||
return unpackWay(messagePack, id);
|
||||
}
|
||||
|
||||
private static OsmElement.Way unpackWay(MessageUnpacker messagePack, long id) throws IOException {
|
||||
int version = messagePack.unpackInt();
|
||||
int nodeCount = messagePack.unpackArrayHeader();
|
||||
long nodeId = 0;
|
||||
|
|
@ -179,8 +191,10 @@ public class OsmMirrorUtil {
|
|||
return new OsmElement.Way(id, tags, longs, OsmElement.Info.forVersion(version));
|
||||
}
|
||||
|
||||
static void pack(MessagePacker messagePack, OsmElement.Relation relation) throws IOException {
|
||||
messagePack.packLong(relation.id());
|
||||
static void pack(MessagePacker messagePack, OsmElement.Relation relation, boolean id) throws IOException {
|
||||
if (id) {
|
||||
messagePack.packLong(relation.id());
|
||||
}
|
||||
messagePack.packInt(relation.version());
|
||||
messagePack.packArrayHeader(relation.members().size());
|
||||
var members = relation.members();
|
||||
|
|
@ -202,6 +216,10 @@ public class OsmMirrorUtil {
|
|||
|
||||
private static OsmElement.Relation unpackRelation(MessageUnpacker messagePack) throws IOException {
|
||||
long id = messagePack.unpackLong();
|
||||
return unpackRelation(messagePack, id);
|
||||
}
|
||||
|
||||
private static OsmElement.Relation unpackRelation(MessageUnpacker messagePack, long id) throws IOException {
|
||||
int version = messagePack.unpackInt();
|
||||
int memberCount = messagePack.unpackArrayHeader();
|
||||
List<OsmElement.Relation.Member> members = new ArrayList<>(memberCount);
|
||||
|
|
@ -229,13 +247,37 @@ public class OsmMirrorUtil {
|
|||
);
|
||||
}
|
||||
|
||||
public static void packElement(MessageBufferPacker packer, OsmElement item) throws IOException {
|
||||
public static void packElement(MessageBufferPacker packer, OsmElement item, boolean id) throws IOException {
|
||||
if (item instanceof OsmElement.Node node) {
|
||||
pack(packer, node);
|
||||
pack(packer, node, id);
|
||||
} else if (item instanceof OsmElement.Way way) {
|
||||
pack(packer, way);
|
||||
pack(packer, way, id);
|
||||
} else if (item instanceof OsmElement.Relation relation) {
|
||||
pack(packer, relation);
|
||||
pack(packer, relation, id);
|
||||
}
|
||||
}
|
||||
|
||||
public static OsmElement.Way decodeWay(long id, byte[] bytes) {
|
||||
try (var messagePack = MessagePack.newDefaultUnpacker(bytes)) {
|
||||
return unpackWay(messagePack, id);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static OsmElement.Node decodeNode(long id, byte[] bytes) {
|
||||
try (var messagePack = MessagePack.newDefaultUnpacker(bytes)) {
|
||||
return unpackNode(messagePack, id);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static OsmElement.Relation decodeRelation(long id, byte[] bytes) {
|
||||
try (var messagePack = MessagePack.newDefaultUnpacker(bytes)) {
|
||||
return unpackRelation(messagePack, id);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,43 @@ public interface Serialized<T> {
|
|||
|
||||
byte[] bytes();
|
||||
|
||||
record Node(OsmElement.Node item, byte[] bytes) implements Serialized<OsmElement.Node> {}
|
||||
record Way(OsmElement.Way item, byte[] bytes) implements Serialized<OsmElement.Way> {}
|
||||
record Relation(OsmElement.Relation item, byte[] bytes) implements Serialized<OsmElement.Relation> {}
|
||||
record Node(OsmElement.Node item, byte[] bytes) implements Serialized<OsmElement.Node> {
|
||||
Node(OsmElement.Node item, boolean id) {
|
||||
this(item, OsmMirrorUtil.encode(item, id));
|
||||
}
|
||||
|
||||
Node(byte[] item) {
|
||||
this(OsmMirrorUtil.decodeNode(item), item);
|
||||
}
|
||||
|
||||
Node(byte[] item, long id) {
|
||||
this(OsmMirrorUtil.decodeNode(id, item), item);
|
||||
}
|
||||
}
|
||||
record Way(OsmElement.Way item, byte[] bytes) implements Serialized<OsmElement.Way> {
|
||||
Way(OsmElement.Way item, boolean id) {
|
||||
this(item, OsmMirrorUtil.encode(item, id));
|
||||
}
|
||||
|
||||
Way(byte[] item) {
|
||||
this(OsmMirrorUtil.decodeWay(item), item);
|
||||
}
|
||||
|
||||
Way(byte[] item, long id) {
|
||||
this(OsmMirrorUtil.decodeWay(id, item), item);
|
||||
}
|
||||
}
|
||||
record Relation(OsmElement.Relation item, byte[] bytes) implements Serialized<OsmElement.Relation> {
|
||||
Relation(OsmElement.Relation item, boolean id) {
|
||||
this(item, OsmMirrorUtil.encode(item, id));
|
||||
}
|
||||
|
||||
Relation(byte[] item) {
|
||||
this(OsmMirrorUtil.decodeRelation(item), item);
|
||||
}
|
||||
|
||||
Relation(byte[] item, long id) {
|
||||
this(OsmMirrorUtil.decodeRelation(id, item), item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
package com.onthegomap.planetiler.osmmirror;
|
||||
|
||||
import static com.onthegomap.planetiler.osmmirror.OsmMirrorUtil.encodeTags;
|
||||
import static com.onthegomap.planetiler.osmmirror.OsmMirrorUtil.parseTags;
|
||||
|
||||
import com.carrotsearch.hppc.LongArrayList;
|
||||
import com.carrotsearch.hppc.LongHashSet;
|
||||
import com.carrotsearch.hppc.LongSet;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.onthegomap.planetiler.config.Arguments;
|
||||
import com.onthegomap.planetiler.mbtiles.Mbtiles;
|
||||
|
|
@ -16,7 +15,6 @@ import java.sql.DriverManager;
|
|||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
|
|
@ -26,18 +24,8 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
import org.sqlite.SQLiteConfig;
|
||||
|
||||
// java -Xmx30g -cp planetiler-osmmirror.jar com.onthegomap.planetiler.osmmirror.OsmMirror --input data/sources/planet.osm.pbf --output planet.db --threads 10 --db sqlite 2>&1 | tee sqlite2.txt
|
||||
public class SqliteOsmMirror implements OsmMirror {
|
||||
// TODO
|
||||
// - try letting it finish on a cheaper machine (ran out of 450g disk at 534m ways)
|
||||
// - try custom mapdb implementation
|
||||
// - nodes
|
||||
// - ways
|
||||
// - nodeToParentWay
|
||||
// - rels
|
||||
// - nodeToParentRel
|
||||
// - wayToParentRel
|
||||
// - relToParentRel
|
||||
|
||||
private static final boolean INSERT_IGNORE = false;
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SqliteOsmMirror.class);
|
||||
|
|
@ -97,42 +85,42 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
version INTEGER,
|
||||
tags BLOB,
|
||||
location INTEGER
|
||||
data BLOB
|
||||
) WITHOUT ROWID""");
|
||||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS ways (
|
||||
id INTEGER PRIMARY KEY,
|
||||
version INTEGER,
|
||||
tags BLOB
|
||||
data BLOB
|
||||
) WITHOUT ROWID""");
|
||||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS relations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
version INTEGER,
|
||||
tags BLOB
|
||||
data BLOB
|
||||
) WITHOUT ROWID""");
|
||||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS way_members (
|
||||
way_id INTEGER,
|
||||
`order` INTEGER,
|
||||
node_id INTEGER,
|
||||
version INTEGER,
|
||||
PRIMARY KEY(way_id, `order`)
|
||||
CREATE TABLE IF NOT EXISTS node_to_way (
|
||||
child_id INTEGER,
|
||||
parent_id INTEGER,
|
||||
PRIMARY KEY(child_id, parent_id)
|
||||
) WITHOUT ROWID""");
|
||||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS relation_members (
|
||||
relation_id INTEGER,
|
||||
`order` INTEGER,
|
||||
type INTEGER,
|
||||
version INTEGER,
|
||||
role TEXT,
|
||||
ref INTEGER,
|
||||
PRIMARY KEY(relation_id, `order`)
|
||||
CREATE TABLE IF NOT EXISTS node_to_relation (
|
||||
child_id INTEGER,
|
||||
parent_id INTEGER,
|
||||
PRIMARY KEY(child_id, parent_id)
|
||||
) WITHOUT ROWID""");
|
||||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS way_to_relation (
|
||||
child_id INTEGER,
|
||||
parent_id INTEGER,
|
||||
PRIMARY KEY(child_id, parent_id)
|
||||
) WITHOUT ROWID""");
|
||||
execute("""
|
||||
CREATE TABLE IF NOT EXISTS relation_to_relation (
|
||||
child_id INTEGER,
|
||||
parent_id INTEGER,
|
||||
PRIMARY KEY(child_id, parent_id)
|
||||
) WITHOUT ROWID""");
|
||||
execute("CREATE INDEX IF NOT EXISTS way_members_node_idx ON way_members (node_id)");
|
||||
execute("CREATE INDEX IF NOT EXISTS relation_members_type_ref_idx ON relation_members (type, ref)");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -183,24 +171,7 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
private OsmElement.Way parseWay(ResultSet result) {
|
||||
try {
|
||||
String nodeString = result.getString("nodes");
|
||||
LongArrayList nodeIds;
|
||||
if (nodeString != null && !nodeString.isBlank()) {
|
||||
String[] nodes = result.getString("nodes").split(",");
|
||||
long[] nodeIdArray = new long[nodes.length];
|
||||
for (int i = 0; i < nodes.length; i++) {
|
||||
nodeIdArray[i] = Long.parseLong(nodes[i]);
|
||||
}
|
||||
nodeIds = LongArrayList.from(nodeIdArray);
|
||||
} else {
|
||||
nodeIds = LongArrayList.from();
|
||||
}
|
||||
return new OsmElement.Way(
|
||||
result.getInt("id"),
|
||||
parseTags(result.getBytes("tags")),
|
||||
nodeIds,
|
||||
OsmElement.Info.forVersion(result.getInt("version"))
|
||||
);
|
||||
return OsmMirrorUtil.decodeWay(result.getLong("id"), result.getBytes("data"));
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
|
@ -208,12 +179,7 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
private OsmElement.Node parseNode(ResultSet result) {
|
||||
try {
|
||||
return new OsmElement.Node(
|
||||
result.getInt("id"),
|
||||
parseTags(result.getBytes("tags")),
|
||||
result.getLong("location"),
|
||||
OsmElement.Info.forVersion(result.getInt("version"))
|
||||
);
|
||||
return OsmMirrorUtil.decodeNode(result.getLong("id"), result.getBytes("data"));
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
|
@ -221,30 +187,7 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
private OsmElement.Relation parseRelation(ResultSet result) {
|
||||
try {
|
||||
String refsString = result.getString("refs");
|
||||
List<OsmElement.Relation.Member> members;
|
||||
if (refsString != null && !refsString.isBlank()) {
|
||||
String[] refs = result.getString("refs").split(",");
|
||||
String[] types = result.getString("types").split(",");
|
||||
String[] roles = result.getString("roles").split(",");
|
||||
members = new ArrayList<>(refs.length);
|
||||
|
||||
for (int i = 0; i < refs.length; i++) {
|
||||
members.add(new OsmElement.Relation.Member(
|
||||
OsmElement.Type.values()[Integer.parseInt(types[i])],
|
||||
Long.parseLong(refs[i]),
|
||||
i >= roles.length ? "" : roles[i]
|
||||
));
|
||||
}
|
||||
} else {
|
||||
members = List.of();
|
||||
}
|
||||
return new OsmElement.Relation(
|
||||
result.getInt("id"),
|
||||
parseTags(result.getBytes("tags")),
|
||||
members,
|
||||
OsmElement.Info.forVersion(result.getInt("version"))
|
||||
);
|
||||
return OsmMirrorUtil.decodeRelation(result.getLong("id"), result.getBytes("data"));
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
|
@ -252,21 +195,7 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
@Override
|
||||
public LongArrayList getParentWaysForNode(long nodeId) {
|
||||
try (
|
||||
var statement = connection.prepareStatement(
|
||||
"""
|
||||
SELECT way_id
|
||||
FROM way_members
|
||||
WHERE node_id=?
|
||||
ORDER BY way_id ASC
|
||||
""")
|
||||
) {
|
||||
statement.setLong(1, nodeId);
|
||||
var result = statement.executeQuery();
|
||||
return parseLongList(result);
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
return getParents(nodeId, "node_to_way");
|
||||
}
|
||||
|
||||
private LongArrayList parseLongList(ResultSet resultSet) {
|
||||
|
|
@ -283,19 +212,10 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
@Override
|
||||
public OsmElement.Way getWay(long id) {
|
||||
try (
|
||||
var statement = connection.prepareStatement(
|
||||
"""
|
||||
SELECT ways.*, group_concat(way_members.node_id) nodes
|
||||
FROM ways
|
||||
INNER JOIN way_members on ways.id=way_members.way_id
|
||||
WHERE ways.id=?
|
||||
ORDER BY way_members.`order` ASC
|
||||
""")
|
||||
) {
|
||||
try (var statement = connection.prepareStatement("SELECT * FROM ways WHERE id=?")) {
|
||||
statement.setLong(1, id);
|
||||
var result = statement.executeQuery();
|
||||
return result.next() && result.getLong("id") == id ? parseWay(result) : null;
|
||||
return result.next() ? parseWay(result) : null;
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
|
|
@ -303,23 +223,10 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
@Override
|
||||
public OsmElement.Relation getRelation(long id) {
|
||||
try (
|
||||
var statement = connection.prepareStatement(
|
||||
"""
|
||||
SELECT
|
||||
relations.*,
|
||||
group_concat(relation_members.role) roles,
|
||||
group_concat(relation_members.type) types,
|
||||
group_concat(relation_members.ref) refs
|
||||
FROM relations
|
||||
JOIN relation_members on relations.id=relation_members.relation_id
|
||||
WHERE relations.id=?
|
||||
ORDER BY relation_members.`order` ASC
|
||||
""")
|
||||
) {
|
||||
try (var statement = connection.prepareStatement("SELECT * FROM relations WHERE id=?")) {
|
||||
statement.setLong(1, id);
|
||||
var result = statement.executeQuery();
|
||||
return result.next() && result.getLong("id") == id ? parseRelation(result) : null;
|
||||
return result.next() ? parseRelation(result) : null;
|
||||
} catch (SQLException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
|
|
@ -327,21 +234,20 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
@Override
|
||||
public LongArrayList getParentRelationsForNode(long nodeId) {
|
||||
return getParentRelations(nodeId, OsmElement.Type.NODE);
|
||||
return getParents(nodeId, "node_to_relation");
|
||||
}
|
||||
|
||||
private LongArrayList getParentRelations(long nodeId, OsmElement.Type type) {
|
||||
private LongArrayList getParents(long childId, String table) {
|
||||
try (
|
||||
var statement = connection.prepareStatement(
|
||||
"""
|
||||
SELECT relation_id
|
||||
FROM relation_members
|
||||
WHERE ref=? AND type=?
|
||||
ORDER BY relation_id ASC
|
||||
""")
|
||||
SELECT parent_id
|
||||
FROM %s
|
||||
WHERE child_id=?
|
||||
ORDER BY parent_id ASC
|
||||
""".formatted(table))
|
||||
) {
|
||||
statement.setLong(1, nodeId);
|
||||
statement.setInt(2, type.ordinal());
|
||||
statement.setLong(1, childId);
|
||||
var result = statement.executeQuery();
|
||||
return parseLongList(result);
|
||||
} catch (SQLException e) {
|
||||
|
|
@ -351,12 +257,12 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
|
||||
@Override
|
||||
public LongArrayList getParentRelationsForWay(long nodeId) {
|
||||
return getParentRelations(nodeId, OsmElement.Type.WAY);
|
||||
return getParents(nodeId, "way_to_relation");
|
||||
}
|
||||
|
||||
@Override
|
||||
public LongArrayList getParentRelationsForRelation(long nodeId) {
|
||||
return getParentRelations(nodeId, OsmElement.Type.RELATION);
|
||||
return getParents(nodeId, "relation_to_relation");
|
||||
}
|
||||
|
||||
private <T> Iterator<T> resultToIterator(ResultSet resultSet, Function<ResultSet, T> parse) {
|
||||
|
|
@ -396,25 +302,8 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
public CloseableIterator<OsmElement> iterator() {
|
||||
try {
|
||||
var nodes = connection.prepareStatement("SELECT * FROM nodes ORDER BY id ASC").executeQuery();
|
||||
var ways = connection.prepareStatement(
|
||||
"""
|
||||
SELECT ways.*, group_concat(way_members.node_id) nodes
|
||||
FROM ways
|
||||
JOIN way_members on ways.id=way_members.way_id
|
||||
GROUP BY ways.id
|
||||
ORDER BY ways.id ASC, way_members.`order` ASC
|
||||
""").executeQuery();
|
||||
var relations = connection.prepareStatement("""
|
||||
SELECT
|
||||
relations.*,
|
||||
group_concat(relation_members.role) roles,
|
||||
group_concat(relation_members.type) types,
|
||||
group_concat(relation_members.ref) refs
|
||||
FROM relations
|
||||
JOIN relation_members on relations.id=relation_members.relation_id
|
||||
GROUP BY relations.id
|
||||
ORDER BY relations.id ASC, relation_members.`order` ASC
|
||||
""").executeQuery();
|
||||
var ways = connection.prepareStatement("SELECT * FROM ways ORDER BY id ASC").executeQuery();
|
||||
var relations = connection.prepareStatement("SELECT * FROM relations ORDER BY id ASC").executeQuery();
|
||||
var nodesIter = resultToIterator(nodes, this::parseNode);
|
||||
var waysIter = resultToIterator(ways, this::parseWay);
|
||||
var relsIter = resultToIterator(relations, this::parseRelation);
|
||||
|
|
@ -452,34 +341,56 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
}
|
||||
}
|
||||
|
||||
private record ParentChild<P> (P parent, int idx) {}
|
||||
private record ParentChild(long child, long parent) {}
|
||||
|
||||
private class Bulk implements BulkWriter {
|
||||
private final NodeWriter nodeWriter = new NodeWriter(connection);
|
||||
private final WayWriter wayWriter = new WayWriter(connection);
|
||||
private final WayMembersWriter wayMemberWriter = new WayMembersWriter(connection);
|
||||
private final RelationWriter relationWriter = new RelationWriter(connection);
|
||||
private final RelationMembersWriter relationMemberWriter = new RelationMembersWriter(connection);
|
||||
private final ElementWriter nodeWriter = new ElementWriter(connection, "nodes");
|
||||
private final ElementWriter wayWriter = new ElementWriter(connection, "ways");
|
||||
private final ChildToParentWriter wayMemberWriter = new ChildToParentWriter(connection, "node_to_way");
|
||||
private final ElementWriter relationWriter = new ElementWriter(connection, "relations");
|
||||
private final ChildToParentWriter nodeToRelWriter = new ChildToParentWriter(connection, "node_to_relation");
|
||||
private final ChildToParentWriter wayToRelWriter = new ChildToParentWriter(connection, "way_to_relation");
|
||||
private final ChildToParentWriter relToRelWriter = new ChildToParentWriter(connection, "relation_to_relation");
|
||||
|
||||
@Override
|
||||
public void putNode(OsmElement.Node node) {
|
||||
public void putNode(Serialized.Node node) {
|
||||
nodeWriter.write(node);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putWay(OsmElement.Way way) {
|
||||
public void putWay(Serialized.Way way) {
|
||||
wayWriter.write(way);
|
||||
// TODO write way members separately, then insert in order into table in close
|
||||
for (int i = 0; i < way.nodes().size(); i++) {
|
||||
wayMemberWriter.write(new ParentChild<>(way, i));
|
||||
var nodes = way.item().nodes();
|
||||
LongSet written = new LongHashSet();
|
||||
for (int i = 0; i < nodes.size(); i++) {
|
||||
long id = nodes.get(i);
|
||||
if (written.add(id)) {
|
||||
wayMemberWriter.write(new ParentChild(nodes.get(i), way.item().id()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putRelation(OsmElement.Relation relation) {
|
||||
public void putRelation(Serialized.Relation relation) {
|
||||
relationWriter.write(relation);
|
||||
for (int i = 0; i < relation.members().size(); i++) {
|
||||
relationMemberWriter.write(new ParentChild<>(relation, i));
|
||||
LongHashSet nodes = new LongHashSet();
|
||||
LongHashSet ways = new LongHashSet();
|
||||
LongHashSet rels = new LongHashSet();
|
||||
for (int i = 0; i < relation.item().members().size(); i++) {
|
||||
var member = relation.item().members().get(i);
|
||||
var set = (switch (member.type()) {
|
||||
case NODE -> nodes;
|
||||
case WAY -> ways;
|
||||
case RELATION -> rels;
|
||||
});
|
||||
if (set.add(member.ref())) {
|
||||
(switch (member.type()) {
|
||||
case NODE -> nodeToRelWriter;
|
||||
case WAY -> wayToRelWriter;
|
||||
case RELATION -> relToRelWriter;
|
||||
}).write(new ParentChild(member.ref(), relation.item().id()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -490,91 +401,37 @@ public class SqliteOsmMirror implements OsmMirror {
|
|||
wayWriter.close();
|
||||
wayMemberWriter.close();
|
||||
relationWriter.close();
|
||||
relationMemberWriter.close();
|
||||
nodeToRelWriter.close();
|
||||
wayToRelWriter.close();
|
||||
relToRelWriter.close();
|
||||
}
|
||||
}
|
||||
|
||||
private class NodeWriter extends Mbtiles.BatchedTableWriterBase<OsmElement.Node> {
|
||||
NodeWriter(Connection conn) {
|
||||
super("nodes", List.of("id", "version", "tags", "location"), INSERT_IGNORE, conn);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, OsmElement.Node item)
|
||||
throws SQLException {
|
||||
statement.setLong(positionOffset++, item.id());
|
||||
statement.setInt(positionOffset++, item.info().version());
|
||||
statement.setBytes(positionOffset++, encodeTags(item.tags()));
|
||||
statement.setLong(positionOffset++, item.encodedLocation());
|
||||
return positionOffset;
|
||||
}
|
||||
}
|
||||
|
||||
private class WayWriter extends Mbtiles.BatchedTableWriterBase<OsmElement.Way> {
|
||||
WayWriter(Connection conn) {
|
||||
super("ways", List.of("id", "version", "tags"), INSERT_IGNORE, conn);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, OsmElement.Way item)
|
||||
throws SQLException {
|
||||
statement.setLong(positionOffset++, item.id());
|
||||
statement.setInt(positionOffset++, item.info().version());
|
||||
statement.setBytes(positionOffset++, encodeTags(item.tags()));
|
||||
return positionOffset;
|
||||
}
|
||||
}
|
||||
|
||||
private class RelationWriter extends Mbtiles.BatchedTableWriterBase<OsmElement.Relation> {
|
||||
RelationWriter(Connection conn) {
|
||||
super("relations", List.of("id", "version", "tags"), INSERT_IGNORE, conn);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, OsmElement.Relation item)
|
||||
throws SQLException {
|
||||
statement.setLong(positionOffset++, item.id());
|
||||
statement.setInt(positionOffset++, item.info().version());
|
||||
statement.setBytes(positionOffset++, encodeTags(item.tags()));
|
||||
return positionOffset;
|
||||
}
|
||||
}
|
||||
|
||||
private class RelationMembersWriter
|
||||
extends Mbtiles.BatchedTableWriterBase<ParentChild<OsmElement.Relation>> {
|
||||
RelationMembersWriter(Connection conn) {
|
||||
super("relation_members", List.of("relation_id", "`order`", "type", "version", "role", "ref"), INSERT_IGNORE,
|
||||
conn);
|
||||
private class ElementWriter extends Mbtiles.BatchedTableWriterBase<Serialized<? extends OsmElement>> {
|
||||
ElementWriter(Connection conn, String table) {
|
||||
super(table, List.of("id", "data"), INSERT_IGNORE, conn);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement,
|
||||
ParentChild<OsmElement.Relation> item) throws SQLException {
|
||||
var relation = item.parent;
|
||||
var member = item.parent.members().get(item.idx);
|
||||
statement.setLong(positionOffset++, relation.id());
|
||||
statement.setInt(positionOffset++, item.idx);
|
||||
statement.setInt(positionOffset++, member.type().ordinal());
|
||||
statement.setInt(positionOffset++, relation.info().version());
|
||||
statement.setString(positionOffset++, member.role());
|
||||
statement.setLong(positionOffset++, member.ref());
|
||||
Serialized<? extends OsmElement> item)
|
||||
throws SQLException {
|
||||
statement.setLong(positionOffset++, item.item().id());
|
||||
statement.setBytes(positionOffset++, item.bytes());
|
||||
return positionOffset;
|
||||
}
|
||||
}
|
||||
|
||||
private class WayMembersWriter extends Mbtiles.BatchedTableWriterBase<ParentChild<OsmElement.Way>> {
|
||||
WayMembersWriter(Connection conn) {
|
||||
super("way_members", List.of("way_id", "`order`", "node_id", "version"), INSERT_IGNORE, conn);
|
||||
private class ChildToParentWriter extends Mbtiles.BatchedTableWriterBase<ParentChild> {
|
||||
ChildToParentWriter(Connection conn, String table) {
|
||||
super(table, List.of("child_id", "parent_id"), INSERT_IGNORE, conn);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement,
|
||||
ParentChild<OsmElement.Way> item) throws SQLException {
|
||||
var way = item.parent;
|
||||
statement.setLong(positionOffset++, way.id());
|
||||
statement.setInt(positionOffset++, item.idx);
|
||||
statement.setLong(positionOffset++, way.nodes().get(item.idx));
|
||||
statement.setInt(positionOffset++, way.info().version());
|
||||
ParentChild item) throws SQLException {
|
||||
statement.setLong(positionOffset++, item.child());
|
||||
statement.setLong(positionOffset++, item.parent());
|
||||
return positionOffset;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import org.junit.jupiter.api.io.TempDir;
|
|||
|
||||
abstract class OsmMirrorTest {
|
||||
OsmMirror fixture;
|
||||
protected boolean serializeId = true;
|
||||
|
||||
@Test
|
||||
void testEmpty() {
|
||||
|
|
@ -32,7 +33,7 @@ abstract class OsmMirrorTest {
|
|||
void testInsertNode() throws IOException {
|
||||
var node = new OsmElement.Node(1, Map.of("key", "value"), 2, 3, infoVersion(0));
|
||||
try (var writer = fixture.newBulkWriter()) {
|
||||
writer.putNode(node);
|
||||
writer.putNode(new Serialized.Node(node, serializeId));
|
||||
}
|
||||
assertEquals(node, fixture.getNode(1));
|
||||
assertEquals(List.of(node), fixture.iterator().toList());
|
||||
|
|
@ -41,10 +42,10 @@ abstract class OsmMirrorTest {
|
|||
@Test
|
||||
void testInsertWay() throws IOException {
|
||||
var way = new OsmElement.Way(2, Map.of("key", "value"), LongArrayList.from(1), infoVersion(0));
|
||||
var way2 = new OsmElement.Way(3, Map.of("key", "value"), LongArrayList.from(1), infoVersion(0));
|
||||
var way2 = new OsmElement.Way(3, Map.of("key", "value"), LongArrayList.from(1, 1), infoVersion(0));
|
||||
try (var writer = fixture.newBulkWriter()) {
|
||||
writer.putWay(way);
|
||||
writer.putWay(way2);
|
||||
writer.putWay(new Serialized.Way(way, serializeId));
|
||||
writer.putWay(new Serialized.Way(way2, serializeId));
|
||||
}
|
||||
assertEquals(way, fixture.getWay(2));
|
||||
assertEquals(way2, fixture.getWay(3));
|
||||
|
|
@ -63,8 +64,8 @@ abstract class OsmMirrorTest {
|
|||
new OsmElement.Relation.Member(OsmElement.Type.RELATION, 3, "rel")
|
||||
), infoVersion(0));
|
||||
try (var writer = fixture.newBulkWriter()) {
|
||||
writer.putRelation(rel);
|
||||
writer.putRelation(rel2);
|
||||
writer.putRelation(new Serialized.Relation(rel, serializeId));
|
||||
writer.putRelation(new Serialized.Relation(rel2, serializeId));
|
||||
}
|
||||
assertEquals(rel, fixture.getRelation(3));
|
||||
assertEquals(rel2, fixture.getRelation(4));
|
||||
|
|
@ -86,9 +87,11 @@ abstract class OsmMirrorTest {
|
|||
}
|
||||
static class SqliteTest extends OsmMirrorTest {
|
||||
|
||||
|
||||
@BeforeEach
|
||||
void setup() {
|
||||
fixture = OsmMirror.newSqliteMemory();
|
||||
serializeId = false;
|
||||
}
|
||||
}
|
||||
static class MapdbTest extends OsmMirrorTest {
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue