From e381f54f62bceded712951db71b3b00236dcbc8b Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Fri, 7 Apr 2023 06:16:05 -0400 Subject: [PATCH] optimize sqlite writer --- .../planetiler/osmmirror/DummyOsmMirror.java | 6 +- .../osmmirror/InMemoryOsmMirror.java | 24 +- .../planetiler/osmmirror/OsmMirror.java | 31 +- .../planetiler/osmmirror/OsmMirrorUtil.java | 74 +++- .../planetiler/osmmirror/Serialized.java | 42 ++- .../planetiler/osmmirror/SqliteOsmMirror.java | 337 +++++------------- .../planetiler/osmmirror/OsmMirrorTest.java | 15 +- 7 files changed, 225 insertions(+), 304 deletions(-) diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/DummyOsmMirror.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/DummyOsmMirror.java index c9cda9c5..2700e6ea 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/DummyOsmMirror.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/DummyOsmMirror.java @@ -11,17 +11,17 @@ public class DummyOsmMirror implements OsmMirror { private class Bulk implements BulkWriter { @Override - public void putNode(OsmElement.Node node) { + public void putNode(Serialized.Node node) { } @Override - public void putWay(OsmElement.Way way) { + public void putWay(Serialized.Way way) { } @Override - public void putRelation(OsmElement.Relation node) { + public void putRelation(Serialized.Relation node) { } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/InMemoryOsmMirror.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/InMemoryOsmMirror.java index 52b1e330..9b47cc68 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/InMemoryOsmMirror.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/InMemoryOsmMirror.java @@ -20,41 +20,41 @@ public class InMemoryOsmMirror implements OsmMirror { private class Bulk implements BulkWriter { @Override - public void putNode(OsmElement.Node node) { - nodes.put(node.id(), node); + public void putNode(Serialized.Node node) { + nodes.put(node.item().id(), node.item()); } @Override - public void putWay(OsmElement.Way way) { - var previous = ways.put(way.id(), way); + public void putWay(Serialized.Way way) { + var previous = ways.put(way.item().id(), way.item()); if (previous != null) { for (var node : previous.nodes()) { - nodesToParentWay.get(node.value).removeAll(way.id()); + nodesToParentWay.get(node.value).removeAll(way.item().id()); } } - for (var node : way.nodes()) { - nodesToParentWay.computeIfAbsent(node.value, c -> new LongArrayList()).add(way.id()); + for (var node : way.item().nodes()) { + nodesToParentWay.computeIfAbsent(node.value, c -> new LongArrayList()).add(way.item().id()); } } @Override - public void putRelation(OsmElement.Relation relation) { - var previous = relations.put(relation.id(), relation); + public void putRelation(Serialized.Relation relation) { + var previous = relations.put(relation.item().id(), relation.item()); if (previous != null) { for (var member : previous.members()) { (switch (member.type()) { case NODE -> nodesToParentRelation; case WAY -> waysToParentRelation; case RELATION -> relationsToParentRelation; - }).get(member.ref()).removeAll(relation.id()); + }).get(member.ref()).removeAll(relation.item().id()); } } - for (var member : relation.members()) { + for (var member : relation.item().members()) { (switch (member.type()) { case NODE -> nodesToParentRelation; case WAY -> waysToParentRelation; case RELATION -> relationsToParentRelation; - }).computeIfAbsent(member.ref(), c -> new LongArrayList()).add(relation.id()); + }).computeIfAbsent(member.ref(), c -> new LongArrayList()).add(relation.item().id()); } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirror.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirror.java index a58c7499..c4664dde 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirror.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirror.java @@ -54,6 +54,7 @@ public interface OsmMirror extends AutoCloseable { int processThreads = arguments.threads(); OsmInputFile in = new OsmInputFile(input); var blockCounter = new AtomicLong(); + boolean id = !type.contains("sqlite"); try (var blocks = in.get()) { record Batch(CompletableFuture>> results, OsmBlockSource.Block block) {} var queue = new WorkQueue("batches", processThreads * 2, 1, stats); @@ -76,17 +77,17 @@ public interface OsmMirror extends AutoCloseable { packer.clear(); if (item instanceof OsmElement.Node node) { if (doNodes) { - OsmMirrorUtil.pack(packer, node); + OsmMirrorUtil.pack(packer, node, id); result.add(new Serialized.Node(node, packer.toByteArray())); } } else if (item instanceof OsmElement.Way way) { if (doWays) { - OsmMirrorUtil.pack(packer, way); + OsmMirrorUtil.pack(packer, way, id); result.add(new Serialized.Way(way, packer.toByteArray())); } } else if (item instanceof OsmElement.Relation relation) { if (doRelations) { - OsmMirrorUtil.pack(packer, relation); + OsmMirrorUtil.pack(packer, relation, id); result.add(new Serialized.Relation(relation, packer.toByteArray())); } } @@ -177,29 +178,11 @@ public interface OsmMirror extends AutoCloseable { interface BulkWriter extends Closeable { - default void putNode(Serialized.Node node) { - putNode(node.item()); - } + void putNode(Serialized.Node node); - default void putWay(Serialized.Way way) { - putWay(way.item()); - } + void putWay(Serialized.Way way); - default void putRelation(Serialized.Relation node) { - putRelation(node.item()); - } - - default void putNode(OsmElement.Node node) { - putNode(new Serialized.Node(node, OsmMirrorUtil.encode(node))); - } - - default void putWay(OsmElement.Way way) { - putWay(new Serialized.Way(way, OsmMirrorUtil.encode(way))); - } - - default void putRelation(OsmElement.Relation node) { - putRelation(new Serialized.Relation(node, OsmMirrorUtil.encode(node))); - } + void putRelation(Serialized.Relation node); } BulkWriter newBulkWriter(); diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirrorUtil.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirrorUtil.java index 1b88c8a2..9173e7c1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirrorUtil.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/OsmMirrorUtil.java @@ -66,9 +66,9 @@ public class OsmMirrorUtil { return result; } - public static byte[] encode(OsmElement.Node node) { + public static byte[] encode(OsmElement.Node node, boolean id) { try (var messagePack = MessagePack.newDefaultBufferPacker()) { - pack(messagePack, node); + pack(messagePack, node, id); return messagePack.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); @@ -83,9 +83,9 @@ public class OsmMirrorUtil { } } - public static byte[] encode(OsmElement.Way way) { + public static byte[] encode(OsmElement.Way way, boolean id) { try (var messagePack = MessagePack.newDefaultBufferPacker()) { - pack(messagePack, way); + pack(messagePack, way, id); return messagePack.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); @@ -100,9 +100,9 @@ public class OsmMirrorUtil { } } - public static byte[] encode(OsmElement.Relation relation) { + public static byte[] encode(OsmElement.Relation relation, boolean id) { try (var messagePack = MessagePack.newDefaultBufferPacker()) { - pack(messagePack, relation); + pack(messagePack, relation, id); return messagePack.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); @@ -118,8 +118,10 @@ public class OsmMirrorUtil { } - static void pack(MessagePacker messagePack, OsmElement.Node node) throws IOException { - messagePack.packLong(node.id()); + static void pack(MessagePacker messagePack, OsmElement.Node node, boolean id) throws IOException { + if (id) { + messagePack.packLong(node.id()); + } messagePack.packInt(node.version()); messagePack.packLong(node.encodedLocation()); packTags(node.tags(), messagePack); @@ -127,6 +129,10 @@ public class OsmMirrorUtil { private static OsmElement.Node unpackNode(MessageUnpacker messagePack) throws IOException { long id = messagePack.unpackLong(); + return unpackNode(messagePack, id); + } + + private static OsmElement.Node unpackNode(MessageUnpacker messagePack, long id) throws IOException { int version = messagePack.unpackInt(); long location = messagePack.unpackLong(); Map tags = unpackTags(messagePack); @@ -151,8 +157,10 @@ public class OsmMirrorUtil { return (n >>> 1) ^ -(n & 1); } - static void pack(MessagePacker messagePack, OsmElement.Way way) throws IOException { - messagePack.packLong(way.id()); + static void pack(MessagePacker messagePack, OsmElement.Way way, boolean id) throws IOException { + if (id) { + messagePack.packLong(way.id()); + } messagePack.packInt(way.version()); messagePack.packArrayHeader(way.nodes().size()); LongArrayList nodes = way.nodes(); @@ -167,6 +175,10 @@ public class OsmMirrorUtil { private static OsmElement.Way unpackWay(MessageUnpacker messagePack) throws IOException { long id = messagePack.unpackLong(); + return unpackWay(messagePack, id); + } + + private static OsmElement.Way unpackWay(MessageUnpacker messagePack, long id) throws IOException { int version = messagePack.unpackInt(); int nodeCount = messagePack.unpackArrayHeader(); long nodeId = 0; @@ -179,8 +191,10 @@ public class OsmMirrorUtil { return new OsmElement.Way(id, tags, longs, OsmElement.Info.forVersion(version)); } - static void pack(MessagePacker messagePack, OsmElement.Relation relation) throws IOException { - messagePack.packLong(relation.id()); + static void pack(MessagePacker messagePack, OsmElement.Relation relation, boolean id) throws IOException { + if (id) { + messagePack.packLong(relation.id()); + } messagePack.packInt(relation.version()); messagePack.packArrayHeader(relation.members().size()); var members = relation.members(); @@ -202,6 +216,10 @@ public class OsmMirrorUtil { private static OsmElement.Relation unpackRelation(MessageUnpacker messagePack) throws IOException { long id = messagePack.unpackLong(); + return unpackRelation(messagePack, id); + } + + private static OsmElement.Relation unpackRelation(MessageUnpacker messagePack, long id) throws IOException { int version = messagePack.unpackInt(); int memberCount = messagePack.unpackArrayHeader(); List members = new ArrayList<>(memberCount); @@ -229,13 +247,37 @@ public class OsmMirrorUtil { ); } - public static void packElement(MessageBufferPacker packer, OsmElement item) throws IOException { + public static void packElement(MessageBufferPacker packer, OsmElement item, boolean id) throws IOException { if (item instanceof OsmElement.Node node) { - pack(packer, node); + pack(packer, node, id); } else if (item instanceof OsmElement.Way way) { - pack(packer, way); + pack(packer, way, id); } else if (item instanceof OsmElement.Relation relation) { - pack(packer, relation); + pack(packer, relation, id); + } + } + + public static OsmElement.Way decodeWay(long id, byte[] bytes) { + try (var messagePack = MessagePack.newDefaultUnpacker(bytes)) { + return unpackWay(messagePack, id); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static OsmElement.Node decodeNode(long id, byte[] bytes) { + try (var messagePack = MessagePack.newDefaultUnpacker(bytes)) { + return unpackNode(messagePack, id); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static OsmElement.Relation decodeRelation(long id, byte[] bytes) { + try (var messagePack = MessagePack.newDefaultUnpacker(bytes)) { + return unpackRelation(messagePack, id); + } catch (IOException e) { + throw new RuntimeException(e); } } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/Serialized.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/Serialized.java index 2fc13731..780b6fd1 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/Serialized.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/Serialized.java @@ -8,7 +8,43 @@ public interface Serialized { byte[] bytes(); - record Node(OsmElement.Node item, byte[] bytes) implements Serialized {} - record Way(OsmElement.Way item, byte[] bytes) implements Serialized {} - record Relation(OsmElement.Relation item, byte[] bytes) implements Serialized {} + record Node(OsmElement.Node item, byte[] bytes) implements Serialized { + Node(OsmElement.Node item, boolean id) { + this(item, OsmMirrorUtil.encode(item, id)); + } + + Node(byte[] item) { + this(OsmMirrorUtil.decodeNode(item), item); + } + + Node(byte[] item, long id) { + this(OsmMirrorUtil.decodeNode(id, item), item); + } + } + record Way(OsmElement.Way item, byte[] bytes) implements Serialized { + Way(OsmElement.Way item, boolean id) { + this(item, OsmMirrorUtil.encode(item, id)); + } + + Way(byte[] item) { + this(OsmMirrorUtil.decodeWay(item), item); + } + + Way(byte[] item, long id) { + this(OsmMirrorUtil.decodeWay(id, item), item); + } + } + record Relation(OsmElement.Relation item, byte[] bytes) implements Serialized { + Relation(OsmElement.Relation item, boolean id) { + this(item, OsmMirrorUtil.encode(item, id)); + } + + Relation(byte[] item) { + this(OsmMirrorUtil.decodeRelation(item), item); + } + + Relation(byte[] item, long id) { + this(OsmMirrorUtil.decodeRelation(id, item), item); + } + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/SqliteOsmMirror.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/SqliteOsmMirror.java index e920c7e0..5136e2e8 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/SqliteOsmMirror.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/osmmirror/SqliteOsmMirror.java @@ -1,9 +1,8 @@ package com.onthegomap.planetiler.osmmirror; -import static com.onthegomap.planetiler.osmmirror.OsmMirrorUtil.encodeTags; -import static com.onthegomap.planetiler.osmmirror.OsmMirrorUtil.parseTags; - import com.carrotsearch.hppc.LongArrayList; +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongSet; import com.google.common.collect.Iterators; import com.onthegomap.planetiler.config.Arguments; import com.onthegomap.planetiler.mbtiles.Mbtiles; @@ -16,7 +15,6 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -26,18 +24,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sqlite.SQLiteConfig; +// java -Xmx30g -cp planetiler-osmmirror.jar com.onthegomap.planetiler.osmmirror.OsmMirror --input data/sources/planet.osm.pbf --output planet.db --threads 10 --db sqlite 2>&1 | tee sqlite2.txt public class SqliteOsmMirror implements OsmMirror { - // TODO - // - try letting it finish on a cheaper machine (ran out of 450g disk at 534m ways) - // - try custom mapdb implementation - // - nodes - // - ways - // - nodeToParentWay - // - rels - // - nodeToParentRel - // - wayToParentRel - // - relToParentRel - private static final boolean INSERT_IGNORE = false; private static final Logger LOGGER = LoggerFactory.getLogger(SqliteOsmMirror.class); @@ -97,42 +85,42 @@ public class SqliteOsmMirror implements OsmMirror { execute(""" CREATE TABLE IF NOT EXISTS nodes ( id INTEGER PRIMARY KEY, - version INTEGER, - tags BLOB, - location INTEGER + data BLOB ) WITHOUT ROWID"""); execute(""" CREATE TABLE IF NOT EXISTS ways ( id INTEGER PRIMARY KEY, - version INTEGER, - tags BLOB + data BLOB ) WITHOUT ROWID"""); execute(""" CREATE TABLE IF NOT EXISTS relations ( id INTEGER PRIMARY KEY, - version INTEGER, - tags BLOB + data BLOB ) WITHOUT ROWID"""); execute(""" - CREATE TABLE IF NOT EXISTS way_members ( - way_id INTEGER, - `order` INTEGER, - node_id INTEGER, - version INTEGER, - PRIMARY KEY(way_id, `order`) + CREATE TABLE IF NOT EXISTS node_to_way ( + child_id INTEGER, + parent_id INTEGER, + PRIMARY KEY(child_id, parent_id) ) WITHOUT ROWID"""); execute(""" - CREATE TABLE IF NOT EXISTS relation_members ( - relation_id INTEGER, - `order` INTEGER, - type INTEGER, - version INTEGER, - role TEXT, - ref INTEGER, - PRIMARY KEY(relation_id, `order`) + CREATE TABLE IF NOT EXISTS node_to_relation ( + child_id INTEGER, + parent_id INTEGER, + PRIMARY KEY(child_id, parent_id) + ) WITHOUT ROWID"""); + execute(""" + CREATE TABLE IF NOT EXISTS way_to_relation ( + child_id INTEGER, + parent_id INTEGER, + PRIMARY KEY(child_id, parent_id) + ) WITHOUT ROWID"""); + execute(""" + CREATE TABLE IF NOT EXISTS relation_to_relation ( + child_id INTEGER, + parent_id INTEGER, + PRIMARY KEY(child_id, parent_id) ) WITHOUT ROWID"""); - execute("CREATE INDEX IF NOT EXISTS way_members_node_idx ON way_members (node_id)"); - execute("CREATE INDEX IF NOT EXISTS relation_members_type_ref_idx ON relation_members (type, ref)"); } @Override @@ -183,24 +171,7 @@ public class SqliteOsmMirror implements OsmMirror { private OsmElement.Way parseWay(ResultSet result) { try { - String nodeString = result.getString("nodes"); - LongArrayList nodeIds; - if (nodeString != null && !nodeString.isBlank()) { - String[] nodes = result.getString("nodes").split(","); - long[] nodeIdArray = new long[nodes.length]; - for (int i = 0; i < nodes.length; i++) { - nodeIdArray[i] = Long.parseLong(nodes[i]); - } - nodeIds = LongArrayList.from(nodeIdArray); - } else { - nodeIds = LongArrayList.from(); - } - return new OsmElement.Way( - result.getInt("id"), - parseTags(result.getBytes("tags")), - nodeIds, - OsmElement.Info.forVersion(result.getInt("version")) - ); + return OsmMirrorUtil.decodeWay(result.getLong("id"), result.getBytes("data")); } catch (SQLException e) { throw new RuntimeException(e); } @@ -208,12 +179,7 @@ public class SqliteOsmMirror implements OsmMirror { private OsmElement.Node parseNode(ResultSet result) { try { - return new OsmElement.Node( - result.getInt("id"), - parseTags(result.getBytes("tags")), - result.getLong("location"), - OsmElement.Info.forVersion(result.getInt("version")) - ); + return OsmMirrorUtil.decodeNode(result.getLong("id"), result.getBytes("data")); } catch (SQLException e) { throw new RuntimeException(e); } @@ -221,30 +187,7 @@ public class SqliteOsmMirror implements OsmMirror { private OsmElement.Relation parseRelation(ResultSet result) { try { - String refsString = result.getString("refs"); - List members; - if (refsString != null && !refsString.isBlank()) { - String[] refs = result.getString("refs").split(","); - String[] types = result.getString("types").split(","); - String[] roles = result.getString("roles").split(","); - members = new ArrayList<>(refs.length); - - for (int i = 0; i < refs.length; i++) { - members.add(new OsmElement.Relation.Member( - OsmElement.Type.values()[Integer.parseInt(types[i])], - Long.parseLong(refs[i]), - i >= roles.length ? "" : roles[i] - )); - } - } else { - members = List.of(); - } - return new OsmElement.Relation( - result.getInt("id"), - parseTags(result.getBytes("tags")), - members, - OsmElement.Info.forVersion(result.getInt("version")) - ); + return OsmMirrorUtil.decodeRelation(result.getLong("id"), result.getBytes("data")); } catch (SQLException e) { throw new RuntimeException(e); } @@ -252,21 +195,7 @@ public class SqliteOsmMirror implements OsmMirror { @Override public LongArrayList getParentWaysForNode(long nodeId) { - try ( - var statement = connection.prepareStatement( - """ - SELECT way_id - FROM way_members - WHERE node_id=? - ORDER BY way_id ASC - """) - ) { - statement.setLong(1, nodeId); - var result = statement.executeQuery(); - return parseLongList(result); - } catch (SQLException e) { - throw new IllegalStateException(e); - } + return getParents(nodeId, "node_to_way"); } private LongArrayList parseLongList(ResultSet resultSet) { @@ -283,19 +212,10 @@ public class SqliteOsmMirror implements OsmMirror { @Override public OsmElement.Way getWay(long id) { - try ( - var statement = connection.prepareStatement( - """ - SELECT ways.*, group_concat(way_members.node_id) nodes - FROM ways - INNER JOIN way_members on ways.id=way_members.way_id - WHERE ways.id=? - ORDER BY way_members.`order` ASC - """) - ) { + try (var statement = connection.prepareStatement("SELECT * FROM ways WHERE id=?")) { statement.setLong(1, id); var result = statement.executeQuery(); - return result.next() && result.getLong("id") == id ? parseWay(result) : null; + return result.next() ? parseWay(result) : null; } catch (SQLException e) { throw new IllegalStateException(e); } @@ -303,23 +223,10 @@ public class SqliteOsmMirror implements OsmMirror { @Override public OsmElement.Relation getRelation(long id) { - try ( - var statement = connection.prepareStatement( - """ - SELECT - relations.*, - group_concat(relation_members.role) roles, - group_concat(relation_members.type) types, - group_concat(relation_members.ref) refs - FROM relations - JOIN relation_members on relations.id=relation_members.relation_id - WHERE relations.id=? - ORDER BY relation_members.`order` ASC - """) - ) { + try (var statement = connection.prepareStatement("SELECT * FROM relations WHERE id=?")) { statement.setLong(1, id); var result = statement.executeQuery(); - return result.next() && result.getLong("id") == id ? parseRelation(result) : null; + return result.next() ? parseRelation(result) : null; } catch (SQLException e) { throw new IllegalStateException(e); } @@ -327,21 +234,20 @@ public class SqliteOsmMirror implements OsmMirror { @Override public LongArrayList getParentRelationsForNode(long nodeId) { - return getParentRelations(nodeId, OsmElement.Type.NODE); + return getParents(nodeId, "node_to_relation"); } - private LongArrayList getParentRelations(long nodeId, OsmElement.Type type) { + private LongArrayList getParents(long childId, String table) { try ( var statement = connection.prepareStatement( """ - SELECT relation_id - FROM relation_members - WHERE ref=? AND type=? - ORDER BY relation_id ASC - """) + SELECT parent_id + FROM %s + WHERE child_id=? + ORDER BY parent_id ASC + """.formatted(table)) ) { - statement.setLong(1, nodeId); - statement.setInt(2, type.ordinal()); + statement.setLong(1, childId); var result = statement.executeQuery(); return parseLongList(result); } catch (SQLException e) { @@ -351,12 +257,12 @@ public class SqliteOsmMirror implements OsmMirror { @Override public LongArrayList getParentRelationsForWay(long nodeId) { - return getParentRelations(nodeId, OsmElement.Type.WAY); + return getParents(nodeId, "way_to_relation"); } @Override public LongArrayList getParentRelationsForRelation(long nodeId) { - return getParentRelations(nodeId, OsmElement.Type.RELATION); + return getParents(nodeId, "relation_to_relation"); } private Iterator resultToIterator(ResultSet resultSet, Function parse) { @@ -396,25 +302,8 @@ public class SqliteOsmMirror implements OsmMirror { public CloseableIterator iterator() { try { var nodes = connection.prepareStatement("SELECT * FROM nodes ORDER BY id ASC").executeQuery(); - var ways = connection.prepareStatement( - """ - SELECT ways.*, group_concat(way_members.node_id) nodes - FROM ways - JOIN way_members on ways.id=way_members.way_id - GROUP BY ways.id - ORDER BY ways.id ASC, way_members.`order` ASC - """).executeQuery(); - var relations = connection.prepareStatement(""" - SELECT - relations.*, - group_concat(relation_members.role) roles, - group_concat(relation_members.type) types, - group_concat(relation_members.ref) refs - FROM relations - JOIN relation_members on relations.id=relation_members.relation_id - GROUP BY relations.id - ORDER BY relations.id ASC, relation_members.`order` ASC - """).executeQuery(); + var ways = connection.prepareStatement("SELECT * FROM ways ORDER BY id ASC").executeQuery(); + var relations = connection.prepareStatement("SELECT * FROM relations ORDER BY id ASC").executeQuery(); var nodesIter = resultToIterator(nodes, this::parseNode); var waysIter = resultToIterator(ways, this::parseWay); var relsIter = resultToIterator(relations, this::parseRelation); @@ -452,34 +341,56 @@ public class SqliteOsmMirror implements OsmMirror { } } - private record ParentChild

(P parent, int idx) {} + private record ParentChild(long child, long parent) {} private class Bulk implements BulkWriter { - private final NodeWriter nodeWriter = new NodeWriter(connection); - private final WayWriter wayWriter = new WayWriter(connection); - private final WayMembersWriter wayMemberWriter = new WayMembersWriter(connection); - private final RelationWriter relationWriter = new RelationWriter(connection); - private final RelationMembersWriter relationMemberWriter = new RelationMembersWriter(connection); + private final ElementWriter nodeWriter = new ElementWriter(connection, "nodes"); + private final ElementWriter wayWriter = new ElementWriter(connection, "ways"); + private final ChildToParentWriter wayMemberWriter = new ChildToParentWriter(connection, "node_to_way"); + private final ElementWriter relationWriter = new ElementWriter(connection, "relations"); + private final ChildToParentWriter nodeToRelWriter = new ChildToParentWriter(connection, "node_to_relation"); + private final ChildToParentWriter wayToRelWriter = new ChildToParentWriter(connection, "way_to_relation"); + private final ChildToParentWriter relToRelWriter = new ChildToParentWriter(connection, "relation_to_relation"); @Override - public void putNode(OsmElement.Node node) { + public void putNode(Serialized.Node node) { nodeWriter.write(node); } @Override - public void putWay(OsmElement.Way way) { + public void putWay(Serialized.Way way) { wayWriter.write(way); // TODO write way members separately, then insert in order into table in close - for (int i = 0; i < way.nodes().size(); i++) { - wayMemberWriter.write(new ParentChild<>(way, i)); + var nodes = way.item().nodes(); + LongSet written = new LongHashSet(); + for (int i = 0; i < nodes.size(); i++) { + long id = nodes.get(i); + if (written.add(id)) { + wayMemberWriter.write(new ParentChild(nodes.get(i), way.item().id())); + } } } @Override - public void putRelation(OsmElement.Relation relation) { + public void putRelation(Serialized.Relation relation) { relationWriter.write(relation); - for (int i = 0; i < relation.members().size(); i++) { - relationMemberWriter.write(new ParentChild<>(relation, i)); + LongHashSet nodes = new LongHashSet(); + LongHashSet ways = new LongHashSet(); + LongHashSet rels = new LongHashSet(); + for (int i = 0; i < relation.item().members().size(); i++) { + var member = relation.item().members().get(i); + var set = (switch (member.type()) { + case NODE -> nodes; + case WAY -> ways; + case RELATION -> rels; + }); + if (set.add(member.ref())) { + (switch (member.type()) { + case NODE -> nodeToRelWriter; + case WAY -> wayToRelWriter; + case RELATION -> relToRelWriter; + }).write(new ParentChild(member.ref(), relation.item().id())); + } } } @@ -490,91 +401,37 @@ public class SqliteOsmMirror implements OsmMirror { wayWriter.close(); wayMemberWriter.close(); relationWriter.close(); - relationMemberWriter.close(); + nodeToRelWriter.close(); + wayToRelWriter.close(); + relToRelWriter.close(); } } - private class NodeWriter extends Mbtiles.BatchedTableWriterBase { - NodeWriter(Connection conn) { - super("nodes", List.of("id", "version", "tags", "location"), INSERT_IGNORE, conn); - } - - @Override - protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, OsmElement.Node item) - throws SQLException { - statement.setLong(positionOffset++, item.id()); - statement.setInt(positionOffset++, item.info().version()); - statement.setBytes(positionOffset++, encodeTags(item.tags())); - statement.setLong(positionOffset++, item.encodedLocation()); - return positionOffset; - } - } - - private class WayWriter extends Mbtiles.BatchedTableWriterBase { - WayWriter(Connection conn) { - super("ways", List.of("id", "version", "tags"), INSERT_IGNORE, conn); - } - - @Override - protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, OsmElement.Way item) - throws SQLException { - statement.setLong(positionOffset++, item.id()); - statement.setInt(positionOffset++, item.info().version()); - statement.setBytes(positionOffset++, encodeTags(item.tags())); - return positionOffset; - } - } - - private class RelationWriter extends Mbtiles.BatchedTableWriterBase { - RelationWriter(Connection conn) { - super("relations", List.of("id", "version", "tags"), INSERT_IGNORE, conn); - } - - @Override - protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, OsmElement.Relation item) - throws SQLException { - statement.setLong(positionOffset++, item.id()); - statement.setInt(positionOffset++, item.info().version()); - statement.setBytes(positionOffset++, encodeTags(item.tags())); - return positionOffset; - } - } - - private class RelationMembersWriter - extends Mbtiles.BatchedTableWriterBase> { - RelationMembersWriter(Connection conn) { - super("relation_members", List.of("relation_id", "`order`", "type", "version", "role", "ref"), INSERT_IGNORE, - conn); + private class ElementWriter extends Mbtiles.BatchedTableWriterBase> { + ElementWriter(Connection conn, String table) { + super(table, List.of("id", "data"), INSERT_IGNORE, conn); } @Override protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, - ParentChild item) throws SQLException { - var relation = item.parent; - var member = item.parent.members().get(item.idx); - statement.setLong(positionOffset++, relation.id()); - statement.setInt(positionOffset++, item.idx); - statement.setInt(positionOffset++, member.type().ordinal()); - statement.setInt(positionOffset++, relation.info().version()); - statement.setString(positionOffset++, member.role()); - statement.setLong(positionOffset++, member.ref()); + Serialized item) + throws SQLException { + statement.setLong(positionOffset++, item.item().id()); + statement.setBytes(positionOffset++, item.bytes()); return positionOffset; } } - private class WayMembersWriter extends Mbtiles.BatchedTableWriterBase> { - WayMembersWriter(Connection conn) { - super("way_members", List.of("way_id", "`order`", "node_id", "version"), INSERT_IGNORE, conn); + private class ChildToParentWriter extends Mbtiles.BatchedTableWriterBase { + ChildToParentWriter(Connection conn, String table) { + super(table, List.of("child_id", "parent_id"), INSERT_IGNORE, conn); } @Override protected int setParamsInStatementForItem(int positionOffset, PreparedStatement statement, - ParentChild item) throws SQLException { - var way = item.parent; - statement.setLong(positionOffset++, way.id()); - statement.setInt(positionOffset++, item.idx); - statement.setLong(positionOffset++, way.nodes().get(item.idx)); - statement.setInt(positionOffset++, way.info().version()); + ParentChild item) throws SQLException { + statement.setLong(positionOffset++, item.child()); + statement.setLong(positionOffset++, item.parent()); return positionOffset; } } diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/osmmirror/OsmMirrorTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/osmmirror/OsmMirrorTest.java index f9ab92df..4deda67f 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/osmmirror/OsmMirrorTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/osmmirror/OsmMirrorTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.io.TempDir; abstract class OsmMirrorTest { OsmMirror fixture; + protected boolean serializeId = true; @Test void testEmpty() { @@ -32,7 +33,7 @@ abstract class OsmMirrorTest { void testInsertNode() throws IOException { var node = new OsmElement.Node(1, Map.of("key", "value"), 2, 3, infoVersion(0)); try (var writer = fixture.newBulkWriter()) { - writer.putNode(node); + writer.putNode(new Serialized.Node(node, serializeId)); } assertEquals(node, fixture.getNode(1)); assertEquals(List.of(node), fixture.iterator().toList()); @@ -41,10 +42,10 @@ abstract class OsmMirrorTest { @Test void testInsertWay() throws IOException { var way = new OsmElement.Way(2, Map.of("key", "value"), LongArrayList.from(1), infoVersion(0)); - var way2 = new OsmElement.Way(3, Map.of("key", "value"), LongArrayList.from(1), infoVersion(0)); + var way2 = new OsmElement.Way(3, Map.of("key", "value"), LongArrayList.from(1, 1), infoVersion(0)); try (var writer = fixture.newBulkWriter()) { - writer.putWay(way); - writer.putWay(way2); + writer.putWay(new Serialized.Way(way, serializeId)); + writer.putWay(new Serialized.Way(way2, serializeId)); } assertEquals(way, fixture.getWay(2)); assertEquals(way2, fixture.getWay(3)); @@ -63,8 +64,8 @@ abstract class OsmMirrorTest { new OsmElement.Relation.Member(OsmElement.Type.RELATION, 3, "rel") ), infoVersion(0)); try (var writer = fixture.newBulkWriter()) { - writer.putRelation(rel); - writer.putRelation(rel2); + writer.putRelation(new Serialized.Relation(rel, serializeId)); + writer.putRelation(new Serialized.Relation(rel2, serializeId)); } assertEquals(rel, fixture.getRelation(3)); assertEquals(rel2, fixture.getRelation(4)); @@ -86,9 +87,11 @@ abstract class OsmMirrorTest { } static class SqliteTest extends OsmMirrorTest { + @BeforeEach void setup() { fixture = OsmMirror.newSqliteMemory(); + serializeId = false; } } static class MapdbTest extends OsmMirrorTest {