2023-04-06 10:20:47 +00:00
|
|
|
package com.onthegomap.planetiler.osmmirror;
|
|
|
|
|
|
|
|
import static com.onthegomap.planetiler.worker.Worker.joinFutures;
|
|
|
|
|
|
|
|
import com.carrotsearch.hppc.LongArrayList;
|
|
|
|
import com.onthegomap.planetiler.config.Arguments;
|
|
|
|
import com.onthegomap.planetiler.reader.osm.OsmBlockSource;
|
|
|
|
import com.onthegomap.planetiler.reader.osm.OsmElement;
|
|
|
|
import com.onthegomap.planetiler.reader.osm.OsmInputFile;
|
|
|
|
import com.onthegomap.planetiler.reader.osm.OsmPhaser;
|
|
|
|
import com.onthegomap.planetiler.stats.ProgressLoggers;
|
|
|
|
import com.onthegomap.planetiler.util.CloseableIterator;
|
|
|
|
import com.onthegomap.planetiler.util.FileUtils;
|
|
|
|
import com.onthegomap.planetiler.worker.WorkQueue;
|
|
|
|
import com.onthegomap.planetiler.worker.WorkerPipeline;
|
|
|
|
import java.io.Closeable;
|
|
|
|
import java.nio.file.Path;
|
|
|
|
import java.time.Duration;
|
2023-04-07 09:10:40 +00:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.concurrent.CompletableFuture;
|
2023-04-06 10:20:47 +00:00
|
|
|
import java.util.concurrent.atomic.AtomicLong;
|
|
|
|
import org.msgpack.core.MessagePack;
|
|
|
|
|
|
|
|
public interface OsmMirror extends AutoCloseable {
|
|
|
|
|
|
|
|
static OsmMirror newInMemory() {
|
|
|
|
return new InMemoryOsmMirror();
|
|
|
|
}
|
|
|
|
|
2023-04-08 09:51:50 +00:00
|
|
|
static OsmMirror newSqliteWrite(Path path, int maxWorkers) {
|
|
|
|
return SqliteOsmMirror.newWriteToFileDatabase(path, Arguments.of(), maxWorkers);
|
2023-04-06 10:20:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static OsmMirror newSqliteMemory() {
|
|
|
|
return SqliteOsmMirror.newInMemoryDatabase();
|
|
|
|
}
|
|
|
|
|
|
|
|
static OsmMirror newDummyWriter() {
|
|
|
|
return new DummyOsmMirror();
|
|
|
|
}
|
|
|
|
|
2023-04-07 09:10:40 +00:00
|
|
|
static void main(String[] args) throws Exception {
|
2023-04-06 10:20:47 +00:00
|
|
|
Arguments arguments = Arguments.fromEnvOrArgs(args);
|
|
|
|
var stats = arguments.getStats();
|
|
|
|
String type = arguments.getString("db", "type of db to use", "mapdb");
|
2023-04-07 09:10:40 +00:00
|
|
|
boolean doNodes = arguments.getBoolean("nodes", "process nodes", true);
|
|
|
|
boolean doWays = arguments.getBoolean("ways", "process ways", true);
|
|
|
|
boolean doRelations = arguments.getBoolean("relations", "process relations", true);
|
2023-04-06 10:20:47 +00:00
|
|
|
Path input = arguments.inputFile("input", "input", Path.of("data/sources/massachusetts.osm.pbf"));
|
|
|
|
Path output = arguments.file("output", "output", Path.of("data/tmp/output"));
|
|
|
|
FileUtils.delete(output);
|
|
|
|
FileUtils.createDirectory(output);
|
|
|
|
int processThreads = arguments.threads();
|
|
|
|
OsmInputFile in = new OsmInputFile(input);
|
|
|
|
var blockCounter = new AtomicLong();
|
2023-04-07 10:16:05 +00:00
|
|
|
boolean id = !type.contains("sqlite");
|
2023-04-08 09:51:50 +00:00
|
|
|
try (
|
|
|
|
var blocks = in.get();
|
|
|
|
OsmMirror out = newWriter(type, output.resolve("test.db"), processThreads);
|
|
|
|
var writer = out.newBulkWriter()
|
|
|
|
) {
|
2023-04-07 09:10:40 +00:00
|
|
|
record Batch(CompletableFuture<List<Serialized<? extends OsmElement>>> results, OsmBlockSource.Block block) {}
|
2023-04-06 10:20:47 +00:00
|
|
|
var queue = new WorkQueue<Batch>("batches", processThreads * 2, 1, stats);
|
|
|
|
|
2023-04-07 09:10:40 +00:00
|
|
|
var phaser = new OsmPhaser(1);
|
2023-04-06 10:20:47 +00:00
|
|
|
var pipeline = WorkerPipeline.start("osm2sqlite", stats);
|
|
|
|
var readBranch = pipeline.<Batch>fromGenerator("pbf", next -> {
|
|
|
|
blocks.forEachBlock(block -> {
|
2023-04-07 09:10:40 +00:00
|
|
|
var result = new Batch(new CompletableFuture<>(), block);
|
2023-04-06 10:20:47 +00:00
|
|
|
queue.accept(result);
|
|
|
|
next.accept(result);
|
|
|
|
});
|
|
|
|
})
|
|
|
|
.addBuffer("pbf_blocks", processThreads * 2)
|
|
|
|
.sinkTo("parse", processThreads, (prev) -> {
|
|
|
|
try (queue; var packer = MessagePack.newDefaultBufferPacker()) {
|
|
|
|
for (var batch : prev) {
|
2023-04-07 09:10:40 +00:00
|
|
|
List<Serialized<? extends OsmElement>> result = new ArrayList<>();
|
|
|
|
for (var item : batch.block) {
|
|
|
|
packer.clear();
|
|
|
|
if (item instanceof OsmElement.Node node) {
|
|
|
|
if (doNodes) {
|
2023-04-07 10:16:05 +00:00
|
|
|
OsmMirrorUtil.pack(packer, node, id);
|
2023-04-07 09:10:40 +00:00
|
|
|
result.add(new Serialized.Node(node, packer.toByteArray()));
|
|
|
|
}
|
|
|
|
} else if (item instanceof OsmElement.Way way) {
|
|
|
|
if (doWays) {
|
2023-04-07 10:16:05 +00:00
|
|
|
OsmMirrorUtil.pack(packer, way, id);
|
2023-04-07 09:10:40 +00:00
|
|
|
result.add(new Serialized.Way(way, packer.toByteArray()));
|
|
|
|
}
|
|
|
|
} else if (item instanceof OsmElement.Relation relation) {
|
|
|
|
if (doRelations) {
|
2023-04-07 10:16:05 +00:00
|
|
|
OsmMirrorUtil.pack(packer, relation, id);
|
2023-04-07 09:10:40 +00:00
|
|
|
result.add(new Serialized.Relation(relation, packer.toByteArray()));
|
2023-04-06 10:20:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-04-07 09:10:40 +00:00
|
|
|
batch.results.complete(result);
|
2023-04-06 10:20:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
var writeBranch = pipeline.readFromQueue(queue)
|
|
|
|
.sinkTo("write", 1, prev -> {
|
|
|
|
|
2023-04-08 09:51:50 +00:00
|
|
|
try (var phaserForWorker = phaser.forWorker()) {
|
2023-04-06 10:20:47 +00:00
|
|
|
System.err.println("Using " + out.getClass().getSimpleName());
|
|
|
|
for (var batch : prev) {
|
2023-04-07 09:10:40 +00:00
|
|
|
for (var item : batch.results.get()) {
|
|
|
|
if (item instanceof Serialized.Node node) {
|
2023-04-06 10:20:47 +00:00
|
|
|
phaserForWorker.arrive(OsmPhaser.Phase.NODES);
|
|
|
|
writer.putNode(node);
|
2023-04-07 09:10:40 +00:00
|
|
|
} else if (item instanceof Serialized.Way way) {
|
2023-04-06 10:20:47 +00:00
|
|
|
phaserForWorker.arrive(OsmPhaser.Phase.WAYS);
|
|
|
|
writer.putWay(way);
|
2023-04-07 09:10:40 +00:00
|
|
|
} else if (item instanceof Serialized.Relation relation) {
|
2023-04-06 10:20:47 +00:00
|
|
|
phaserForWorker.arrive(OsmPhaser.Phase.RELATIONS);
|
|
|
|
writer.putRelation(relation);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
blockCounter.incrementAndGet();
|
|
|
|
}
|
|
|
|
phaserForWorker.arrive(OsmPhaser.Phase.DONE);
|
|
|
|
}
|
|
|
|
phaser.printSummary();
|
|
|
|
});
|
|
|
|
|
|
|
|
ProgressLoggers loggers = ProgressLoggers.create()
|
|
|
|
.addRateCounter("blocks", blockCounter)
|
2023-04-07 09:10:40 +00:00
|
|
|
.addRateCounter("nodes", phaser::nodes, true)
|
|
|
|
.addRateCounter("ways", phaser::ways, true)
|
|
|
|
.addRateCounter("rels", phaser::relations, true)
|
2023-04-06 10:20:47 +00:00
|
|
|
.addFileSize(() -> FileUtils.size(output))
|
|
|
|
.newLine()
|
|
|
|
.addProcessStats()
|
|
|
|
.newLine()
|
|
|
|
.addPipelineStats(readBranch)
|
|
|
|
.addPipelineStats(writeBranch);
|
|
|
|
|
|
|
|
loggers.awaitAndLog(joinFutures(readBranch.done(), writeBranch.done()), Duration.ofSeconds(10));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-08 09:51:50 +00:00
|
|
|
static OsmMirror newWriter(String type, Path path, int maxWorkers) {
|
2023-04-06 10:20:47 +00:00
|
|
|
return switch (type) {
|
|
|
|
case "mapdb" -> newMapdbWrite(path);
|
|
|
|
case "mapdb-memory" -> newMapdbMemory();
|
2023-04-08 09:51:50 +00:00
|
|
|
case "sqlite" -> newSqliteWrite(path, maxWorkers);
|
2023-04-06 10:20:47 +00:00
|
|
|
case "sqlite-memory" -> newSqliteMemory();
|
|
|
|
case "memory" -> newInMemory();
|
|
|
|
case "lmdb" -> newLmdbWrite(path);
|
2023-04-07 09:10:40 +00:00
|
|
|
case "dummy" -> newDummyWriter();
|
|
|
|
default -> throw new IllegalArgumentException("Unrecognized type: " + type);
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2023-04-08 09:51:50 +00:00
|
|
|
static OsmMirror newReader(String type, Path path, int maxWorkers) {
|
2023-04-07 09:10:40 +00:00
|
|
|
return switch (type) {
|
|
|
|
case "mapdb" -> MapDbOsmMirror.newReadFromFile(path);
|
2023-04-08 09:51:50 +00:00
|
|
|
case "sqlite" -> newSqliteWrite(path, maxWorkers);
|
2023-04-07 09:10:40 +00:00
|
|
|
case "lmdb" -> newLmdbWrite(path);
|
2023-04-06 10:20:47 +00:00
|
|
|
default -> throw new IllegalArgumentException("Unrecognized type: " + type);
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
static OsmMirror newMapdbWrite(Path file) {
|
|
|
|
return MapDbOsmMirror.newWriteToFile(file);
|
|
|
|
}
|
|
|
|
|
|
|
|
static OsmMirror newMapdbMemory() {
|
|
|
|
return MapDbOsmMirror.newInMemory();
|
|
|
|
}
|
|
|
|
|
|
|
|
static OsmMirror newLmdbWrite(Path file) {
|
|
|
|
return new LmdbOsmMirror(file);
|
|
|
|
}
|
|
|
|
|
|
|
|
interface BulkWriter extends Closeable {
|
|
|
|
|
2023-04-07 10:16:05 +00:00
|
|
|
void putNode(Serialized.Node node);
|
2023-04-06 10:20:47 +00:00
|
|
|
|
2023-04-07 10:16:05 +00:00
|
|
|
void putWay(Serialized.Way way);
|
2023-04-06 10:20:47 +00:00
|
|
|
|
2023-04-07 10:16:05 +00:00
|
|
|
void putRelation(Serialized.Relation node);
|
2023-04-06 10:20:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BulkWriter newBulkWriter();
|
|
|
|
|
|
|
|
void deleteNode(long nodeId, int version);
|
|
|
|
|
|
|
|
void deleteWay(long wayId, int version);
|
|
|
|
|
|
|
|
void deleteRelation(long relationId, int version);
|
|
|
|
|
|
|
|
OsmElement.Node getNode(long id);
|
|
|
|
|
|
|
|
LongArrayList getParentWaysForNode(long nodeId);
|
|
|
|
|
|
|
|
OsmElement.Way getWay(long id);
|
|
|
|
|
|
|
|
OsmElement.Relation getRelation(long id);
|
|
|
|
|
|
|
|
LongArrayList getParentRelationsForNode(long nodeId);
|
|
|
|
|
|
|
|
LongArrayList getParentRelationsForWay(long nodeId);
|
|
|
|
|
|
|
|
LongArrayList getParentRelationsForRelation(long nodeId);
|
|
|
|
|
|
|
|
CloseableIterator<OsmElement> iterator();
|
|
|
|
}
|