kopia lustrzana https://github.com/onthegomap/planetiler
iterator tweaks
rodzic
1c6ae5bee2
commit
ed97a45958
|
@ -54,6 +54,8 @@
|
|||
<package name="" withSubpackages="true" static="false" />
|
||||
</value>
|
||||
</option>
|
||||
<option name="NEW_LINE_AFTER_LPAREN_IN_RECORD_HEADER" value="true" />
|
||||
<option name="RPAREN_ON_NEW_LINE_IN_RECORD_HEADER" value="true" />
|
||||
</JavaCodeStyleSettings>
|
||||
<Objective-C>
|
||||
<option name="INDENT_NAMESPACE_MEMBERS" value="0" />
|
||||
|
@ -578,9 +580,7 @@
|
|||
<codeStyleSettings language="protobuf">
|
||||
<option name="RIGHT_MARGIN" value="80" />
|
||||
<indentOptions>
|
||||
<option name="INDENT_SIZE" value="2" />
|
||||
<option name="CONTINUATION_INDENT_SIZE" value="2" />
|
||||
<option name="TAB_SIZE" value="2" />
|
||||
</indentOptions>
|
||||
</codeStyleSettings>
|
||||
</code_scheme>
|
||||
|
|
|
@ -18,6 +18,8 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
public class MbtilesWriter {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class);
|
||||
|
||||
private final AtomicLong featuresProcessed = new AtomicLong(0);
|
||||
private final AtomicLong memoizedTiles = new AtomicLong(0);
|
||||
private final AtomicLong tiles = new AtomicLong(0);
|
||||
|
@ -27,8 +29,6 @@ public class MbtilesWriter {
|
|||
this.stats = stats;
|
||||
}
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MbtilesWriter.class);
|
||||
|
||||
private static record RenderedTile(TileCoord tile, byte[] contents) {
|
||||
|
||||
}
|
||||
|
@ -39,7 +39,7 @@ public class MbtilesWriter {
|
|||
MbtilesWriter writer = new MbtilesWriter(config.stats());
|
||||
|
||||
var topology = Topology.start("mbtiles", stats)
|
||||
.readFromIterator("reader", features.getAll())
|
||||
.readFrom("reader", features)
|
||||
.addBuffer("reader_queue", 50_000, 1_000)
|
||||
.addWorker("encoder", config.threads(), writer::tileEncoder)
|
||||
.addBuffer("writer_queue", 50_000, 1_000)
|
||||
|
|
|
@ -3,6 +3,7 @@ package com.onthegomap.flatmap.collections;
|
|||
import com.carrotsearch.hppc.LongArrayList;
|
||||
import com.onthegomap.flatmap.RenderedFeature;
|
||||
import com.onthegomap.flatmap.VectorTileEncoder;
|
||||
import com.onthegomap.flatmap.collections.MergeSortFeatureMap.TileFeatures;
|
||||
import com.onthegomap.flatmap.geo.TileCoord;
|
||||
import com.onthegomap.flatmap.monitoring.Stats;
|
||||
import java.nio.file.Path;
|
||||
|
@ -11,7 +12,7 @@ import java.util.Iterator;
|
|||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class MergeSortFeatureMap implements Consumer<RenderedFeature> {
|
||||
public class MergeSortFeatureMap implements Consumer<RenderedFeature>, Iterable<TileFeatures> {
|
||||
|
||||
private volatile boolean prepared = false;
|
||||
|
||||
|
@ -34,7 +35,8 @@ public class MergeSortFeatureMap implements Consumer<RenderedFeature> {
|
|||
return 0;
|
||||
}
|
||||
|
||||
public Iterator<TileFeatures> getAll() {
|
||||
@Override
|
||||
public Iterator<TileFeatures> iterator() {
|
||||
if (!prepared) {
|
||||
throw new IllegalStateException("Attempting to iterate over features but not prepared yet");
|
||||
}
|
||||
|
|
|
@ -91,7 +91,8 @@ public record Topology<T>(
|
|||
return fromGenerator(name, producer, 1);
|
||||
}
|
||||
|
||||
public <T> Bufferable<?, T> readFromIterator(String name, Iterator<T> iter) {
|
||||
public <T> Bufferable<?, T> readFrom(String name, Iterable<T> iterable) {
|
||||
Iterator<T> iter = iterable.iterator();
|
||||
return fromGenerator(name, next -> {
|
||||
while (iter.hasNext()) {
|
||||
next.accept(iter.next());
|
||||
|
|
|
@ -18,14 +18,14 @@ public class MergeSortFeatureMapTest {
|
|||
public void testEmpty() {
|
||||
var features = new MergeSortFeatureMap(tmpDir, new InMemory());
|
||||
features.sort();
|
||||
assertFalse(features.getAll().hasNext());
|
||||
assertFalse(features.iterator().hasNext());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testThrowsWhenPreparedOutOfOrder() {
|
||||
var features = new MergeSortFeatureMap(tmpDir, new InMemory());
|
||||
features.accept(new RenderedFeature(1, new byte[]{}));
|
||||
assertThrows(IllegalStateException.class, features::getAll);
|
||||
assertThrows(IllegalStateException.class, features::iterator);
|
||||
features.sort();
|
||||
assertThrows(IllegalStateException.class, () -> features.accept(new RenderedFeature(1, new byte[]{})));
|
||||
}
|
||||
|
|
|
@ -71,7 +71,7 @@ public class TopologyTest {
|
|||
public void testTopologyFromIterator() {
|
||||
Set<Integer> result = Collections.synchronizedSet(new TreeSet<>());
|
||||
var topology = Topology.start("test", stats)
|
||||
.readFromIterator("reader", List.of(0, 1).iterator())
|
||||
.readFrom("reader", List.of(0, 1))
|
||||
.addBuffer("reader_queue", 1)
|
||||
.<Integer>addWorker("process", 1, (prev, next) -> {
|
||||
Integer item;
|
||||
|
|
Ładowanie…
Reference in New Issue